@browserbasehq/orca 3.1.0-patch.4 → 3.2.0-preview.1

package/dist/cjs/lib/v3/agent/tools/extract.d.ts

@@ -1,6 +1,6 @@
 import type { V3 } from "../../v3.js";
 import type { AgentModelConfig } from "../../types/public/agent.js";
-export declare const extractTool: (v3: V3, executionModel?: string | AgentModelConfig) => import("ai").Tool<{
+export declare const extractTool: (v3: V3, executionModel?: string | AgentModelConfig, toolTimeout?: number) => import("ai").Tool<{
     instruction: string;
     schema?: {
         [x: string]: unknown;
@@ -16,6 +16,6 @@ export declare const extractTool: (v3: V3, executionModel?: string | AgentModelC
     error?: undefined;
 } | {
     success: boolean;
-    error: string;
+    error: any;
     result?: undefined;
 }>;

package/dist/cjs/lib/v3/agent/tools/extract.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.extractTool = void 0;
 const ai_1 = require("ai");
 const zod_1 = require("zod");
+const sdkErrors_js_1 = require("../../types/public/sdkErrors.js");
 function jsonSchemaToZod(schema) {
     switch (schema.type) {
         case "object": {
@@ -39,7 +40,7 @@ function jsonSchemaToZod(schema) {
             return zod_1.z.any();
     }
 }
-const extractTool = (v3, executionModel) => (0, ai_1.tool)({
+const extractTool = (v3, executionModel, toolTimeout) => (0, ai_1.tool)({
     description: `Extract structured data from the current page based on a provided schema.
     
     USAGE GUIDELINES:
@@ -84,12 +85,24 @@ const extractTool = (v3, executionModel) => (0, ai_1.tool)({
                 : undefined;
             const result = await v3.extract(instruction, parsedSchema, {
                 ...(executionModel ? { model: executionModel } : {}),
+                timeout: toolTimeout,
             });
             return { success: true, result };
         }
         catch (error) {
-            const err = error;
-            return { success: false, error: err?.message ?? String(error) };
+            if (error instanceof sdkErrors_js_1.TimeoutError) {
+                const timeoutMessage = `TimeoutError: extract() timed out — try using a smaller or simpler schema`;
+                v3.logger({
+                    category: "agent",
+                    message: timeoutMessage,
+                    level: 0,
+                });
+                return {
+                    success: false,
+                    error: timeoutMessage,
+                };
+            }
+            return { success: false, error: error?.message ?? String(error) };
         }
     },
 });

package/dist/cjs/lib/v3/agent/tools/extract.js.map

@@ -1 +1 @@
-{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/extract.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAoC;AAYpC,SAAS,eAAe,CAAC,MAAkB;IACzC,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,KAAK,GAA+B,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;gBACtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC7D,KAAK,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;gBACtC,CAAC;YACH,CAAC;YACD,OAAO,OAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;QACD,KAAK,OAAO;YACV,OAAO,OAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QACzE,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,IAAI,CAAC,GAAG,OAAC,CAAC,MAAM,EAAE,CAAC;YACnB,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK;gBAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO;gBAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM;gBAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAC3C,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;gBACvC,OAAO,OAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAA6B,CAAC,CAAC;YACtD,OAAO,CAAC,CAAC;QACX,CAAC;QACD,KAAK,QAAQ,CAAC;QACd,KAAK,SAAS;YACZ,OAAO,OAAC,CAAC,MAAM,EAAE,CAAC;QACpB,KAAK,SAAS;YACZ,OAAO,OAAC,CAAC,OAAO,EAAE,CAAC;QACrB,KAAK,MAAM;YACT,OAAO,OAAC,CAAC,IAAI,EAAE,CAAC;QAClB;YACE,OAAO,OAAC,CAAC,GAAG,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAEM,MAAM,WAAW,GAAG,CACzB,EAAM,EACN,cAA0C,EAC1C,EAAE,CACF,IAAA,SAAI,EAAC;IACH,WAAW,EAAE;;;;;;;;;;;;;;;;;;;;;;0FAsByE;IACtF,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,WAAW,EAAE,OAAC,CAAC,MAAM,EAAE;QACvB,MAAM,EAAE,OAAC;aACN,MAAM,CAAC;YACN,IAAI,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC3B,UAAU,EAAE,OAAC,CAAC,MAAM,CAAC,OAAC,CAAC,MAAM,EAAE,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;YACxD,KAAK,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;YAC7B,IAAI,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;YACpC,MAAM,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;SACpD,CAAC;aACD,WAAW,EAAE;aACb,QAAQ,EAAE;aACV,QAAQ,CAAC,wDAAwD,CAAC;KACtE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE;QACzC,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,MAAM;gBACzB,CAAC,CAAC,eAAe,CAAC,MAAoB,CAAC;gBACvC,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,YAAY,EAAE;gBACzD,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACrD,CAAC,CAAC;YACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,GAAG,GAAG,KAAc,CAAC;YAC3B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAClE,CAAC;IACH,CAAC;CACF,CAAC,CAAC;AAxDQ,QAAA,WAAW,eAwDnB","sourcesContent":["import { tool } from \"ai\";\nimport { z, ZodTypeAny } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { AgentModelConfig } from \"../../types/public/agent.js\";\n\ninterface JsonSchema {\n  type?: string;\n  properties?: Record<string, JsonSchema>;\n  items?: JsonSchema;\n  enum?: string[];\n  format?: \"url\" | \"email\" | \"uuid\";\n}\n\nfunction jsonSchemaToZod(schema: JsonSchema): ZodTypeAny {\n  switch (schema.type) {\n    case \"object\": {\n      const shape: Record<string, ZodTypeAny> = {};\n      if (schema.properties) {\n        for (const [key, value] of Object.entries(schema.properties)) {\n          shape[key] = jsonSchemaToZod(value);\n        }\n      }\n      return z.object(shape);\n    }\n    case \"array\":\n      return z.array(schema.items ? jsonSchemaToZod(schema.items) : z.any());\n    case \"string\": {\n      let s = z.string();\n      if (schema.format === \"url\") s = s.url();\n      if (schema.format === \"email\") s = s.email();\n      if (schema.format === \"uuid\") s = s.uuid();\n      if (schema.enum && schema.enum.length > 0)\n        return z.enum(schema.enum as [string, ...string[]]);\n      return s;\n    }\n    case \"number\":\n    case \"integer\":\n      return z.number();\n    case \"boolean\":\n      return z.boolean();\n    case \"null\":\n      return z.null();\n    default:\n      return z.any();\n  }\n}\n\nexport const extractTool = (\n  v3: V3,\n  executionModel?: string | AgentModelConfig,\n) =>\n  tool({\n    description: `Extract structured data from the current page based on a provided schema.\n    \n    USAGE GUIDELINES:\n    - Keep schemas MINIMAL - only include fields essential for the task\n    - IMPORTANT: only use this if explicitly asked for structured output. In most scenarios, you should use the aria tree tool over this.\n    - For URL fields, use format: \"url\"\n    \n    EXAMPLES:\n    1. Extract a single value:\n       instruction: \"extract the product price\"\n       schema: { type: \"object\", properties: { price: { type: \"number\" } } }\n    \n    2. Extract multiple fields:\n       instruction: \"extract product name and price\"\n       schema: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } }\n    \n    3. Extract arrays:\n       instruction: \"extract all product names and prices\"\n       schema: { type: \"object\", properties: { products: { type: \"array\", items: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } } } } }\n    \n    4. Extract a URL:\n       instruction: \"extract the link\"\n       schema: { type: \"object\", properties: { url: { type: \"string\", format: \"url\" } } }`,\n    inputSchema: z.object({\n      instruction: z.string(),\n      schema: z\n        .object({\n          type: z.string().optional(),\n          properties: z.record(z.string(), z.unknown()).optional(),\n          items: z.unknown().optional(),\n          enum: z.array(z.string()).optional(),\n          format: z.enum([\"url\", \"email\", \"uuid\"]).optional(),\n        })\n        .passthrough()\n        .optional()\n        .describe(\"JSON Schema object describing the structure to extract\"),\n    }),\n    execute: async ({ instruction, schema }) => {\n      try {\n        const parsedSchema = schema\n          ? jsonSchemaToZod(schema as JsonSchema)\n          : undefined;\n        const result = await v3.extract(instruction, parsedSchema, {\n          ...(executionModel ? { model: executionModel } : {}),\n        });\n        return { success: true, result };\n      } catch (error) {\n        const err = error as Error;\n        return { success: false, error: err?.message ?? String(error) };\n      }\n    },\n  });\n"]}
+{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/extract.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAoC;AAGpC,kEAA+D;AAU/D,SAAS,eAAe,CAAC,MAAkB;IACzC,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,KAAK,GAA+B,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;gBACtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC7D,KAAK,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;gBACtC,CAAC;YACH,CAAC;YACD,OAAO,OAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;QACD,KAAK,OAAO;YACV,OAAO,OAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QACzE,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,IAAI,CAAC,GAAG,OAAC,CAAC,MAAM,EAAE,CAAC;YACnB,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK;gBAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO;gBAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM;gBAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAC3C,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;gBACvC,OAAO,OAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAA6B,CAAC,CAAC;YACtD,OAAO,CAAC,CAAC;QACX,CAAC;QACD,KAAK,QAAQ,CAAC;QACd,KAAK,SAAS;YACZ,OAAO,OAAC,CAAC,MAAM,EAAE,CAAC;QACpB,KAAK,SAAS;YACZ,OAAO,OAAC,CAAC,OAAO,EAAE,CAAC;QACrB,KAAK,MAAM;YACT,OAAO,OAAC,CAAC,IAAI,EAAE,CAAC;QAClB;YACE,OAAO,OAAC,CAAC,GAAG,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAEM,MAAM,WAAW,GAAG,CACzB,EAAM,EACN,cAA0C,EAC1C,WAAoB,EACpB,EAAE,CACF,IAAA,SAAI,EAAC;IACH,WAAW,EAAE;;;;;;;;;;;;;;;;;;;;;;0FAsByE;IACtF,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,WAAW,EAAE,OAAC,CAAC,MAAM,EAAE;QACvB,MAAM,EAAE,OAAC;aACN,MAAM,CAAC;YACN,IAAI,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC3B,UAAU,EAAE,OAAC,CAAC,MAAM,CAAC,OAAC,CAAC,MAAM,EAAE,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;YACxD,KAAK,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;YAC7B,IAAI,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;YACpC,MAAM,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;SACpD,CAAC;aACD,WAAW,EAAE;aACb,QAAQ,EAAE;aACV,QAAQ,CAAC,wDAAwD,CAAC;KACtE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE;QACzC,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,MAAM;gBACzB,CAAC,CAAC,eAAe,CAAC,MAAoB,CAAC;gBACvC,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,YAAY,EAAE;gBACzD,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpD,OAAO,EAAE,WAAW;aACrB,CAAC,CAAC;YACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,2BAAY,EAAE,CAAC;gBAClC,MAAM,cAAc,GAAG,2EAA2E,CAAC;gBACnG,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,cAAc;oBACvB,KAAK,EAAE,CAAC;iBACT,CAAC,CAAC;gBACH,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,cAAc;iBACtB,CAAC;YACJ,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QACpE,CAAC;IACH,CAAC;CACF,CAAC,CAAC;AArEQ,QAAA,WAAW,eAqEnB","sourcesContent":["import { tool } from \"ai\";\nimport { z, ZodTypeAny } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { AgentModelConfig } from \"../../types/public/agent.js\";\nimport { TimeoutError } from \"../../types/public/sdkErrors.js\";\n\ninterface JsonSchema {\n  type?: string;\n  properties?: Record<string, JsonSchema>;\n  items?: JsonSchema;\n  enum?: string[];\n  format?: \"url\" | \"email\" | \"uuid\";\n}\n\nfunction jsonSchemaToZod(schema: JsonSchema): ZodTypeAny {\n  switch (schema.type) {\n    case \"object\": {\n      const shape: Record<string, ZodTypeAny> = {};\n      if (schema.properties) {\n        for (const [key, value] of Object.entries(schema.properties)) {\n          shape[key] = jsonSchemaToZod(value);\n        }\n      }\n      return z.object(shape);\n    }\n    case \"array\":\n      return z.array(schema.items ? jsonSchemaToZod(schema.items) : z.any());\n    case \"string\": {\n      let s = z.string();\n      if (schema.format === \"url\") s = s.url();\n      if (schema.format === \"email\") s = s.email();\n      if (schema.format === \"uuid\") s = s.uuid();\n      if (schema.enum && schema.enum.length > 0)\n        return z.enum(schema.enum as [string, ...string[]]);\n      return s;\n    }\n    case \"number\":\n    case \"integer\":\n      return z.number();\n    case \"boolean\":\n      return z.boolean();\n    case \"null\":\n      return z.null();\n    default:\n      return z.any();\n  }\n}\n\nexport const extractTool = (\n  v3: V3,\n  executionModel?: string | AgentModelConfig,\n  toolTimeout?: number,\n) =>\n  tool({\n    description: `Extract structured data from the current page based on a provided schema.\n    \n    USAGE GUIDELINES:\n    - Keep schemas MINIMAL - only include fields essential for the task\n    - IMPORTANT: only use this if explicitly asked for structured output. In most scenarios, you should use the aria tree tool over this.\n    - For URL fields, use format: \"url\"\n    \n    EXAMPLES:\n    1. Extract a single value:\n       instruction: \"extract the product price\"\n       schema: { type: \"object\", properties: { price: { type: \"number\" } } }\n    \n    2. Extract multiple fields:\n       instruction: \"extract product name and price\"\n       schema: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } }\n    \n    3. Extract arrays:\n       instruction: \"extract all product names and prices\"\n       schema: { type: \"object\", properties: { products: { type: \"array\", items: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } } } } }\n    \n    4. Extract a URL:\n       instruction: \"extract the link\"\n       schema: { type: \"object\", properties: { url: { type: \"string\", format: \"url\" } } }`,\n    inputSchema: z.object({\n      instruction: z.string(),\n      schema: z\n        .object({\n          type: z.string().optional(),\n          properties: z.record(z.string(), z.unknown()).optional(),\n          items: z.unknown().optional(),\n          enum: z.array(z.string()).optional(),\n          format: z.enum([\"url\", \"email\", \"uuid\"]).optional(),\n        })\n        .passthrough()\n        .optional()\n        .describe(\"JSON Schema object describing the structure to extract\"),\n    }),\n    execute: async ({ instruction, schema }) => {\n      try {\n        const parsedSchema = schema\n          ? jsonSchemaToZod(schema as JsonSchema)\n          : undefined;\n        const result = await v3.extract(instruction, parsedSchema, {\n          ...(executionModel ? { model: executionModel } : {}),\n          timeout: toolTimeout,\n        });\n        return { success: true, result };\n      } catch (error) {\n        if (error instanceof TimeoutError) {\n          const timeoutMessage = `TimeoutError: extract() timed out — try using a smaller or simpler schema`;\n          v3.logger({\n            category: \"agent\",\n            message: timeoutMessage,\n            level: 0,\n          });\n          return {\n            success: false,\n            error: timeoutMessage,\n          };\n        }\n        return { success: false, error: error?.message ?? String(error) };\n      }\n    },\n  });\n"]}

package/dist/cjs/lib/v3/agent/tools/fillFormVision.js

@@ -120,37 +120,37 @@ MANDATORY USE CASES (always use fillFormVision for these):
             }
         },
         toModelOutput: (result) => {
-            if (result.success) {
-                const content = [
-                    {
-                        type: "text",
-                        text: JSON.stringify({
-                            success: result.success,
-                            fieldsCount: result.playwrightArguments?.length ?? 0,
-                        }),
-                    },
-                ];
-                if (result.screenshotBase64) {
-                    content.push({
-                        type: "media",
-                        mediaType: "image/png",
-                        data: result.screenshotBase64,
-                    });
-                }
-                return { type: "content", value: content };
+            if (result.success === false || result.error !== undefined) {
+                return {
+                    type: "content",
+                    value: [
+                        {
+                            type: "text",
+                            text: JSON.stringify({
+                                success: result.success,
+                                error: result.error,
+                            }),
+                        },
+                    ],
+                };
             }
-            return {
-                type: "content",
-                value: [
-                    {
-                        type: "text",
-                        text: JSON.stringify({
-                            success: result.success,
-                            error: result.error,
-                        }),
-                    },
-                ],
-            };
+            const content = [
+                {
+                    type: "text",
+                    text: JSON.stringify({
+                        success: result.success,
+                        fieldsCount: result.playwrightArguments?.length ?? 0,
+                    }),
+                },
+            ];
+            if (result.screenshotBase64) {
+                content.push({
+                    type: "media",
+                    mediaType: "image/png",
+                    data: result.screenshotBase64,
+                });
+            }
+            return { type: "content", value: content };
         },
     });
 };

package/dist/cjs/lib/v3/agent/tools/fillFormVision.js.map

@@ -1 +1 @@
-{"version":3,"file":"fillFormVision.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillFormVision.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAQxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AACzE,wDAA4D;AAErD,MAAM,kBAAkB,GAAG,CAChC,EAAM,EACN,QAAiB,EACjB,SAAqB,EACrB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,qGAAqG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAC1I,CAAC,CAAC,oCAAoC,CAAC;IAEzC,OAAO,IAAA,SAAI,EAAC;QACV,WAAW,EAAE;;;;;;;;;;;;;;;;2CAgB0B;QACvC,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,OAAC;iBACN,KAAK,CACJ,OAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,OAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,sEAAsE,CACvE;gBACH,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBAC5C,WAAW,EAAE,OAAC;qBACX,MAAM,CAAC;oBACN,CAAC,EAAE,OAAC,CAAC,MAAM,EAAE;oBACb,CAAC,EAAE,OAAC,CAAC,MAAM,EAAE;iBACd,CAAC;qBACD,QAAQ,CAAC,iCAAiC,CAAC;aAC/C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,qCAAqC,CAAC;SACjD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAqC,EAAE;YAC/D,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAEhD,8DAA8D;gBAC9D,0FAA0F;gBAC1F,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;oBAC3C,MAAM,SAAS,GAAG,IAAA,+CAAkB,EAClC,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;oBACF,OAAO;wBACL,GAAG,KAAK;wBACR,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE,wCAAwC;wBACpE,KAAK,EAAE,IAAA,kCAAmB,EAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC;wBAClD,WAAW,EAAE,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;qBAChD,CAAC;gBACJ,CAAC,CAAC,CAAC;gBAEH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,oCAAoC;oBAC7C,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,+BAA+B;4BAClE,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,OAAO,GAAa,EAAE,CAAC;gBAE7B,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;oBACpC,iEAAiE;oBACjE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAC5B,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB;wBACE,WAAW,EAAE,kBAAkB;qBAChC,CACF,CAAC;oBACF,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;oBAE7B,uEAAuE;oBACvE,qFAAqF;oBACrF,IAAI,kBAAkB,EAAE,CAAC;wBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;wBAC3C,IAAI,eAAe,EAAE,CAAC;4BACpB,OAAO,CAAC,IAAI,CAAC;gCACX,QAAQ,EAAE,eAAe;gCACzB,WAAW,EAAE,KAAK,CAAC,MAAM;gCACzB,MAAM,EAAE,MAAM;gCACd,SAAS,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC;6BACjC,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,6BAA6B;oBAC7B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;gBAC3D,CAAC;gBAED,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAEnE,wFAAwF;gBACxF,IAAI,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7C,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;wBAChD,OAAO;wBACP,iBAAiB,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;qBACvD,CAAC,CAAC;gBACL,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,mBAAmB,EAAE,eAAe;oBACpC,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,uBAAwB,KAAe,CAAC,OAAO,EAAE;iBACzD,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,MAAM,OAAO,GAA6B;oBACxC;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;4BACnB,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,WAAW,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,IAAI,CAAC;yBACrD,CAAC;qBACH;iBACF,CAAC;gBACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;oBAC5B,OAAO,CAAC,IAAI,CAAC;wBACX,IAAI,EAAE,OAAO;wBACb,SAAS,EAAE,WAAW;wBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;qBAC9B,CAAC,CAAC;gBACL,CAAC;gBACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;YAC7C,CAAC;YACD,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE;oBACL;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;4BACnB,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,KAAK,EAAE,MAAM,CAAC,KAAK;yBACpB,CAAC;qBACH;iBACF;aACF,CAAC;QACJ,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC;AA5KW,QAAA,kBAAkB,sBA4K7B","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  FillFormVisionToolResult,\n  ModelOutputContentItem,\n  Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const fillFormVisionTool = (\n  v3: V3,\n  provider?: string,\n  variables?: Variables,\n) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const valueDescription = hasVariables\n    ? `Text to type into the target field. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"Text to type into the target field\";\n\n  return tool({\n    description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL\n\nCRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)\nIMPORTANT: Ensure the fields are visible within the current viewport\n\nWHY THIS TOOL EXISTS:\n- Forms are the #1 use case for multi-field input\n- Optimized specifically for input/textarea elements\n- 4-6x faster than individual typing actions\n\nUse fillFormVision: Pure form filling (inputs, textareas only)\nMANDATORY USE CASES (always use fillFormVision for these):\n- Registration forms: name, email, password fields\n- Contact forms: name, email, message fields\n- Checkout forms: address, payment info fields\n- Profile updates: multiple user data fields\n- Search filters: multiple criteria inputs`,\n    inputSchema: z.object({\n      fields: z\n        .array(\n          z.object({\n            action: z\n              .string()\n              .describe(\n                \"Description of the typing action, e.g. 'type foo into the bar field'\",\n              ),\n            value: z.string().describe(valueDescription),\n            coordinates: z\n              .object({\n                x: z.number(),\n                y: z.number(),\n              })\n              .describe(\"Coordinates of the target field\"),\n          }),\n        )\n        .min(2, \"Provide at least two fields to fill\"),\n    }),\n    execute: async ({ fields }): Promise<FillFormVisionToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n\n        // Process coordinates and substitute variables for each field\n        // Keep original values (with %tokens%) for logging/caching, substituted values for typing\n        const processedFields = fields.map((field) => {\n          const processed = processCoordinates(\n            field.coordinates.x,\n            field.coordinates.y,\n            provider,\n            v3,\n          );\n          return {\n            ...field,\n            originalValue: field.value, // Keep original with %tokens% for cache\n            value: substituteVariables(field.value, variables),\n            coordinates: { x: processed.x, y: processed.y },\n          };\n        });\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: fillFormVision`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ fields }), // Don't log substituted values\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const actions: Action[] = [];\n\n        for (const field of processedFields) {\n          // Click the field, only requesting XPath when caching is enabled\n          const xpath = await page.click(\n            field.coordinates.x,\n            field.coordinates.y,\n            {\n              returnXpath: shouldCollectXpath,\n            },\n          );\n          await page.type(field.value);\n\n          // Build Action with XPath for deterministic replay (only when caching)\n          // Use originalValue (with %tokens%) so cache stores references, not sensitive values\n          if (shouldCollectXpath) {\n            const normalizedXpath = ensureXPath(xpath);\n            if (normalizedXpath) {\n              actions.push({\n                selector: normalizedXpath,\n                description: field.action,\n                method: \"type\",\n                arguments: [field.originalValue],\n              });\n            }\n          }\n\n          // Small delay between fields\n          await new Promise((resolve) => setTimeout(resolve, 100));\n        }\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n        // Record as \"act\" step with proper Actions for deterministic replay (only when caching)\n        if (shouldCollectXpath && actions.length > 0) {\n          v3.recordAgentReplayStep({\n            type: \"act\",\n            instruction: `Fill ${fields.length} form fields`,\n            actions,\n            actionDescription: `Fill ${fields.length} form fields`,\n          });\n        }\n\n        return {\n          success: true,\n          playwrightArguments: processedFields,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error filling form: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success) {\n        const content: ModelOutputContentItem[] = [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              fieldsCount: result.playwrightArguments?.length ?? 0,\n            }),\n          },\n        ];\n        if (result.screenshotBase64) {\n          content.push({\n            type: \"media\",\n            mediaType: \"image/png\",\n            data: result.screenshotBase64,\n          });\n        }\n        return { type: \"content\", value: content };\n      }\n      return {\n        type: \"content\",\n        value: [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              error: result.error,\n            }),\n          },\n        ],\n      };\n    },\n  });\n};\n"]}
+{"version":3,"file":"fillFormVision.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillFormVision.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAQxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AACzE,wDAA4D;AAErD,MAAM,kBAAkB,GAAG,CAChC,EAAM,EACN,QAAiB,EACjB,SAAqB,EACrB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,qGAAqG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAC1I,CAAC,CAAC,oCAAoC,CAAC;IAEzC,OAAO,IAAA,SAAI,EAAC;QACV,WAAW,EAAE;;;;;;;;;;;;;;;;2CAgB0B;QACvC,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,OAAC;iBACN,KAAK,CACJ,OAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,OAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,sEAAsE,CACvE;gBACH,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBAC5C,WAAW,EAAE,OAAC;qBACX,MAAM,CAAC;oBACN,CAAC,EAAE,OAAC,CAAC,MAAM,EAAE;oBACb,CAAC,EAAE,OAAC,CAAC,MAAM,EAAE;iBACd,CAAC;qBACD,QAAQ,CAAC,iCAAiC,CAAC;aAC/C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,qCAAqC,CAAC;SACjD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAqC,EAAE;YAC/D,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAEhD,8DAA8D;gBAC9D,0FAA0F;gBAC1F,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;oBAC3C,MAAM,SAAS,GAAG,IAAA,+CAAkB,EAClC,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;oBACF,OAAO;wBACL,GAAG,KAAK;wBACR,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE,wCAAwC;wBACpE,KAAK,EAAE,IAAA,kCAAmB,EAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC;wBAClD,WAAW,EAAE,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;qBAChD,CAAC;gBACJ,CAAC,CAAC,CAAC;gBAEH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,oCAAoC;oBAC7C,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,+BAA+B;4BAClE,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,OAAO,GAAa,EAAE,CAAC;gBAE7B,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;oBACpC,iEAAiE;oBACjE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAC5B,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB;wBACE,WAAW,EAAE,kBAAkB;qBAChC,CACF,CAAC;oBACF,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;oBAE7B,uEAAuE;oBACvE,qFAAqF;oBACrF,IAAI,kBAAkB,EAAE,CAAC;wBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;wBAC3C,IAAI,eAAe,EAAE,CAAC;4BACpB,OAAO,CAAC,IAAI,CAAC;gCACX,QAAQ,EAAE,eAAe;gCACzB,WAAW,EAAE,KAAK,CAAC,MAAM;gCACzB,MAAM,EAAE,MAAM;gCACd,SAAS,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC;6BACjC,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,6BAA6B;oBAC7B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;gBAC3D,CAAC;gBAED,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAEnE,wFAAwF;gBACxF,IAAI,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7C,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;wBAChD,OAAO;wBACP,iBAAiB,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;qBACvD,CAAC,CAAC;gBACL,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,mBAAmB,EAAE,eAAe;oBACpC,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,uBAAwB,KAAe,CAAC,OAAO,EAAE;iBACzD,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE;wBACL;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gCACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gCACvB,KAAK,EAAE,MAAM,CAAC,KAAK;6BACpB,CAAC;yBACH;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,WAAW,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,IAAI,CAAC;qBACrD,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC;AA7KW,QAAA,kBAAkB,sBA6K7B","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  FillFormVisionToolResult,\n  ModelOutputContentItem,\n  Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const fillFormVisionTool = (\n  v3: V3,\n  provider?: string,\n  variables?: Variables,\n) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const valueDescription = hasVariables\n    ? `Text to type into the target field. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"Text to type into the target field\";\n\n  return tool({\n    description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL\n\nCRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)\nIMPORTANT: Ensure the fields are visible within the current viewport\n\nWHY THIS TOOL EXISTS:\n- Forms are the #1 use case for multi-field input\n- Optimized specifically for input/textarea elements\n- 4-6x faster than individual typing actions\n\nUse fillFormVision: Pure form filling (inputs, textareas only)\nMANDATORY USE CASES (always use fillFormVision for these):\n- Registration forms: name, email, password fields\n- Contact forms: name, email, message fields\n- Checkout forms: address, payment info fields\n- Profile updates: multiple user data fields\n- Search filters: multiple criteria inputs`,\n    inputSchema: z.object({\n      fields: z\n        .array(\n          z.object({\n            action: z\n              .string()\n              .describe(\n                \"Description of the typing action, e.g. 'type foo into the bar field'\",\n              ),\n            value: z.string().describe(valueDescription),\n            coordinates: z\n              .object({\n                x: z.number(),\n                y: z.number(),\n              })\n              .describe(\"Coordinates of the target field\"),\n          }),\n        )\n        .min(2, \"Provide at least two fields to fill\"),\n    }),\n    execute: async ({ fields }): Promise<FillFormVisionToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n\n        // Process coordinates and substitute variables for each field\n        // Keep original values (with %tokens%) for logging/caching, substituted values for typing\n        const processedFields = fields.map((field) => {\n          const processed = processCoordinates(\n            field.coordinates.x,\n            field.coordinates.y,\n            provider,\n            v3,\n          );\n          return {\n            ...field,\n            originalValue: field.value, // Keep original with %tokens% for cache\n            value: substituteVariables(field.value, variables),\n            coordinates: { x: processed.x, y: processed.y },\n          };\n        });\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: fillFormVision`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ fields }), // Don't log substituted values\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const actions: Action[] = [];\n\n        for (const field of processedFields) {\n          // Click the field, only requesting XPath when caching is enabled\n          const xpath = await page.click(\n            field.coordinates.x,\n            field.coordinates.y,\n            {\n              returnXpath: shouldCollectXpath,\n            },\n          );\n          await page.type(field.value);\n\n          // Build Action with XPath for deterministic replay (only when caching)\n          // Use originalValue (with %tokens%) so cache stores references, not sensitive values\n          if (shouldCollectXpath) {\n            const normalizedXpath = ensureXPath(xpath);\n            if (normalizedXpath) {\n              actions.push({\n                selector: normalizedXpath,\n                description: field.action,\n                method: \"type\",\n                arguments: [field.originalValue],\n              });\n            }\n          }\n\n          // Small delay between fields\n          await new Promise((resolve) => setTimeout(resolve, 100));\n        }\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n        // Record as \"act\" step with proper Actions for deterministic replay (only when caching)\n        if (shouldCollectXpath && actions.length > 0) {\n          v3.recordAgentReplayStep({\n            type: \"act\",\n            instruction: `Fill ${fields.length} form fields`,\n            actions,\n            actionDescription: `Fill ${fields.length} form fields`,\n          });\n        }\n\n        return {\n          success: true,\n          playwrightArguments: processedFields,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error filling form: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [\n            {\n              type: \"text\",\n              text: JSON.stringify({\n                success: result.success,\n                error: result.error,\n              }),\n            },\n          ],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            fieldsCount: result.playwrightArguments?.length ?? 0,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n};\n"]}

package/dist/cjs/lib/v3/agent/tools/fillform.d.ts

@@ -1,7 +1,7 @@
 import type { V3 } from "../../v3.js";
 import type { Action } from "../../types/public/methods.js";
 import type { AgentModelConfig, Variables } from "../../types/public/agent.js";
-export declare const fillFormTool: (v3: V3, executionModel?: string | AgentModelConfig, variables?: Variables) => import("ai").Tool<{
+export declare const fillFormTool: (v3: V3, executionModel?: string | AgentModelConfig, variables?: Variables, toolTimeout?: number) => import("ai").Tool<{
     fields: {
         action: string;
         value: string;
@@ -10,4 +10,10 @@ export declare const fillFormTool: (v3: V3, executionModel?: string | AgentModel
     success: boolean;
     actions: unknown[];
     playwrightArguments: Action[];
+    error?: undefined;
+} | {
+    success: boolean;
+    error: any;
+    actions?: undefined;
+    playwrightArguments?: undefined;
 }>;

package/dist/cjs/lib/v3/agent/tools/fillform.js

@@ -3,7 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.fillFormTool = void 0;
 const ai_1 = require("ai");
 const zod_1 = require("zod");
-const fillFormTool = (v3, executionModel, variables) => {
+const sdkErrors_js_1 = require("../../types/public/sdkErrors.js");
+const fillFormTool = (v3, executionModel, variables, toolTimeout) => {
     const hasVariables = variables && Object.keys(variables).length > 0;
     const valueDescription = hasVariables
         ? `Text to type into the target. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(", ")}`
@@ -21,45 +22,67 @@ const fillFormTool = (v3, executionModel, variables) => {
                 .min(1, "Provide at least one field to fill"),
         }),
         execute: async ({ fields }) => {
-            v3.logger({
-                category: "agent",
-                message: `Agent calling tool: fillForm`,
-                level: 1,
-                auxiliary: {
-                    arguments: {
-                        value: JSON.stringify(fields),
-                        type: "object",
+            try {
+                v3.logger({
+                    category: "agent",
+                    message: `Agent calling tool: fillForm`,
+                    level: 1,
+                    auxiliary: {
+                        arguments: {
+                            value: JSON.stringify(fields),
+                            type: "object",
+                        },
                     },
-                },
-            });
-            const instruction = `Return observation results for the following actions: ${fields
-                .map((f) => f.action)
-                .join(", ")}`;
-            const observeOptions = executionModel
-                ? { model: executionModel }
-                : undefined;
-            const observeResults = await v3.observe(instruction, observeOptions);
-            const completed = [];
-            const replayableActions = [];
-            for (const res of observeResults) {
-                const actOptions = variables ? { variables } : undefined;
-                const actResult = await v3.act(res, actOptions);
-                completed.push(actResult);
-                if (Array.isArray(actResult.actions)) {
-                    replayableActions.push(...actResult.actions);
+                });
+                const instruction = `Return observation results for the following actions: ${fields
+                    .map((f) => f.action)
+                    .join(", ")}`;
+                const observeOptions = executionModel
+                    ? { model: executionModel, timeout: toolTimeout }
+                    : { timeout: toolTimeout };
+                const observeResults = await v3.observe(instruction, observeOptions);
+                const completed = [];
+                const replayableActions = [];
+                for (const res of observeResults) {
+                    const actOptions = variables
+                        ? { variables, timeout: toolTimeout }
+                        : { timeout: toolTimeout };
+                    const actResult = await v3.act(res, actOptions);
+                    completed.push(actResult);
+                    if (Array.isArray(actResult.actions)) {
+                        replayableActions.push(...actResult.actions);
+                    }
                 }
+                v3.recordAgentReplayStep({
+                    type: "fillForm",
+                    fields,
+                    observeResults,
+                    actions: replayableActions,
+                });
+                return {
+                    success: true,
+                    actions: completed,
+                    playwrightArguments: replayableActions,
+                };
+            }
+            catch (error) {
+                if (error instanceof sdkErrors_js_1.TimeoutError) {
+                    const timeoutMessage = `TimeoutError while waiting for fillForm() to complete (it may continue executing in the background)`;
+                    v3.logger({
+                        category: "agent",
+                        message: timeoutMessage,
+                        level: 0,
+                    });
+                    return {
+                        success: false,
+                        error: `${timeoutMessage} — try filling fewer fields at once or use a different tool`,
+                    };
+                }
+                return {
+                    success: false,
+                    error: error?.message ?? String(error),
+                };
             }
-            v3.recordAgentReplayStep({
-                type: "fillForm",
-                fields,
-                observeResults,
-                actions: replayableActions,
-            });
-            return {
-                success: true,
-                actions: completed,
-                playwrightArguments: replayableActions,
-            };
         },
     });
 };

package/dist/cjs/lib/v3/agent/tools/fillform.js.map

@@ -1 +1 @@
-{"version":3,"file":"fillform.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillform.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAKjB,MAAM,YAAY,GAAG,CAC1B,EAAM,EACN,cAA0C,EAC1C,SAAqB,EACrB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,+FAA+F,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACpI,CAAC,CAAC,8BAA8B,CAAC;IAEnC,OAAO,IAAA,SAAI,EAAC;QACV,WAAW,EAAE,8GAA8G;QAC3H,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,OAAC;iBACN,KAAK,CACJ,OAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,OAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,oEAAoE,CACrE;gBACH,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;aAC7C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,oCAAoC,CAAC;SAChD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;YAC5B,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,8BAA8B;gBACvC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;wBAC7B,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YACH,MAAM,WAAW,GAAG,yDAAyD,MAAM;iBAChF,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;iBACpB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAEhB,MAAM,cAAc,GAAG,cAAc;gBACnC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE;gBAC3B,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,cAAc,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;YAErE,MAAM,SAAS,GAAG,EAAe,CAAC;YAClC,MAAM,iBAAiB,GAAa,EAAE,CAAC;YACvC,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;gBACjC,MAAM,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;gBACzD,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;gBAChD,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;oBACrC,iBAAiB,CAAC,IAAI,CAAC,GAAI,SAAS,CAAC,OAAoB,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC;YACD,EAAE,CAAC,qBAAqB,CAAC;gBACvB,IAAI,EAAE,UAAU;gBAChB,MAAM;gBACN,cAAc;gBACd,OAAO,EAAE,iBAAiB;aAC3B,CAAC,CAAC;YACH,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE,SAAS;gBAClB,mBAAmB,EAAE,iBAAiB;aACvC,CAAC;QACJ,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC;AAtEW,QAAA,YAAY,gBAsEvB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type { AgentModelConfig, Variables } from \"../../types/public/agent.js\";\n\nexport const fillFormTool = (\n  v3: V3,\n  executionModel?: string | AgentModelConfig,\n  variables?: Variables,\n) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const valueDescription = hasVariables\n    ? `Text to type into the target. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"Text to type into the target\";\n\n  return tool({\n    description: `📝 FORM FILL - MULTI-FIELD INPUT TOOL\\nFor any form with 2+ inputs/textareas. Faster than individual typing.`,\n    inputSchema: z.object({\n      fields: z\n        .array(\n          z.object({\n            action: z\n              .string()\n              .describe(\n                'Description of typing action, e.g. \"type foo into the email field\"',\n              ),\n            value: z.string().describe(valueDescription),\n          }),\n        )\n        .min(1, \"Provide at least one field to fill\"),\n    }),\n    execute: async ({ fields }) => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: fillForm`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: JSON.stringify(fields),\n            type: \"object\",\n          },\n        },\n      });\n      const instruction = `Return observation results for the following actions: ${fields\n        .map((f) => f.action)\n        .join(\", \")}`;\n\n      const observeOptions = executionModel\n        ? { model: executionModel }\n        : undefined;\n      const observeResults = await v3.observe(instruction, observeOptions);\n\n      const completed = [] as unknown[];\n      const replayableActions: Action[] = [];\n      for (const res of observeResults) {\n        const actOptions = variables ? { variables } : undefined;\n        const actResult = await v3.act(res, actOptions);\n        completed.push(actResult);\n        if (Array.isArray(actResult.actions)) {\n          replayableActions.push(...(actResult.actions as Action[]));\n        }\n      }\n      v3.recordAgentReplayStep({\n        type: \"fillForm\",\n        fields,\n        observeResults,\n        actions: replayableActions,\n      });\n      return {\n        success: true,\n        actions: completed,\n        playwrightArguments: replayableActions,\n      };\n    },\n  });\n};\n"]}
+{"version":3,"file":"fillform.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillform.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAIxB,kEAA+D;AAExD,MAAM,YAAY,GAAG,CAC1B,EAAM,EACN,cAA0C,EAC1C,SAAqB,EACrB,WAAoB,EACpB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,+FAA+F,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACpI,CAAC,CAAC,8BAA8B,CAAC;IAEnC,OAAO,IAAA,SAAI,EAAC;QACV,WAAW,EAAE,8GAA8G;QAC3H,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,OAAC;iBACN,KAAK,CACJ,OAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,OAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,oEAAoE,CACrE;gBACH,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;aAC7C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,oCAAoC,CAAC;SAChD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;YAC5B,IAAI,CAAC;gBACH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,8BAA8B;oBACvC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;4BAC7B,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBACH,MAAM,WAAW,GAAG,yDAAyD,MAAM;qBAChF,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;qBACpB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAEhB,MAAM,cAAc,GAAG,cAAc;oBACnC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,WAAW,EAAE;oBACjD,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;gBAC7B,MAAM,cAAc,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;gBAErE,MAAM,SAAS,GAAG,EAAe,CAAC;gBAClC,MAAM,iBAAiB,GAAa,EAAE,CAAC;gBACvC,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;oBACjC,MAAM,UAAU,GAAG,SAAS;wBAC1B,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE;wBACrC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;oBAC7B,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;oBAChD,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;wBACrC,iBAAiB,CAAC,IAAI,CAAC,GAAI,SAAS,CAAC,OAAoB,CAAC,CAAC;oBAC7D,CAAC;gBACH,CAAC;gBACD,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,UAAU;oBAChB,MAAM;oBACN,cAAc;oBACd,OAAO,EAAE,iBAAiB;iBAC3B,CAAC,CAAC;gBACH,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,OAAO,EAAE,SAAS;oBAClB,mBAAmB,EAAE,iBAAiB;iBACvC,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,2BAAY,EAAE,CAAC;oBAClC,MAAM,cAAc,GAAG,qGAAqG,CAAC;oBAC7H,EAAE,CAAC,MAAM,CAAC;wBACR,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,cAAc;wBACvB,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBACH,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,KAAK,EAAE,GAAG,cAAc,6DAA6D;qBACtF,CAAC;gBACJ,CAAC;gBACD,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC;iBACvC,CAAC;YACJ,CAAC;QACH,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC;AA5FW,QAAA,YAAY,gBA4FvB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type { AgentModelConfig, Variables } from \"../../types/public/agent.js\";\nimport { TimeoutError } from \"../../types/public/sdkErrors.js\";\n\nexport const fillFormTool = (\n  v3: V3,\n  executionModel?: string | AgentModelConfig,\n  variables?: Variables,\n  toolTimeout?: number,\n) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const valueDescription = hasVariables\n    ? `Text to type into the target. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"Text to type into the target\";\n\n  return tool({\n    description: `📝 FORM FILL - MULTI-FIELD INPUT TOOL\\nFor any form with 2+ inputs/textareas. Faster than individual typing.`,\n    inputSchema: z.object({\n      fields: z\n        .array(\n          z.object({\n            action: z\n              .string()\n              .describe(\n                'Description of typing action, e.g. \"type foo into the email field\"',\n              ),\n            value: z.string().describe(valueDescription),\n          }),\n        )\n        .min(1, \"Provide at least one field to fill\"),\n    }),\n    execute: async ({ fields }) => {\n      try {\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: fillForm`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify(fields),\n              type: \"object\",\n            },\n          },\n        });\n        const instruction = `Return observation results for the following actions: ${fields\n          .map((f) => f.action)\n          .join(\", \")}`;\n\n        const observeOptions = executionModel\n          ? { model: executionModel, timeout: toolTimeout }\n          : { timeout: toolTimeout };\n        const observeResults = await v3.observe(instruction, observeOptions);\n\n        const completed = [] as unknown[];\n        const replayableActions: Action[] = [];\n        for (const res of observeResults) {\n          const actOptions = variables\n            ? { variables, timeout: toolTimeout }\n            : { timeout: toolTimeout };\n          const actResult = await v3.act(res, actOptions);\n          completed.push(actResult);\n          if (Array.isArray(actResult.actions)) {\n            replayableActions.push(...(actResult.actions as Action[]));\n          }\n        }\n        v3.recordAgentReplayStep({\n          type: \"fillForm\",\n          fields,\n          observeResults,\n          actions: replayableActions,\n        });\n        return {\n          success: true,\n          actions: completed,\n          playwrightArguments: replayableActions,\n        };\n      } catch (error) {\n        if (error instanceof TimeoutError) {\n          const timeoutMessage = `TimeoutError while waiting for fillForm() to complete (it may continue executing in the background)`;\n          v3.logger({\n            category: \"agent\",\n            message: timeoutMessage,\n            level: 0,\n          });\n          return {\n            success: false,\n            error: `${timeoutMessage} — try filling fewer fields at once or use a different tool`,\n          };\n        }\n        return {\n          success: false,\n          error: error?.message ?? String(error),\n        };\n      }\n    },\n  });\n};\n"]}

package/dist/cjs/lib/v3/agent/tools/index.d.ts

@@ -42,6 +42,11 @@ export interface V3AgentToolOptions {
      * When provided, these tools will have an optional useVariable field.
      */
     variables?: Variables;
+    /**
+     * Timeout in milliseconds for tool calls that invoke v3 methods (act, extract, fillForm, ariaTree).
+     * Forwarded to the underlying v3 call's `timeout` option.
+     */
+    toolTimeout?: number;
 }
 export declare function createAgentTools(v3: V3, options?: V3AgentToolOptions): ToolSet;
 export type AgentTools = ReturnType<typeof createAgentTools>;

package/dist/cjs/lib/v3/agent/tools/index.js

@@ -51,14 +51,15 @@ function createAgentTools(v3, options) {
     const provider = options?.provider;
     const excludeTools = options?.excludeTools;
     const variables = options?.variables;
+    const toolTimeout = options?.toolTimeout;
     const allTools = {
-        act: (0, act_js_1.actTool)(v3, executionModel, variables),
-        ariaTree: (0, ariaTree_js_1.ariaTreeTool)(v3),
+        act: (0, act_js_1.actTool)(v3, executionModel, variables, toolTimeout),
+        ariaTree: (0, ariaTree_js_1.ariaTreeTool)(v3, toolTimeout),
         click: (0, click_js_1.clickTool)(v3, provider),
         clickAndHold: (0, clickAndHold_js_1.clickAndHoldTool)(v3, provider),
         dragAndDrop: (0, dragAndDrop_js_1.dragAndDropTool)(v3, provider),
-        extract: (0, extract_js_1.extractTool)(v3, executionModel),
-        fillForm: (0, fillform_js_1.fillFormTool)(v3, executionModel, variables),
+        extract: (0, extract_js_1.extractTool)(v3, executionModel, toolTimeout),
+        fillForm: (0, fillform_js_1.fillFormTool)(v3, executionModel, variables, toolTimeout),
         fillFormVision: (0, fillFormVision_js_1.fillFormVisionTool)(v3, provider, variables),
         goto: (0, goto_js_1.gotoTool)(v3),
         keys: (0, keys_js_1.keysTool)(v3),
@@ -69,7 +70,6 @@ function createAgentTools(v3, options) {
         type: (0, type_js_1.typeTool)(v3, provider, variables),
         wait: (0, wait_js_1.waitTool)(v3, mode),
     };
-    // Only include search tool if BRAVE_API_KEY is configured
     if (process.env.BRAVE_API_KEY) {
         allTools.search = (0, search_js_1.searchTool)(v3);
     }

package/dist/cjs/lib/v3/agent/tools/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/index.ts"],"names":[],"mappings":";;AAsFA,4CAgCC;AAtHD,uCAAqC;AACrC,qCAAmC;AACnC,mDAAiD;AACjD,uCAAqC;AACrC,6CAA2C;AAC3C,+CAA6C;AAC7C,+CAA6C;AAC7C,2CAA2D;AAC3D,6CAA2C;AAC3C,yCAAuC;AACvC,uCAAqC;AACrC,qDAAmD;AACnD,uDAAqD;AACrD,uCAAqC;AACrC,2DAAyD;AACzD,yCAAuC;AACvC,2CAAyC;AAoCzC;;;;;GAKG;AACH,SAAS,WAAW,CAClB,KAAc,EACd,IAAmB,EACnB,YAAuB;IAEvB,MAAM,QAAQ,GAAY,EAAE,GAAG,KAAK,EAAE,CAAC;IAEvC,uBAAuB;IACvB,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC,QAAQ,CAAC;IAC3B,CAAC;SAAM,CAAC;QACN,qBAAqB;QACrB,OAAO,QAAQ,CAAC,KAAK,CAAC;QACtB,OAAO,QAAQ,CAAC,IAAI,CAAC;QACrB,OAAO,QAAQ,CAAC,WAAW,CAAC;QAC5B,OAAO,QAAQ,CAAC,YAAY,CAAC;QAC7B,OAAO,QAAQ,CAAC,cAAc,CAAC;IACjC,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,KAAK,MAAM,QAAQ,IAAI,YAAY,EAAE,CAAC;YACpC,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAgB,gBAAgB,CAAC,EAAM,EAAE,OAA4B;IACnE,MAAM,cAAc,GAAG,OAAO,EAAE,cAAc,CAAC;IAC/C,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,KAAK,CAAC;IACpC,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,CAAC;IACnC,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,CAAC;IAC3C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,CAAC;IAErC,MAAM,QAAQ,GAAY;QACxB,GAAG,EAAE,IAAA,gBAAO,EAAC,EAAE,EAAE,cAAc,EAAE,SAAS,CAAC;QAC3C,QAAQ,EAAE,IAAA,0BAAY,EAAC,EAAE,CAAC;QAC1B,KAAK,EAAE,IAAA,oBAAS,EAAC,EAAE,EAAE,QAAQ,CAAC;QAC9B,YAAY,EAAE,IAAA,kCAAgB,EAAC,EAAE,EAAE,QAAQ,CAAC;QAC5C,WAAW,EAAE,IAAA,gCAAe,EAAC,EAAE,EAAE,QAAQ,CAAC;QAC1C,OAAO,EAAE,IAAA,wBAAW,EAAC,EAAE,EAAE,cAAc,CAAC;QACxC,QAAQ,EAAE,IAAA,0BAAY,EAAC,EAAE,EAAE,cAAc,EAAE,SAAS,CAAC;QACrD,cAAc,EAAE,IAAA,sCAAkB,EAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC;QAC3D,IAAI,EAAE,IAAA,kBAAQ,EAAC,EAAE,CAAC;QAClB,IAAI,EAAE,IAAA,kBAAQ,EAAC,EAAE,CAAC;QAClB,OAAO,EAAE,IAAA,wBAAW,EAAC,EAAE,CAAC;QACxB,UAAU,EAAE,IAAA,8BAAc,EAAC,EAAE,CAAC;QAC9B,MAAM,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAA,4BAAgB,EAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAA,sBAAU,EAAC,EAAE,CAAC;QAC3E,KAAK,EAAE,IAAA,oBAAS,GAAE;QAClB,IAAI,EAAE,IAAA,kBAAQ,EAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC;QACvC,IAAI,EAAE,IAAA,kBAAQ,EAAC,EAAE,EAAE,IAAI,CAAC;KACzB,CAAC;IAEF,0DAA0D;IAC1D,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;QAC9B,QAAQ,CAAC,MAAM,GAAG,IAAA,sBAAU,EAAC,EAAE,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,WAAW,CAAC,QAAQ,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;AACnD,CAAC","sourcesContent":["import { gotoTool } from \"./goto.js\";\nimport { actTool } from \"./act.js\";\nimport { screenshotTool } from \"./screenshot.js\";\nimport { waitTool } from \"./wait.js\";\nimport { navBackTool } from \"./navback.js\";\nimport { ariaTreeTool } from \"./ariaTree.js\";\nimport { fillFormTool } from \"./fillform.js\";\nimport { scrollTool, scrollVisionTool } from \"./scroll.js\";\nimport { extractTool } from \"./extract.js\";\nimport { clickTool } from \"./click.js\";\nimport { typeTool } from \"./type.js\";\nimport { dragAndDropTool } from \"./dragAndDrop.js\";\nimport { clickAndHoldTool } from \"./clickAndHold.js\";\nimport { keysTool } from \"./keys.js\";\nimport { fillFormVisionTool } from \"./fillFormVision.js\";\nimport { thinkTool } from \"./think.js\";\nimport { searchTool } from \"./search.js\";\n\nimport type { ToolSet, InferUITools } from \"ai\";\nimport type { V3 } from \"../../v3.js\";\nimport type { LogLine } from \"../../types/public/logs.js\";\nimport type {\n  AgentToolMode,\n  AgentModelConfig,\n  Variables,\n} from \"../../types/public/agent.js\";\n\nexport interface V3AgentToolOptions {\n  executionModel?: string | AgentModelConfig;\n  logger?: (message: LogLine) => void;\n  /**\n   * Tool mode determines which set of tools are available.\n   * - 'dom' (default): Uses DOM-based tools (act, fillForm) - removes coordinate-based tools\n   * - 'hybrid': Uses coordinate-based tools (click, type, dragAndDrop, etc.) - removes fillForm\n   */\n  mode?: AgentToolMode;\n  /**\n   * The model provider. Used for model-specific coordinate handling\n   */\n  provider?: string;\n  /**\n   * Tools to exclude from the available toolset.\n   * These tools will be filtered out after mode-based filtering.\n   */\n  excludeTools?: string[];\n  /**\n   * Variables available to the agent for use in act/type tools.\n   * When provided, these tools will have an optional useVariable field.\n   */\n  variables?: Variables;\n}\n\n/**\n * Filters tools based on mode and explicit exclusions.\n * - 'dom' mode: Removes coordinate-based tools (click, type, dragAndDrop, clickAndHold, fillFormVision)\n * - 'hybrid' mode: Removes DOM-based form tool (fillForm) in favor of coordinate-based fillFormVision\n * - excludeTools: Additional tools to remove from the toolset\n */\nfunction filterTools(\n  tools: ToolSet,\n  mode: AgentToolMode,\n  excludeTools?: string[],\n): ToolSet {\n  const filtered: ToolSet = { ...tools };\n\n  // Mode-based filtering\n  if (mode === \"hybrid\") {\n    delete filtered.fillForm;\n  } else {\n    // DOM mode (default)\n    delete filtered.click;\n    delete filtered.type;\n    delete filtered.dragAndDrop;\n    delete filtered.clickAndHold;\n    delete filtered.fillFormVision;\n  }\n\n  if (excludeTools) {\n    for (const toolName of excludeTools) {\n      delete filtered[toolName];\n    }\n  }\n\n  return filtered;\n}\n\nexport function createAgentTools(v3: V3, options?: V3AgentToolOptions) {\n  const executionModel = options?.executionModel;\n  const mode = options?.mode ?? \"dom\";\n  const provider = options?.provider;\n  const excludeTools = options?.excludeTools;\n  const variables = options?.variables;\n\n  const allTools: ToolSet = {\n    act: actTool(v3, executionModel, variables),\n    ariaTree: ariaTreeTool(v3),\n    click: clickTool(v3, provider),\n    clickAndHold: clickAndHoldTool(v3, provider),\n    dragAndDrop: dragAndDropTool(v3, provider),\n    extract: extractTool(v3, executionModel),\n    fillForm: fillFormTool(v3, executionModel, variables),\n    fillFormVision: fillFormVisionTool(v3, provider, variables),\n    goto: gotoTool(v3),\n    keys: keysTool(v3),\n    navback: navBackTool(v3),\n    screenshot: screenshotTool(v3),\n    scroll: mode === \"hybrid\" ? scrollVisionTool(v3, provider) : scrollTool(v3),\n    think: thinkTool(),\n    type: typeTool(v3, provider, variables),\n    wait: waitTool(v3, mode),\n  };\n\n  // Only include search tool if BRAVE_API_KEY is configured\n  if (process.env.BRAVE_API_KEY) {\n    allTools.search = searchTool(v3);\n  }\n\n  return filterTools(allTools, mode, excludeTools);\n}\n\nexport type AgentTools = ReturnType<typeof createAgentTools>;\n\n/**\n * Type map of all agent tools for strong typing of tool calls and results.\n * Note: `search` is optional as it's only available when BRAVE_API_KEY is configured.\n */\nexport type AgentToolTypesMap = {\n  act: ReturnType<typeof actTool>;\n  ariaTree: ReturnType<typeof ariaTreeTool>;\n  click: ReturnType<typeof clickTool>;\n  clickAndHold: ReturnType<typeof clickAndHoldTool>;\n  dragAndDrop: ReturnType<typeof dragAndDropTool>;\n  extract: ReturnType<typeof extractTool>;\n  fillForm: ReturnType<typeof fillFormTool>;\n  fillFormVision: ReturnType<typeof fillFormVisionTool>;\n  goto: ReturnType<typeof gotoTool>;\n  keys: ReturnType<typeof keysTool>;\n  navback: ReturnType<typeof navBackTool>;\n  screenshot: ReturnType<typeof screenshotTool>;\n  scroll: ReturnType<typeof scrollTool> | ReturnType<typeof scrollVisionTool>;\n  search?: ReturnType<typeof searchTool>;\n  think: ReturnType<typeof thinkTool>;\n  type: ReturnType<typeof typeTool>;\n  wait: ReturnType<typeof waitTool>;\n};\n\n/**\n * Inferred UI tools type for type-safe tool inputs and outputs.\n * Use with UIMessage for full type safety in UI contexts.\n */\nexport type AgentUITools = InferUITools<AgentToolTypesMap>;\n\n/**\n * Union type for all possible agent tool calls.\n * Provides type-safe access to tool call arguments.\n */\nexport type AgentToolCall = {\n  [K in keyof AgentToolTypesMap]: {\n    toolName: K;\n    toolCallId: string;\n    args: AgentUITools[K][\"input\"];\n  };\n}[keyof AgentToolTypesMap];\n\n/**\n * Union type for all possible agent tool results.\n * Provides type-safe access to tool result values.\n */\nexport type AgentToolResult = {\n  [K in keyof AgentToolTypesMap]: {\n    toolName: K;\n    toolCallId: string;\n    result: AgentUITools[K][\"output\"];\n  };\n}[keyof AgentToolTypesMap];\n"]}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/index.ts"],"names":[],"mappings":";;AA2FA,4CAgCC;AA3HD,uCAAqC;AACrC,qCAAmC;AACnC,mDAAiD;AACjD,uCAAqC;AACrC,6CAA2C;AAC3C,+CAA6C;AAC7C,+CAA6C;AAC7C,2CAA2D;AAC3D,6CAA2C;AAC3C,yCAAuC;AACvC,uCAAqC;AACrC,qDAAmD;AACnD,uDAAqD;AACrD,uCAAqC;AACrC,2DAAyD;AACzD,yCAAuC;AACvC,2CAAyC;AAyCzC;;;;;GAKG;AACH,SAAS,WAAW,CAClB,KAAc,EACd,IAAmB,EACnB,YAAuB;IAEvB,MAAM,QAAQ,GAAY,EAAE,GAAG,KAAK,EAAE,CAAC;IAEvC,uBAAuB;IACvB,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC,QAAQ,CAAC;IAC3B,CAAC;SAAM,CAAC;QACN,qBAAqB;QACrB,OAAO,QAAQ,CAAC,KAAK,CAAC;QACtB,OAAO,QAAQ,CAAC,IAAI,CAAC;QACrB,OAAO,QAAQ,CAAC,WAAW,CAAC;QAC5B,OAAO,QAAQ,CAAC,YAAY,CAAC;QAC7B,OAAO,QAAQ,CAAC,cAAc,CAAC;IACjC,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,KAAK,MAAM,QAAQ,IAAI,YAAY,EAAE,CAAC;YACpC,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAgB,gBAAgB,CAAC,EAAM,EAAE,OAA4B;IACnE,MAAM,cAAc,GAAG,OAAO,EAAE,cAAc,CAAC;IAC/C,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,KAAK,CAAC;IACpC,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,CAAC;IACnC,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,CAAC;IAC3C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,CAAC;IACrC,MAAM,WAAW,GAAG,OAAO,EAAE,WAAW,CAAC;IAEzC,MAAM,QAAQ,GAAY;QACxB,GAAG,EAAE,IAAA,gBAAO,EAAC,EAAE,EAAE,cAAc,EAAE,SAAS,EAAE,WAAW,CAAC;QACxD,QAAQ,EAAE,IAAA,0BAAY,EAAC,EAAE,EAAE,WAAW,CAAC;QACvC,KAAK,EAAE,IAAA,oBAAS,EAAC,EAAE,EAAE,QAAQ,CAAC;QAC9B,YAAY,EAAE,IAAA,kCAAgB,EAAC,EAAE,EAAE,QAAQ,CAAC;QAC5C,WAAW,EAAE,IAAA,gCAAe,EAAC,EAAE,EAAE,QAAQ,CAAC;QAC1C,OAAO,EAAE,IAAA,wBAAW,EAAC,EAAE,EAAE,cAAc,EAAE,WAAW,CAAC;QACrD,QAAQ,EAAE,IAAA,0BAAY,EAAC,EAAE,EAAE,cAAc,EAAE,SAAS,EAAE,WAAW,CAAC;QAClE,cAAc,EAAE,IAAA,sCAAkB,EAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC;QAC3D,IAAI,EAAE,IAAA,kBAAQ,EAAC,EAAE,CAAC;QAClB,IAAI,EAAE,IAAA,kBAAQ,EAAC,EAAE,CAAC;QAClB,OAAO,EAAE,IAAA,wBAAW,EAAC,EAAE,CAAC;QACxB,UAAU,EAAE,IAAA,8BAAc,EAAC,EAAE,CAAC;QAC9B,MAAM,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAA,4BAAgB,EAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAA,sBAAU,EAAC,EAAE,CAAC;QAC3E,KAAK,EAAE,IAAA,oBAAS,GAAE;QAClB,IAAI,EAAE,IAAA,kBAAQ,EAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC;QACvC,IAAI,EAAE,IAAA,kBAAQ,EAAC,EAAE,EAAE,IAAI,CAAC;KACzB,CAAC;IAEF,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;QAC9B,QAAQ,CAAC,MAAM,GAAG,IAAA,sBAAU,EAAC,EAAE,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,WAAW,CAAC,QAAQ,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;AACnD,CAAC","sourcesContent":["import { gotoTool } from \"./goto.js\";\nimport { actTool } from \"./act.js\";\nimport { screenshotTool } from \"./screenshot.js\";\nimport { waitTool } from \"./wait.js\";\nimport { navBackTool } from \"./navback.js\";\nimport { ariaTreeTool } from \"./ariaTree.js\";\nimport { fillFormTool } from \"./fillform.js\";\nimport { scrollTool, scrollVisionTool } from \"./scroll.js\";\nimport { extractTool } from \"./extract.js\";\nimport { clickTool } from \"./click.js\";\nimport { typeTool } from \"./type.js\";\nimport { dragAndDropTool } from \"./dragAndDrop.js\";\nimport { clickAndHoldTool } from \"./clickAndHold.js\";\nimport { keysTool } from \"./keys.js\";\nimport { fillFormVisionTool } from \"./fillFormVision.js\";\nimport { thinkTool } from \"./think.js\";\nimport { searchTool } from \"./search.js\";\n\nimport type { ToolSet, InferUITools } from \"ai\";\nimport type { V3 } from \"../../v3.js\";\nimport type { LogLine } from \"../../types/public/logs.js\";\nimport type {\n  AgentToolMode,\n  AgentModelConfig,\n  Variables,\n} from \"../../types/public/agent.js\";\n\nexport interface V3AgentToolOptions {\n  executionModel?: string | AgentModelConfig;\n  logger?: (message: LogLine) => void;\n  /**\n   * Tool mode determines which set of tools are available.\n   * - 'dom' (default): Uses DOM-based tools (act, fillForm) - removes coordinate-based tools\n   * - 'hybrid': Uses coordinate-based tools (click, type, dragAndDrop, etc.) - removes fillForm\n   */\n  mode?: AgentToolMode;\n  /**\n   * The model provider. Used for model-specific coordinate handling\n   */\n  provider?: string;\n  /**\n   * Tools to exclude from the available toolset.\n   * These tools will be filtered out after mode-based filtering.\n   */\n  excludeTools?: string[];\n  /**\n   * Variables available to the agent for use in act/type tools.\n   * When provided, these tools will have an optional useVariable field.\n   */\n  variables?: Variables;\n  /**\n   * Timeout in milliseconds for tool calls that invoke v3 methods (act, extract, fillForm, ariaTree).\n   * Forwarded to the underlying v3 call's `timeout` option.\n   */\n  toolTimeout?: number;\n}\n\n/**\n * Filters tools based on mode and explicit exclusions.\n * - 'dom' mode: Removes coordinate-based tools (click, type, dragAndDrop, clickAndHold, fillFormVision)\n * - 'hybrid' mode: Removes DOM-based form tool (fillForm) in favor of coordinate-based fillFormVision\n * - excludeTools: Additional tools to remove from the toolset\n */\nfunction filterTools(\n  tools: ToolSet,\n  mode: AgentToolMode,\n  excludeTools?: string[],\n): ToolSet {\n  const filtered: ToolSet = { ...tools };\n\n  // Mode-based filtering\n  if (mode === \"hybrid\") {\n    delete filtered.fillForm;\n  } else {\n    // DOM mode (default)\n    delete filtered.click;\n    delete filtered.type;\n    delete filtered.dragAndDrop;\n    delete filtered.clickAndHold;\n    delete filtered.fillFormVision;\n  }\n\n  if (excludeTools) {\n    for (const toolName of excludeTools) {\n      delete filtered[toolName];\n    }\n  }\n\n  return filtered;\n}\n\nexport function createAgentTools(v3: V3, options?: V3AgentToolOptions) {\n  const executionModel = options?.executionModel;\n  const mode = options?.mode ?? \"dom\";\n  const provider = options?.provider;\n  const excludeTools = options?.excludeTools;\n  const variables = options?.variables;\n  const toolTimeout = options?.toolTimeout;\n\n  const allTools: ToolSet = {\n    act: actTool(v3, executionModel, variables, toolTimeout),\n    ariaTree: ariaTreeTool(v3, toolTimeout),\n    click: clickTool(v3, provider),\n    clickAndHold: clickAndHoldTool(v3, provider),\n    dragAndDrop: dragAndDropTool(v3, provider),\n    extract: extractTool(v3, executionModel, toolTimeout),\n    fillForm: fillFormTool(v3, executionModel, variables, toolTimeout),\n    fillFormVision: fillFormVisionTool(v3, provider, variables),\n    goto: gotoTool(v3),\n    keys: keysTool(v3),\n    navback: navBackTool(v3),\n    screenshot: screenshotTool(v3),\n    scroll: mode === \"hybrid\" ? scrollVisionTool(v3, provider) : scrollTool(v3),\n    think: thinkTool(),\n    type: typeTool(v3, provider, variables),\n    wait: waitTool(v3, mode),\n  };\n\n  if (process.env.BRAVE_API_KEY) {\n    allTools.search = searchTool(v3);\n  }\n\n  return filterTools(allTools, mode, excludeTools);\n}\n\nexport type AgentTools = ReturnType<typeof createAgentTools>;\n\n/**\n * Type map of all agent tools for strong typing of tool calls and results.\n * Note: `search` is optional as it's only available when BRAVE_API_KEY is configured.\n */\nexport type AgentToolTypesMap = {\n  act: ReturnType<typeof actTool>;\n  ariaTree: ReturnType<typeof ariaTreeTool>;\n  click: ReturnType<typeof clickTool>;\n  clickAndHold: ReturnType<typeof clickAndHoldTool>;\n  dragAndDrop: ReturnType<typeof dragAndDropTool>;\n  extract: ReturnType<typeof extractTool>;\n  fillForm: ReturnType<typeof fillFormTool>;\n  fillFormVision: ReturnType<typeof fillFormVisionTool>;\n  goto: ReturnType<typeof gotoTool>;\n  keys: ReturnType<typeof keysTool>;\n  navback: ReturnType<typeof navBackTool>;\n  screenshot: ReturnType<typeof screenshotTool>;\n  scroll: ReturnType<typeof scrollTool> | ReturnType<typeof scrollVisionTool>;\n  search?: ReturnType<typeof searchTool>;\n  think: ReturnType<typeof thinkTool>;\n  type: ReturnType<typeof typeTool>;\n  wait: ReturnType<typeof waitTool>;\n};\n\n/**\n * Inferred UI tools type for type-safe tool inputs and outputs.\n * Use with UIMessage for full type safety in UI contexts.\n */\nexport type AgentUITools = InferUITools<AgentToolTypesMap>;\n\n/**\n * Union type for all possible agent tool calls.\n * Provides type-safe access to tool call arguments.\n */\nexport type AgentToolCall = {\n  [K in keyof AgentToolTypesMap]: {\n    toolName: K;\n    toolCallId: string;\n    args: AgentUITools[K][\"input\"];\n  };\n}[keyof AgentToolTypesMap];\n\n/**\n * Union type for all possible agent tool results.\n * Provides type-safe access to tool result values.\n */\nexport type AgentToolResult = {\n  [K in keyof AgentToolTypesMap]: {\n    toolName: K;\n    toolCallId: string;\n    result: AgentUITools[K][\"output\"];\n  };\n}[keyof AgentToolTypesMap];\n"]}

package/dist/cjs/lib/v3/agent/tools/screenshot.d.ts

@@ -1,6 +1,14 @@
 import type { V3 } from "../../v3.js";
 export declare const screenshotTool: (v3: V3) => import("ai").Tool<Record<string, never>, {
+    success: boolean;
     base64: string;
     timestamp: number;
     pageUrl: string;
+    error?: undefined;
+} | {
+    success: boolean;
+    error: string;
+    base64?: undefined;
+    timestamp?: undefined;
+    pageUrl?: undefined;
 }>;

package/dist/cjs/lib/v3/agent/tools/screenshot.js

@@ -7,24 +7,41 @@ const screenshotTool = (v3) => (0, ai_1.tool)({
     description: "Takes a screenshot (PNG) of the current page. Use this to quickly verify page state.",
     inputSchema: zod_1.z.object({}),
     execute: async () => {
-        v3.logger({
-            category: "agent",
-            message: `Agent calling tool: screenshot`,
-            level: 1,
-        });
-        const page = await v3.context.awaitActivePage();
-        const buffer = await page.screenshot({ fullPage: false });
-        const pageUrl = page.url();
+        try {
+            v3.logger({
+                category: "agent",
+                message: `Agent calling tool: screenshot`,
+                level: 1,
+            });
+            const page = await v3.context.awaitActivePage();
+            const buffer = await page.screenshot({ fullPage: false });
+            const pageUrl = page.url();
+            return {
+                success: true,
+                base64: buffer.toString("base64"),
+                timestamp: Date.now(),
+                pageUrl,
+            };
+        }
+        catch (error) {
+            return {
+                success: false,
+                error: `Error taking screenshot: ${error.message}`,
+            };
+        }
+    },
+    toModelOutput: (result) => {
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
+        }
         return {
-            base64: buffer.toString("base64"),
-            timestamp: Date.now(),
-            pageUrl,
+            type: "content",
+            value: [{ type: "media", mediaType: "image/png", data: result.base64 }],
         };
     },
-    toModelOutput: (result) => ({
-        type: "content",
-        value: [{ type: "media", mediaType: "image/png", data: result.base64 }],
-    }),
 });
 exports.screenshotTool = screenshotTool;
 //# sourceMappingURL=screenshot.js.map

package/dist/cjs/lib/v3/agent/tools/screenshot.js.map

@@ -1 +1 @@
-{"version":3,"file":"screenshot.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/screenshot.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAGjB,MAAM,cAAc,GAAG,CAAC,EAAM,EAAE,EAAE,CACvC,IAAA,SAAI,EAAC;IACH,WAAW,EACT,sFAAsF;IACxF,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC,EAAE,CAAC;IACzB,OAAO,EAAE,KAAK,IAAI,EAAE;QAClB,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,gCAAgC;YACzC,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAChD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACjC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,OAAO;SACR,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC1B,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;KACxE,CAAC;CACH,CAAC,CAAC;AAxBQ,QAAA,cAAc,kBAwBtB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport const screenshotTool = (v3: V3) =>\n  tool({\n    description:\n      \"Takes a screenshot (PNG) of the current page. Use this to quickly verify page state.\",\n    inputSchema: z.object({}),\n    execute: async () => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: screenshot`,\n        level: 1,\n      });\n      const page = await v3.context.awaitActivePage();\n      const buffer = await page.screenshot({ fullPage: false });\n      const pageUrl = page.url();\n      return {\n        base64: buffer.toString(\"base64\"),\n        timestamp: Date.now(),\n        pageUrl,\n      };\n    },\n    toModelOutput: (result) => ({\n      type: \"content\",\n      value: [{ type: \"media\", mediaType: \"image/png\", data: result.base64 }],\n    }),\n  });\n"]}
+{"version":3,"file":"screenshot.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/screenshot.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAGjB,MAAM,cAAc,GAAG,CAAC,EAAM,EAAE,EAAE,CACvC,IAAA,SAAI,EAAC;IACH,WAAW,EACT,sFAAsF;IACxF,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC,EAAE,CAAC;IACzB,OAAO,EAAE,KAAK,IAAI,EAAE;QAClB,IAAI,CAAC;YACH,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,gCAAgC;gBACzC,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;YAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC3B,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBACjC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;gBACrB,OAAO;aACR,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,4BAA6B,KAAe,CAAC,OAAO,EAAE;aAC9D,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;SACxE,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAzCQ,QAAA,cAAc,kBAyCtB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport const screenshotTool = (v3: V3) =>\n  tool({\n    description:\n      \"Takes a screenshot (PNG) of the current page. Use this to quickly verify page state.\",\n    inputSchema: z.object({}),\n    execute: async () => {\n      try {\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: screenshot`,\n          level: 1,\n        });\n        const page = await v3.context.awaitActivePage();\n        const buffer = await page.screenshot({ fullPage: false });\n        const pageUrl = page.url();\n        return {\n          success: true,\n          base64: buffer.toString(\"base64\"),\n          timestamp: Date.now(),\n          pageUrl,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error taking screenshot: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      return {\n        type: \"content\",\n        value: [{ type: \"media\", mediaType: \"image/png\", data: result.base64 }],\n      };\n    },\n  });\n"]}

package/dist/cjs/lib/v3/agent/tools/scroll.js

@@ -47,6 +47,12 @@ const scrollTool = (v3) => (0, ai_1.tool)({
         };
     },
     toModelOutput: (result) => {
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
+        }
         return {
             type: "json",
             value: {
@@ -123,6 +129,12 @@ const scrollVisionTool = (v3, provider) => (0, ai_1.tool)({
         };
     },
     toModelOutput: (result) => {
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
+        }
         const content = [
             {
                 type: "text",