@browserbasehq/orca 3.1.0-patch.3 → 3.2.0-preview.1

package/dist/esm/lib/v3/agent/tools/fillFormVision.js

@@ -3,8 +3,14 @@ import { z } from "zod";
 import { processCoordinates } from "../utils/coordinateNormalization.js";
 import { ensureXPath } from "../utils/xpath.js";
 import { waitAndCaptureScreenshot } from "../utils/screenshotHandler.js";
-export const fillFormVisionTool = (v3, provider) => tool({
-    description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL
+import { substituteVariables } from "../utils/variables.js";
+export const fillFormVisionTool = (v3, provider, variables) => {
+    const hasVariables = variables && Object.keys(variables).length > 0;
+    const valueDescription = hasVariables
+        ? `Text to type into the target field. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(", ")}`
+        : "Text to type into the target field";
+    return tool({
+        description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL
 
 CRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)
 IMPORTANT: Ensure the fields are visible within the current viewport
@@ -21,93 +27,110 @@ MANDATORY USE CASES (always use fillFormVision for these):
 - Checkout forms: address, payment info fields
 - Profile updates: multiple user data fields
 - Search filters: multiple criteria inputs`,
-    inputSchema: z.object({
-        fields: z
-            .array(z.object({
-            action: z
-                .string()
-                .describe("Description of the typing action, e.g. 'type foo into the bar field'"),
-            value: z.string().describe("Text to type into the target field"),
-            coordinates: z
-                .object({
-                x: z.number(),
-                y: z.number(),
-            })
-                .describe("Coordinates of the target field"),
-        }))
-            .min(2, "Provide at least two fields to fill"),
-    }),
-    execute: async ({ fields }) => {
-        try {
-            const page = await v3.context.awaitActivePage();
-            // Process coordinates for each field
-            const processedFields = fields.map((field) => {
-                const processed = processCoordinates(field.coordinates.x, field.coordinates.y, provider, v3);
-                return {
-                    ...field,
-                    coordinates: { x: processed.x, y: processed.y },
-                };
-            });
-            v3.logger({
-                category: "agent",
-                message: `Agent calling tool: fillFormVision`,
-                level: 1,
-                auxiliary: {
-                    arguments: {
-                        value: JSON.stringify({ fields, processedFields }),
-                        type: "object",
+        inputSchema: z.object({
+            fields: z
+                .array(z.object({
+                action: z
+                    .string()
+                    .describe("Description of the typing action, e.g. 'type foo into the bar field'"),
+                value: z.string().describe(valueDescription),
+                coordinates: z
+                    .object({
+                    x: z.number(),
+                    y: z.number(),
+                })
+                    .describe("Coordinates of the target field"),
+            }))
+                .min(2, "Provide at least two fields to fill"),
+        }),
+        execute: async ({ fields }) => {
+            try {
+                const page = await v3.context.awaitActivePage();
+                // Process coordinates and substitute variables for each field
+                // Keep original values (with %tokens%) for logging/caching, substituted values for typing
+                const processedFields = fields.map((field) => {
+                    const processed = processCoordinates(field.coordinates.x, field.coordinates.y, provider, v3);
+                    return {
+                        ...field,
+                        originalValue: field.value, // Keep original with %tokens% for cache
+                        value: substituteVariables(field.value, variables),
+                        coordinates: { x: processed.x, y: processed.y },
+                    };
+                });
+                v3.logger({
+                    category: "agent",
+                    message: `Agent calling tool: fillFormVision`,
+                    level: 1,
+                    auxiliary: {
+                        arguments: {
+                            value: JSON.stringify({ fields }), // Don't log substituted values
+                            type: "object",
+                        },
                     },
-                },
-            });
-            // Only request XPath when caching is enabled to avoid unnecessary computation
-            const shouldCollectXpath = v3.isAgentReplayActive();
-            const actions = [];
-            for (const field of processedFields) {
-                // Click the field, only requesting XPath when caching is enabled
-                const xpath = await page.click(field.coordinates.x, field.coordinates.y, {
-                    returnXpath: shouldCollectXpath,
                 });
-                await page.type(field.value);
-                // Build Action with XPath for deterministic replay (only when caching)
-                if (shouldCollectXpath) {
-                    const normalizedXpath = ensureXPath(xpath);
-                    if (normalizedXpath) {
-                        actions.push({
-                            selector: normalizedXpath,
-                            description: field.action,
-                            method: "type",
-                            arguments: [field.value],
-                        });
+                // Only request XPath when caching is enabled to avoid unnecessary computation
+                const shouldCollectXpath = v3.isAgentReplayActive();
+                const actions = [];
+                for (const field of processedFields) {
+                    // Click the field, only requesting XPath when caching is enabled
+                    const xpath = await page.click(field.coordinates.x, field.coordinates.y, {
+                        returnXpath: shouldCollectXpath,
+                    });
+                    await page.type(field.value);
+                    // Build Action with XPath for deterministic replay (only when caching)
+                    // Use originalValue (with %tokens%) so cache stores references, not sensitive values
+                    if (shouldCollectXpath) {
+                        const normalizedXpath = ensureXPath(xpath);
+                        if (normalizedXpath) {
+                            actions.push({
+                                selector: normalizedXpath,
+                                description: field.action,
+                                method: "type",
+                                arguments: [field.originalValue],
+                            });
+                        }
                     }
+                    // Small delay between fields
+                    await new Promise((resolve) => setTimeout(resolve, 100));
+                }
+                const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);
+                // Record as "act" step with proper Actions for deterministic replay (only when caching)
+                if (shouldCollectXpath && actions.length > 0) {
+                    v3.recordAgentReplayStep({
+                        type: "act",
+                        instruction: `Fill ${fields.length} form fields`,
+                        actions,
+                        actionDescription: `Fill ${fields.length} form fields`,
+                    });
                 }
-                // Small delay between fields
-                await new Promise((resolve) => setTimeout(resolve, 100));
+                return {
+                    success: true,
+                    playwrightArguments: processedFields,
+                    screenshotBase64,
+                };
             }
-            const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);
-            // Record as "act" step with proper Actions for deterministic replay (only when caching)
-            if (shouldCollectXpath && actions.length > 0) {
-                v3.recordAgentReplayStep({
-                    type: "act",
-                    instruction: `Fill ${fields.length} form fields`,
-                    actions,
-                    actionDescription: `Fill ${fields.length} form fields`,
-                });
+            catch (error) {
+                return {
+                    success: false,
+                    error: `Error filling form: ${error.message}`,
+                };
+            }
+        },
+        toModelOutput: (result) => {
+            if (result.success === false || result.error !== undefined) {
+                return {
+                    type: "content",
+                    value: [
+                        {
+                            type: "text",
+                            text: JSON.stringify({
+                                success: result.success,
+                                error: result.error,
+                            }),
+                        },
+                    ],
+                };
             }
-            return {
-                success: true,
-                playwrightArguments: processedFields,
-                screenshotBase64,
-            };
-        }
-        catch (error) {
-            return {
-                success: false,
-                error: `Error filling form: ${error.message}`,
-            };
-        }
-    },
-    toModelOutput: (result) => {
-        if (result.success) {
             const content = [
                 {
                     type: "text",
@@ -125,19 +148,7 @@ MANDATORY USE CASES (always use fillFormVision for these):
                 });
             }
             return { type: "content", value: content };
-        }
-        return {
-            type: "content",
-            value: [
-                {
-                    type: "text",
-                    text: JSON.stringify({
-                        success: result.success,
-                        error: result.error,
-                    }),
-                },
-            ],
-        };
-    },
-});
+        },
+    });
+};
 //# sourceMappingURL=fillFormVision.js.map

package/dist/esm/lib/v3/agent/tools/fillFormVision.js.map

@@ -1 +1 @@
-{"version":3,"file":"fillFormVision.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillFormVision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC9D,IAAI,CAAC;IACH,WAAW,EAAE;;;;;;;;;;;;;;;;2CAgB0B;IACvC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,MAAM,EAAE,CAAC;aACN,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;YACP,MAAM,EAAE,CAAC;iBACN,MAAM,EAAE;iBACR,QAAQ,CACP,sEAAsE,CACvE;YACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,oCAAoC,CAAC;YAChE,WAAW,EAAE,CAAC;iBACX,MAAM,CAAC;gBACN,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;gBACb,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;aACd,CAAC;iBACD,QAAQ,CAAC,iCAAiC,CAAC;SAC/C,CAAC,CACH;aACA,GAAG,CAAC,CAAC,EAAE,qCAAqC,CAAC;KACjD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAqC,EAAE;QAC/D,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAEhD,qCAAqC;YACrC,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC3C,MAAM,SAAS,GAAG,kBAAkB,CAClC,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;gBACF,OAAO;oBACL,GAAG,KAAK;oBACR,WAAW,EAAE,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;iBAChD,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,oCAAoC;gBAC7C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;wBAClD,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,OAAO,GAAa,EAAE,CAAC;YAE7B,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;gBACpC,iEAAiE;gBACjE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAC5B,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB;oBACE,WAAW,EAAE,kBAAkB;iBAChC,CACF,CAAC;gBACF,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAE7B,uEAAuE;gBACvE,IAAI,kBAAkB,EAAE,CAAC;oBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;oBAC3C,IAAI,eAAe,EAAE,CAAC;wBACpB,OAAO,CAAC,IAAI,CAAC;4BACX,QAAQ,EAAE,eAAe;4BACzB,WAAW,EAAE,KAAK,CAAC,MAAM;4BACzB,MAAM,EAAE,MAAM;4BACd,SAAS,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;yBACzB,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAED,6BAA6B;gBAC7B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YAC3D,CAAC;YAED,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAEnE,wFAAwF;YACxF,IAAI,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7C,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,KAAK;oBACX,WAAW,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;oBAChD,OAAO;oBACP,iBAAiB,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;iBACvD,CAAC,CAAC;YACL,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,mBAAmB,EAAE,eAAe;gBACpC,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,uBAAwB,KAAe,CAAC,OAAO,EAAE;aACzD,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,WAAW,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,IAAI,CAAC;qBACrD,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;QACD,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,KAAK,EAAE,MAAM,CAAC,KAAK;qBACpB,CAAC;iBACH;aACF;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  FillFormVisionToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const fillFormVisionTool = (v3: V3, provider?: string) =>\n  tool({\n    description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL\n\nCRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)\nIMPORTANT: Ensure the fields are visible within the current viewport\n\nWHY THIS TOOL EXISTS:\n- Forms are the #1 use case for multi-field input\n- Optimized specifically for input/textarea elements\n- 4-6x faster than individual typing actions\n\nUse fillFormVision: Pure form filling (inputs, textareas only)\nMANDATORY USE CASES (always use fillFormVision for these):\n- Registration forms: name, email, password fields\n- Contact forms: name, email, message fields\n- Checkout forms: address, payment info fields\n- Profile updates: multiple user data fields\n- Search filters: multiple criteria inputs`,\n    inputSchema: z.object({\n      fields: z\n        .array(\n          z.object({\n            action: z\n              .string()\n              .describe(\n                \"Description of the typing action, e.g. 'type foo into the bar field'\",\n              ),\n            value: z.string().describe(\"Text to type into the target field\"),\n            coordinates: z\n              .object({\n                x: z.number(),\n                y: z.number(),\n              })\n              .describe(\"Coordinates of the target field\"),\n          }),\n        )\n        .min(2, \"Provide at least two fields to fill\"),\n    }),\n    execute: async ({ fields }): Promise<FillFormVisionToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n\n        // Process coordinates for each field\n        const processedFields = fields.map((field) => {\n          const processed = processCoordinates(\n            field.coordinates.x,\n            field.coordinates.y,\n            provider,\n            v3,\n          );\n          return {\n            ...field,\n            coordinates: { x: processed.x, y: processed.y },\n          };\n        });\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: fillFormVision`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ fields, processedFields }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const actions: Action[] = [];\n\n        for (const field of processedFields) {\n          // Click the field, only requesting XPath when caching is enabled\n          const xpath = await page.click(\n            field.coordinates.x,\n            field.coordinates.y,\n            {\n              returnXpath: shouldCollectXpath,\n            },\n          );\n          await page.type(field.value);\n\n          // Build Action with XPath for deterministic replay (only when caching)\n          if (shouldCollectXpath) {\n            const normalizedXpath = ensureXPath(xpath);\n            if (normalizedXpath) {\n              actions.push({\n                selector: normalizedXpath,\n                description: field.action,\n                method: \"type\",\n                arguments: [field.value],\n              });\n            }\n          }\n\n          // Small delay between fields\n          await new Promise((resolve) => setTimeout(resolve, 100));\n        }\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n        // Record as \"act\" step with proper Actions for deterministic replay (only when caching)\n        if (shouldCollectXpath && actions.length > 0) {\n          v3.recordAgentReplayStep({\n            type: \"act\",\n            instruction: `Fill ${fields.length} form fields`,\n            actions,\n            actionDescription: `Fill ${fields.length} form fields`,\n          });\n        }\n\n        return {\n          success: true,\n          playwrightArguments: processedFields,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error filling form: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success) {\n        const content: ModelOutputContentItem[] = [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              fieldsCount: result.playwrightArguments?.length ?? 0,\n            }),\n          },\n        ];\n        if (result.screenshotBase64) {\n          content.push({\n            type: \"media\",\n            mediaType: \"image/png\",\n            data: result.screenshotBase64,\n          });\n        }\n        return { type: \"content\", value: content };\n      }\n      return {\n        type: \"content\",\n        value: [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              error: result.error,\n            }),\n          },\n        ],\n      };\n    },\n  });\n"]}
+{"version":3,"file":"fillFormVision.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillFormVision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAChC,EAAM,EACN,QAAiB,EACjB,SAAqB,EACrB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,qGAAqG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAC1I,CAAC,CAAC,oCAAoC,CAAC;IAEzC,OAAO,IAAI,CAAC;QACV,WAAW,EAAE;;;;;;;;;;;;;;;;2CAgB0B;QACvC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,CAAC;iBACN,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,CAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,sEAAsE,CACvE;gBACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBAC5C,WAAW,EAAE,CAAC;qBACX,MAAM,CAAC;oBACN,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;oBACb,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;iBACd,CAAC;qBACD,QAAQ,CAAC,iCAAiC,CAAC;aAC/C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,qCAAqC,CAAC;SACjD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAqC,EAAE;YAC/D,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAEhD,8DAA8D;gBAC9D,0FAA0F;gBAC1F,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;oBAC3C,MAAM,SAAS,GAAG,kBAAkB,CAClC,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;oBACF,OAAO;wBACL,GAAG,KAAK;wBACR,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE,wCAAwC;wBACpE,KAAK,EAAE,mBAAmB,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC;wBAClD,WAAW,EAAE,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;qBAChD,CAAC;gBACJ,CAAC,CAAC,CAAC;gBAEH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,oCAAoC;oBAC7C,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,+BAA+B;4BAClE,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,OAAO,GAAa,EAAE,CAAC;gBAE7B,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;oBACpC,iEAAiE;oBACjE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAC5B,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB;wBACE,WAAW,EAAE,kBAAkB;qBAChC,CACF,CAAC;oBACF,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;oBAE7B,uEAAuE;oBACvE,qFAAqF;oBACrF,IAAI,kBAAkB,EAAE,CAAC;wBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;wBAC3C,IAAI,eAAe,EAAE,CAAC;4BACpB,OAAO,CAAC,IAAI,CAAC;gCACX,QAAQ,EAAE,eAAe;gCACzB,WAAW,EAAE,KAAK,CAAC,MAAM;gCACzB,MAAM,EAAE,MAAM;gCACd,SAAS,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC;6BACjC,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,6BAA6B;oBAC7B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;gBAC3D,CAAC;gBAED,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAEnE,wFAAwF;gBACxF,IAAI,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7C,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;wBAChD,OAAO;wBACP,iBAAiB,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;qBACvD,CAAC,CAAC;gBACL,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,mBAAmB,EAAE,eAAe;oBACpC,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,uBAAwB,KAAe,CAAC,OAAO,EAAE;iBACzD,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE;wBACL;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gCACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gCACvB,KAAK,EAAE,MAAM,CAAC,KAAK;6BACpB,CAAC;yBACH;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,WAAW,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,IAAI,CAAC;qBACrD,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  FillFormVisionToolResult,\n  ModelOutputContentItem,\n  Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const fillFormVisionTool = (\n  v3: V3,\n  provider?: string,\n  variables?: Variables,\n) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const valueDescription = hasVariables\n    ? `Text to type into the target field. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"Text to type into the target field\";\n\n  return tool({\n    description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL\n\nCRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)\nIMPORTANT: Ensure the fields are visible within the current viewport\n\nWHY THIS TOOL EXISTS:\n- Forms are the #1 use case for multi-field input\n- Optimized specifically for input/textarea elements\n- 4-6x faster than individual typing actions\n\nUse fillFormVision: Pure form filling (inputs, textareas only)\nMANDATORY USE CASES (always use fillFormVision for these):\n- Registration forms: name, email, password fields\n- Contact forms: name, email, message fields\n- Checkout forms: address, payment info fields\n- Profile updates: multiple user data fields\n- Search filters: multiple criteria inputs`,\n    inputSchema: z.object({\n      fields: z\n        .array(\n          z.object({\n            action: z\n              .string()\n              .describe(\n                \"Description of the typing action, e.g. 'type foo into the bar field'\",\n              ),\n            value: z.string().describe(valueDescription),\n            coordinates: z\n              .object({\n                x: z.number(),\n                y: z.number(),\n              })\n              .describe(\"Coordinates of the target field\"),\n          }),\n        )\n        .min(2, \"Provide at least two fields to fill\"),\n    }),\n    execute: async ({ fields }): Promise<FillFormVisionToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n\n        // Process coordinates and substitute variables for each field\n        // Keep original values (with %tokens%) for logging/caching, substituted values for typing\n        const processedFields = fields.map((field) => {\n          const processed = processCoordinates(\n            field.coordinates.x,\n            field.coordinates.y,\n            provider,\n            v3,\n          );\n          return {\n            ...field,\n            originalValue: field.value, // Keep original with %tokens% for cache\n            value: substituteVariables(field.value, variables),\n            coordinates: { x: processed.x, y: processed.y },\n          };\n        });\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: fillFormVision`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ fields }), // Don't log substituted values\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const actions: Action[] = [];\n\n        for (const field of processedFields) {\n          // Click the field, only requesting XPath when caching is enabled\n          const xpath = await page.click(\n            field.coordinates.x,\n            field.coordinates.y,\n            {\n              returnXpath: shouldCollectXpath,\n            },\n          );\n          await page.type(field.value);\n\n          // Build Action with XPath for deterministic replay (only when caching)\n          // Use originalValue (with %tokens%) so cache stores references, not sensitive values\n          if (shouldCollectXpath) {\n            const normalizedXpath = ensureXPath(xpath);\n            if (normalizedXpath) {\n              actions.push({\n                selector: normalizedXpath,\n                description: field.action,\n                method: \"type\",\n                arguments: [field.originalValue],\n              });\n            }\n          }\n\n          // Small delay between fields\n          await new Promise((resolve) => setTimeout(resolve, 100));\n        }\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n        // Record as \"act\" step with proper Actions for deterministic replay (only when caching)\n        if (shouldCollectXpath && actions.length > 0) {\n          v3.recordAgentReplayStep({\n            type: \"act\",\n            instruction: `Fill ${fields.length} form fields`,\n            actions,\n            actionDescription: `Fill ${fields.length} form fields`,\n          });\n        }\n\n        return {\n          success: true,\n          playwrightArguments: processedFields,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error filling form: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [\n            {\n              type: \"text\",\n              text: JSON.stringify({\n                success: result.success,\n                error: result.error,\n              }),\n            },\n          ],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            fieldsCount: result.playwrightArguments?.length ?? 0,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n};\n"]}

package/dist/esm/lib/v3/agent/tools/fillform.d.ts

@@ -1,7 +1,7 @@
 import type { V3 } from "../../v3.js";
 import type { Action } from "../../types/public/methods.js";
-import type { AgentModelConfig } from "../../types/public/agent.js";
-export declare const fillFormTool: (v3: V3, executionModel?: string | AgentModelConfig) => import("ai").Tool<{
+import type { AgentModelConfig, Variables } from "../../types/public/agent.js";
+export declare const fillFormTool: (v3: V3, executionModel?: string | AgentModelConfig, variables?: Variables, toolTimeout?: number) => import("ai").Tool<{
     fields: {
         action: string;
         value: string;
@@ -10,4 +10,10 @@ export declare const fillFormTool: (v3: V3, executionModel?: string | AgentModel
     success: boolean;
     actions: unknown[];
     playwrightArguments: Action[];
+    error?: undefined;
+} | {
+    success: boolean;
+    error: any;
+    actions?: undefined;
+    playwrightArguments?: undefined;
 }>;

package/dist/esm/lib/v3/agent/tools/fillform.js

@@ -1,56 +1,86 @@
 import { tool } from "ai";
 import { z } from "zod";
-export const fillFormTool = (v3, executionModel) => tool({
-    description: `📝 FORM FILL - MULTI-FIELD INPUT TOOL\nFor any form with 2+ inputs/textareas. Faster than individual typing.`,
-    inputSchema: z.object({
-        fields: z
-            .array(z.object({
-            action: z
-                .string()
-                .describe('Description of typing action, e.g. "type foo into the email field"'),
-            value: z.string().describe("Text to type into the target"),
-        }))
-            .min(1, "Provide at least one field to fill"),
-    }),
-    execute: async ({ fields }) => {
-        v3.logger({
-            category: "agent",
-            message: `Agent calling tool: fillForm`,
-            level: 1,
-            auxiliary: {
-                arguments: {
-                    value: JSON.stringify(fields),
-                    type: "object",
-                },
-            },
-        });
-        const instruction = `Return observation results for the following actions: ${fields
-            .map((f) => f.action)
-            .join(", ")}`;
-        const observeOptions = executionModel
-            ? { model: executionModel }
-            : undefined;
-        const observeResults = await v3.observe(instruction, observeOptions);
-        const completed = [];
-        const replayableActions = [];
-        for (const res of observeResults) {
-            const actResult = await v3.act(res);
-            completed.push(actResult);
-            if (Array.isArray(actResult.actions)) {
-                replayableActions.push(...actResult.actions);
+import { TimeoutError } from "../../types/public/sdkErrors.js";
+export const fillFormTool = (v3, executionModel, variables, toolTimeout) => {
+    const hasVariables = variables && Object.keys(variables).length > 0;
+    const valueDescription = hasVariables
+        ? `Text to type into the target. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(", ")}`
+        : "Text to type into the target";
+    return tool({
+        description: `📝 FORM FILL - MULTI-FIELD INPUT TOOL\nFor any form with 2+ inputs/textareas. Faster than individual typing.`,
+        inputSchema: z.object({
+            fields: z
+                .array(z.object({
+                action: z
+                    .string()
+                    .describe('Description of typing action, e.g. "type foo into the email field"'),
+                value: z.string().describe(valueDescription),
+            }))
+                .min(1, "Provide at least one field to fill"),
+        }),
+        execute: async ({ fields }) => {
+            try {
+                v3.logger({
+                    category: "agent",
+                    message: `Agent calling tool: fillForm`,
+                    level: 1,
+                    auxiliary: {
+                        arguments: {
+                            value: JSON.stringify(fields),
+                            type: "object",
+                        },
+                    },
+                });
+                const instruction = `Return observation results for the following actions: ${fields
+                    .map((f) => f.action)
+                    .join(", ")}`;
+                const observeOptions = executionModel
+                    ? { model: executionModel, timeout: toolTimeout }
+                    : { timeout: toolTimeout };
+                const observeResults = await v3.observe(instruction, observeOptions);
+                const completed = [];
+                const replayableActions = [];
+                for (const res of observeResults) {
+                    const actOptions = variables
+                        ? { variables, timeout: toolTimeout }
+                        : { timeout: toolTimeout };
+                    const actResult = await v3.act(res, actOptions);
+                    completed.push(actResult);
+                    if (Array.isArray(actResult.actions)) {
+                        replayableActions.push(...actResult.actions);
+                    }
+                }
+                v3.recordAgentReplayStep({
+                    type: "fillForm",
+                    fields,
+                    observeResults,
+                    actions: replayableActions,
+                });
+                return {
+                    success: true,
+                    actions: completed,
+                    playwrightArguments: replayableActions,
+                };
             }
-        }
-        v3.recordAgentReplayStep({
-            type: "fillForm",
-            fields,
-            observeResults,
-            actions: replayableActions,
-        });
-        return {
-            success: true,
-            actions: completed,
-            playwrightArguments: replayableActions,
-        };
-    },
-});
+            catch (error) {
+                if (error instanceof TimeoutError) {
+                    const timeoutMessage = `TimeoutError while waiting for fillForm() to complete (it may continue executing in the background)`;
+                    v3.logger({
+                        category: "agent",
+                        message: timeoutMessage,
+                        level: 0,
+                    });
+                    return {
+                        success: false,
+                        error: `${timeoutMessage} — try filling fewer fields at once or use a different tool`,
+                    };
+                }
+                return {
+                    success: false,
+                    error: error?.message ?? String(error),
+                };
+            }
+        },
+    });
+};
 //# sourceMappingURL=fillform.js.map

package/dist/esm/lib/v3/agent/tools/fillform.js.map

@@ -1 +1 @@
-{"version":3,"file":"fillform.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillform.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAKxB,MAAM,CAAC,MAAM,YAAY,GAAG,CAC1B,EAAM,EACN,cAA0C,EAC1C,EAAE,CACF,IAAI,CAAC;IACH,WAAW,EAAE,8GAA8G;IAC3H,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,MAAM,EAAE,CAAC;aACN,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;YACP,MAAM,EAAE,CAAC;iBACN,MAAM,EAAE;iBACR,QAAQ,CACP,oEAAoE,CACrE;YACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8BAA8B,CAAC;SAC3D,CAAC,CACH;aACA,GAAG,CAAC,CAAC,EAAE,oCAAoC,CAAC;KAChD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;QAC5B,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,8BAA8B;YACvC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;oBAC7B,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QACH,MAAM,WAAW,GAAG,yDAAyD,MAAM;aAChF,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;aACpB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAEhB,MAAM,cAAc,GAAG,cAAc;YACnC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE;YAC3B,CAAC,CAAC,SAAS,CAAC;QACd,MAAM,cAAc,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;QAErE,MAAM,SAAS,GAAG,EAAe,CAAC;QAClC,MAAM,iBAAiB,GAAa,EAAE,CAAC;QACvC,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;YACjC,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACpC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;gBACrC,iBAAiB,CAAC,IAAI,CAAC,GAAI,SAAS,CAAC,OAAoB,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QACD,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,UAAU;YAChB,MAAM;YACN,cAAc;YACd,OAAO,EAAE,iBAAiB;SAC3B,CAAC,CAAC;QACH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,SAAS;YAClB,mBAAmB,EAAE,iBAAiB;SACvC,CAAC;IACJ,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type { AgentModelConfig } from \"../../types/public/agent.js\";\n\nexport const fillFormTool = (\n  v3: V3,\n  executionModel?: string | AgentModelConfig,\n) =>\n  tool({\n    description: `📝 FORM FILL - MULTI-FIELD INPUT TOOL\\nFor any form with 2+ inputs/textareas. Faster than individual typing.`,\n    inputSchema: z.object({\n      fields: z\n        .array(\n          z.object({\n            action: z\n              .string()\n              .describe(\n                'Description of typing action, e.g. \"type foo into the email field\"',\n              ),\n            value: z.string().describe(\"Text to type into the target\"),\n          }),\n        )\n        .min(1, \"Provide at least one field to fill\"),\n    }),\n    execute: async ({ fields }) => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: fillForm`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: JSON.stringify(fields),\n            type: \"object\",\n          },\n        },\n      });\n      const instruction = `Return observation results for the following actions: ${fields\n        .map((f) => f.action)\n        .join(\", \")}`;\n\n      const observeOptions = executionModel\n        ? { model: executionModel }\n        : undefined;\n      const observeResults = await v3.observe(instruction, observeOptions);\n\n      const completed = [] as unknown[];\n      const replayableActions: Action[] = [];\n      for (const res of observeResults) {\n        const actResult = await v3.act(res);\n        completed.push(actResult);\n        if (Array.isArray(actResult.actions)) {\n          replayableActions.push(...(actResult.actions as Action[]));\n        }\n      }\n      v3.recordAgentReplayStep({\n        type: \"fillForm\",\n        fields,\n        observeResults,\n        actions: replayableActions,\n      });\n      return {\n        success: true,\n        actions: completed,\n        playwrightArguments: replayableActions,\n      };\n    },\n  });\n"]}
+{"version":3,"file":"fillform.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillform.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAE/D,MAAM,CAAC,MAAM,YAAY,GAAG,CAC1B,EAAM,EACN,cAA0C,EAC1C,SAAqB,EACrB,WAAoB,EACpB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,+FAA+F,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACpI,CAAC,CAAC,8BAA8B,CAAC;IAEnC,OAAO,IAAI,CAAC;QACV,WAAW,EAAE,8GAA8G;QAC3H,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,CAAC;iBACN,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,CAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,oEAAoE,CACrE;gBACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;aAC7C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,oCAAoC,CAAC;SAChD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;YAC5B,IAAI,CAAC;gBACH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,8BAA8B;oBACvC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;4BAC7B,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBACH,MAAM,WAAW,GAAG,yDAAyD,MAAM;qBAChF,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;qBACpB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAEhB,MAAM,cAAc,GAAG,cAAc;oBACnC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,WAAW,EAAE;oBACjD,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;gBAC7B,MAAM,cAAc,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;gBAErE,MAAM,SAAS,GAAG,EAAe,CAAC;gBAClC,MAAM,iBAAiB,GAAa,EAAE,CAAC;gBACvC,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;oBACjC,MAAM,UAAU,GAAG,SAAS;wBAC1B,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE;wBACrC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;oBAC7B,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;oBAChD,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;wBACrC,iBAAiB,CAAC,IAAI,CAAC,GAAI,SAAS,CAAC,OAAoB,CAAC,CAAC;oBAC7D,CAAC;gBACH,CAAC;gBACD,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,UAAU;oBAChB,MAAM;oBACN,cAAc;oBACd,OAAO,EAAE,iBAAiB;iBAC3B,CAAC,CAAC;gBACH,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,OAAO,EAAE,SAAS;oBAClB,mBAAmB,EAAE,iBAAiB;iBACvC,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;oBAClC,MAAM,cAAc,GAAG,qGAAqG,CAAC;oBAC7H,EAAE,CAAC,MAAM,CAAC;wBACR,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,cAAc;wBACvB,KAAK,EAAE,CAAC;qBACT,CAAC,CAAC;oBACH,OAAO;wBACL,OAAO,EAAE,KAAK;wBACd,KAAK,EAAE,GAAG,cAAc,6DAA6D;qBACtF,CAAC;gBACJ,CAAC;gBACD,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC;iBACvC,CAAC;YACJ,CAAC;QACH,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type { AgentModelConfig, Variables } from \"../../types/public/agent.js\";\nimport { TimeoutError } from \"../../types/public/sdkErrors.js\";\n\nexport const fillFormTool = (\n  v3: V3,\n  executionModel?: string | AgentModelConfig,\n  variables?: Variables,\n  toolTimeout?: number,\n) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const valueDescription = hasVariables\n    ? `Text to type into the target. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"Text to type into the target\";\n\n  return tool({\n    description: `📝 FORM FILL - MULTI-FIELD INPUT TOOL\\nFor any form with 2+ inputs/textareas. Faster than individual typing.`,\n    inputSchema: z.object({\n      fields: z\n        .array(\n          z.object({\n            action: z\n              .string()\n              .describe(\n                'Description of typing action, e.g. \"type foo into the email field\"',\n              ),\n            value: z.string().describe(valueDescription),\n          }),\n        )\n        .min(1, \"Provide at least one field to fill\"),\n    }),\n    execute: async ({ fields }) => {\n      try {\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: fillForm`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify(fields),\n              type: \"object\",\n            },\n          },\n        });\n        const instruction = `Return observation results for the following actions: ${fields\n          .map((f) => f.action)\n          .join(\", \")}`;\n\n        const observeOptions = executionModel\n          ? { model: executionModel, timeout: toolTimeout }\n          : { timeout: toolTimeout };\n        const observeResults = await v3.observe(instruction, observeOptions);\n\n        const completed = [] as unknown[];\n        const replayableActions: Action[] = [];\n        for (const res of observeResults) {\n          const actOptions = variables\n            ? { variables, timeout: toolTimeout }\n            : { timeout: toolTimeout };\n          const actResult = await v3.act(res, actOptions);\n          completed.push(actResult);\n          if (Array.isArray(actResult.actions)) {\n            replayableActions.push(...(actResult.actions as Action[]));\n          }\n        }\n        v3.recordAgentReplayStep({\n          type: \"fillForm\",\n          fields,\n          observeResults,\n          actions: replayableActions,\n        });\n        return {\n          success: true,\n          actions: completed,\n          playwrightArguments: replayableActions,\n        };\n      } catch (error) {\n        if (error instanceof TimeoutError) {\n          const timeoutMessage = `TimeoutError while waiting for fillForm() to complete (it may continue executing in the background)`;\n          v3.logger({\n            category: \"agent\",\n            message: timeoutMessage,\n            level: 0,\n          });\n          return {\n            success: false,\n            error: `${timeoutMessage} — try filling fewer fields at once or use a different tool`,\n          };\n        }\n        return {\n          success: false,\n          error: error?.message ?? String(error),\n        };\n      }\n    },\n  });\n};\n"]}

package/dist/esm/lib/v3/agent/tools/index.d.ts

@@ -18,7 +18,7 @@ import { searchTool } from "./search.js";
 import type { ToolSet, InferUITools } from "ai";
 import type { V3 } from "../../v3.js";
 import type { LogLine } from "../../types/public/logs.js";
-import type { AgentToolMode, AgentModelConfig } from "../../types/public/agent.js";
+import type { AgentToolMode, AgentModelConfig, Variables } from "../../types/public/agent.js";
 export interface V3AgentToolOptions {
     executionModel?: string | AgentModelConfig;
     logger?: (message: LogLine) => void;
@@ -37,6 +37,16 @@ export interface V3AgentToolOptions {
      * These tools will be filtered out after mode-based filtering.
      */
     excludeTools?: string[];
+    /**
+     * Variables available to the agent for use in act/type tools.
+     * When provided, these tools will have an optional useVariable field.
+     */
+    variables?: Variables;
+    /**
+     * Timeout in milliseconds for tool calls that invoke v3 methods (act, extract, fillForm, ariaTree).
+     * Forwarded to the underlying v3 call's `timeout` option.
+     */
+    toolTimeout?: number;
 }
 export declare function createAgentTools(v3: V3, options?: V3AgentToolOptions): ToolSet;
 export type AgentTools = ReturnType<typeof createAgentTools>;

package/dist/esm/lib/v3/agent/tools/index.js

@@ -47,25 +47,26 @@ export function createAgentTools(v3, options) {
     const mode = options?.mode ?? "dom";
     const provider = options?.provider;
     const excludeTools = options?.excludeTools;
+    const variables = options?.variables;
+    const toolTimeout = options?.toolTimeout;
     const allTools = {
-        act: actTool(v3, executionModel),
-        ariaTree: ariaTreeTool(v3),
+        act: actTool(v3, executionModel, variables, toolTimeout),
+        ariaTree: ariaTreeTool(v3, toolTimeout),
         click: clickTool(v3, provider),
         clickAndHold: clickAndHoldTool(v3, provider),
         dragAndDrop: dragAndDropTool(v3, provider),
-        extract: extractTool(v3, executionModel),
-        fillForm: fillFormTool(v3, executionModel),
-        fillFormVision: fillFormVisionTool(v3, provider),
+        extract: extractTool(v3, executionModel, toolTimeout),
+        fillForm: fillFormTool(v3, executionModel, variables, toolTimeout),
+        fillFormVision: fillFormVisionTool(v3, provider, variables),
         goto: gotoTool(v3),
         keys: keysTool(v3),
         navback: navBackTool(v3),
         screenshot: screenshotTool(v3),
         scroll: mode === "hybrid" ? scrollVisionTool(v3, provider) : scrollTool(v3),
         think: thinkTool(),
-        type: typeTool(v3, provider),
+        type: typeTool(v3, provider, variables),
         wait: waitTool(v3, mode),
     };
-    // Only include search tool if BRAVE_API_KEY is configured
     if (process.env.BRAVE_API_KEY) {
         allTools.search = searchTool(v3);
     }

package/dist/esm/lib/v3/agent/tools/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AACnC,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AA8BzC;;;;;GAKG;AACH,SAAS,WAAW,CAClB,KAAc,EACd,IAAmB,EACnB,YAAuB;IAEvB,MAAM,QAAQ,GAAY,EAAE,GAAG,KAAK,EAAE,CAAC;IAEvC,uBAAuB;IACvB,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC,QAAQ,CAAC;IAC3B,CAAC;SAAM,CAAC;QACN,qBAAqB;QACrB,OAAO,QAAQ,CAAC,KAAK,CAAC;QACtB,OAAO,QAAQ,CAAC,IAAI,CAAC;QACrB,OAAO,QAAQ,CAAC,WAAW,CAAC;QAC5B,OAAO,QAAQ,CAAC,YAAY,CAAC;QAC7B,OAAO,QAAQ,CAAC,cAAc,CAAC;IACjC,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,KAAK,MAAM,QAAQ,IAAI,YAAY,EAAE,CAAC;YACpC,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,EAAM,EAAE,OAA4B;IACnE,MAAM,cAAc,GAAG,OAAO,EAAE,cAAc,CAAC;IAC/C,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,KAAK,CAAC;IACpC,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,CAAC;IACnC,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,CAAC;IAE3C,MAAM,QAAQ,GAAY;QACxB,GAAG,EAAE,OAAO,CAAC,EAAE,EAAE,cAAc,CAAC;QAChC,QAAQ,EAAE,YAAY,CAAC,EAAE,CAAC;QAC1B,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC9B,YAAY,EAAE,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC5C,WAAW,EAAE,eAAe,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC1C,OAAO,EAAE,WAAW,CAAC,EAAE,EAAE,cAAc,CAAC;QACxC,QAAQ,EAAE,YAAY,CAAC,EAAE,EAAE,cAAc,CAAC;QAC1C,cAAc,EAAE,kBAAkB,CAAC,EAAE,EAAE,QAAQ,CAAC;QAChD,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;QAClB,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;QAClB,OAAO,EAAE,WAAW,CAAC,EAAE,CAAC;QACxB,UAAU,EAAE,cAAc,CAAC,EAAE,CAAC;QAC9B,MAAM,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3E,KAAK,EAAE,SAAS,EAAE;QAClB,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC5B,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;KACzB,CAAC;IAEF,0DAA0D;IAC1D,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;QAC9B,QAAQ,CAAC,MAAM,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,WAAW,CAAC,QAAQ,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;AACnD,CAAC","sourcesContent":["import { gotoTool } from \"./goto.js\";\nimport { actTool } from \"./act.js\";\nimport { screenshotTool } from \"./screenshot.js\";\nimport { waitTool } from \"./wait.js\";\nimport { navBackTool } from \"./navback.js\";\nimport { ariaTreeTool } from \"./ariaTree.js\";\nimport { fillFormTool } from \"./fillform.js\";\nimport { scrollTool, scrollVisionTool } from \"./scroll.js\";\nimport { extractTool } from \"./extract.js\";\nimport { clickTool } from \"./click.js\";\nimport { typeTool } from \"./type.js\";\nimport { dragAndDropTool } from \"./dragAndDrop.js\";\nimport { clickAndHoldTool } from \"./clickAndHold.js\";\nimport { keysTool } from \"./keys.js\";\nimport { fillFormVisionTool } from \"./fillFormVision.js\";\nimport { thinkTool } from \"./think.js\";\nimport { searchTool } from \"./search.js\";\n\nimport type { ToolSet, InferUITools } from \"ai\";\nimport type { V3 } from \"../../v3.js\";\nimport type { LogLine } from \"../../types/public/logs.js\";\nimport type {\n  AgentToolMode,\n  AgentModelConfig,\n} from \"../../types/public/agent.js\";\n\nexport interface V3AgentToolOptions {\n  executionModel?: string | AgentModelConfig;\n  logger?: (message: LogLine) => void;\n  /**\n   * Tool mode determines which set of tools are available.\n   * - 'dom' (default): Uses DOM-based tools (act, fillForm) - removes coordinate-based tools\n   * - 'hybrid': Uses coordinate-based tools (click, type, dragAndDrop, etc.) - removes fillForm\n   */\n  mode?: AgentToolMode;\n  /**\n   * The model provider. Used for model-specific coordinate handling\n   */\n  provider?: string;\n  /**\n   * Tools to exclude from the available toolset.\n   * These tools will be filtered out after mode-based filtering.\n   */\n  excludeTools?: string[];\n}\n\n/**\n * Filters tools based on mode and explicit exclusions.\n * - 'dom' mode: Removes coordinate-based tools (click, type, dragAndDrop, clickAndHold, fillFormVision)\n * - 'hybrid' mode: Removes DOM-based form tool (fillForm) in favor of coordinate-based fillFormVision\n * - excludeTools: Additional tools to remove from the toolset\n */\nfunction filterTools(\n  tools: ToolSet,\n  mode: AgentToolMode,\n  excludeTools?: string[],\n): ToolSet {\n  const filtered: ToolSet = { ...tools };\n\n  // Mode-based filtering\n  if (mode === \"hybrid\") {\n    delete filtered.fillForm;\n  } else {\n    // DOM mode (default)\n    delete filtered.click;\n    delete filtered.type;\n    delete filtered.dragAndDrop;\n    delete filtered.clickAndHold;\n    delete filtered.fillFormVision;\n  }\n\n  if (excludeTools) {\n    for (const toolName of excludeTools) {\n      delete filtered[toolName];\n    }\n  }\n\n  return filtered;\n}\n\nexport function createAgentTools(v3: V3, options?: V3AgentToolOptions) {\n  const executionModel = options?.executionModel;\n  const mode = options?.mode ?? \"dom\";\n  const provider = options?.provider;\n  const excludeTools = options?.excludeTools;\n\n  const allTools: ToolSet = {\n    act: actTool(v3, executionModel),\n    ariaTree: ariaTreeTool(v3),\n    click: clickTool(v3, provider),\n    clickAndHold: clickAndHoldTool(v3, provider),\n    dragAndDrop: dragAndDropTool(v3, provider),\n    extract: extractTool(v3, executionModel),\n    fillForm: fillFormTool(v3, executionModel),\n    fillFormVision: fillFormVisionTool(v3, provider),\n    goto: gotoTool(v3),\n    keys: keysTool(v3),\n    navback: navBackTool(v3),\n    screenshot: screenshotTool(v3),\n    scroll: mode === \"hybrid\" ? scrollVisionTool(v3, provider) : scrollTool(v3),\n    think: thinkTool(),\n    type: typeTool(v3, provider),\n    wait: waitTool(v3, mode),\n  };\n\n  // Only include search tool if BRAVE_API_KEY is configured\n  if (process.env.BRAVE_API_KEY) {\n    allTools.search = searchTool(v3);\n  }\n\n  return filterTools(allTools, mode, excludeTools);\n}\n\nexport type AgentTools = ReturnType<typeof createAgentTools>;\n\n/**\n * Type map of all agent tools for strong typing of tool calls and results.\n * Note: `search` is optional as it's only available when BRAVE_API_KEY is configured.\n */\nexport type AgentToolTypesMap = {\n  act: ReturnType<typeof actTool>;\n  ariaTree: ReturnType<typeof ariaTreeTool>;\n  click: ReturnType<typeof clickTool>;\n  clickAndHold: ReturnType<typeof clickAndHoldTool>;\n  dragAndDrop: ReturnType<typeof dragAndDropTool>;\n  extract: ReturnType<typeof extractTool>;\n  fillForm: ReturnType<typeof fillFormTool>;\n  fillFormVision: ReturnType<typeof fillFormVisionTool>;\n  goto: ReturnType<typeof gotoTool>;\n  keys: ReturnType<typeof keysTool>;\n  navback: ReturnType<typeof navBackTool>;\n  screenshot: ReturnType<typeof screenshotTool>;\n  scroll: ReturnType<typeof scrollTool> | ReturnType<typeof scrollVisionTool>;\n  search?: ReturnType<typeof searchTool>;\n  think: ReturnType<typeof thinkTool>;\n  type: ReturnType<typeof typeTool>;\n  wait: ReturnType<typeof waitTool>;\n};\n\n/**\n * Inferred UI tools type for type-safe tool inputs and outputs.\n * Use with UIMessage for full type safety in UI contexts.\n */\nexport type AgentUITools = InferUITools<AgentToolTypesMap>;\n\n/**\n * Union type for all possible agent tool calls.\n * Provides type-safe access to tool call arguments.\n */\nexport type AgentToolCall = {\n  [K in keyof AgentToolTypesMap]: {\n    toolName: K;\n    toolCallId: string;\n    args: AgentUITools[K][\"input\"];\n  };\n}[keyof AgentToolTypesMap];\n\n/**\n * Union type for all possible agent tool results.\n * Provides type-safe access to tool result values.\n */\nexport type AgentToolResult = {\n  [K in keyof AgentToolTypesMap]: {\n    toolName: K;\n    toolCallId: string;\n    result: AgentUITools[K][\"output\"];\n  };\n}[keyof AgentToolTypesMap];\n"]}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AACnC,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAyCzC;;;;;GAKG;AACH,SAAS,WAAW,CAClB,KAAc,EACd,IAAmB,EACnB,YAAuB;IAEvB,MAAM,QAAQ,GAAY,EAAE,GAAG,KAAK,EAAE,CAAC;IAEvC,uBAAuB;IACvB,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC,QAAQ,CAAC;IAC3B,CAAC;SAAM,CAAC;QACN,qBAAqB;QACrB,OAAO,QAAQ,CAAC,KAAK,CAAC;QACtB,OAAO,QAAQ,CAAC,IAAI,CAAC;QACrB,OAAO,QAAQ,CAAC,WAAW,CAAC;QAC5B,OAAO,QAAQ,CAAC,YAAY,CAAC;QAC7B,OAAO,QAAQ,CAAC,cAAc,CAAC;IACjC,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,KAAK,MAAM,QAAQ,IAAI,YAAY,EAAE,CAAC;YACpC,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,EAAM,EAAE,OAA4B;IACnE,MAAM,cAAc,GAAG,OAAO,EAAE,cAAc,CAAC;IAC/C,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,KAAK,CAAC;IACpC,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,CAAC;IACnC,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,CAAC;IAC3C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,CAAC;IACrC,MAAM,WAAW,GAAG,OAAO,EAAE,WAAW,CAAC;IAEzC,MAAM,QAAQ,GAAY;QACxB,GAAG,EAAE,OAAO,CAAC,EAAE,EAAE,cAAc,EAAE,SAAS,EAAE,WAAW,CAAC;QACxD,QAAQ,EAAE,YAAY,CAAC,EAAE,EAAE,WAAW,CAAC;QACvC,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC9B,YAAY,EAAE,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC5C,WAAW,EAAE,eAAe,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC1C,OAAO,EAAE,WAAW,CAAC,EAAE,EAAE,cAAc,EAAE,WAAW,CAAC;QACrD,QAAQ,EAAE,YAAY,CAAC,EAAE,EAAE,cAAc,EAAE,SAAS,EAAE,WAAW,CAAC;QAClE,cAAc,EAAE,kBAAkB,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC;QAC3D,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;QAClB,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;QAClB,OAAO,EAAE,WAAW,CAAC,EAAE,CAAC;QACxB,UAAU,EAAE,cAAc,CAAC,EAAE,CAAC;QAC9B,MAAM,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3E,KAAK,EAAE,SAAS,EAAE;QAClB,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC;QACvC,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;KACzB,CAAC;IAEF,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;QAC9B,QAAQ,CAAC,MAAM,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,WAAW,CAAC,QAAQ,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;AACnD,CAAC","sourcesContent":["import { gotoTool } from \"./goto.js\";\nimport { actTool } from \"./act.js\";\nimport { screenshotTool } from \"./screenshot.js\";\nimport { waitTool } from \"./wait.js\";\nimport { navBackTool } from \"./navback.js\";\nimport { ariaTreeTool } from \"./ariaTree.js\";\nimport { fillFormTool } from \"./fillform.js\";\nimport { scrollTool, scrollVisionTool } from \"./scroll.js\";\nimport { extractTool } from \"./extract.js\";\nimport { clickTool } from \"./click.js\";\nimport { typeTool } from \"./type.js\";\nimport { dragAndDropTool } from \"./dragAndDrop.js\";\nimport { clickAndHoldTool } from \"./clickAndHold.js\";\nimport { keysTool } from \"./keys.js\";\nimport { fillFormVisionTool } from \"./fillFormVision.js\";\nimport { thinkTool } from \"./think.js\";\nimport { searchTool } from \"./search.js\";\n\nimport type { ToolSet, InferUITools } from \"ai\";\nimport type { V3 } from \"../../v3.js\";\nimport type { LogLine } from \"../../types/public/logs.js\";\nimport type {\n  AgentToolMode,\n  AgentModelConfig,\n  Variables,\n} from \"../../types/public/agent.js\";\n\nexport interface V3AgentToolOptions {\n  executionModel?: string | AgentModelConfig;\n  logger?: (message: LogLine) => void;\n  /**\n   * Tool mode determines which set of tools are available.\n   * - 'dom' (default): Uses DOM-based tools (act, fillForm) - removes coordinate-based tools\n   * - 'hybrid': Uses coordinate-based tools (click, type, dragAndDrop, etc.) - removes fillForm\n   */\n  mode?: AgentToolMode;\n  /**\n   * The model provider. Used for model-specific coordinate handling\n   */\n  provider?: string;\n  /**\n   * Tools to exclude from the available toolset.\n   * These tools will be filtered out after mode-based filtering.\n   */\n  excludeTools?: string[];\n  /**\n   * Variables available to the agent for use in act/type tools.\n   * When provided, these tools will have an optional useVariable field.\n   */\n  variables?: Variables;\n  /**\n   * Timeout in milliseconds for tool calls that invoke v3 methods (act, extract, fillForm, ariaTree).\n   * Forwarded to the underlying v3 call's `timeout` option.\n   */\n  toolTimeout?: number;\n}\n\n/**\n * Filters tools based on mode and explicit exclusions.\n * - 'dom' mode: Removes coordinate-based tools (click, type, dragAndDrop, clickAndHold, fillFormVision)\n * - 'hybrid' mode: Removes DOM-based form tool (fillForm) in favor of coordinate-based fillFormVision\n * - excludeTools: Additional tools to remove from the toolset\n */\nfunction filterTools(\n  tools: ToolSet,\n  mode: AgentToolMode,\n  excludeTools?: string[],\n): ToolSet {\n  const filtered: ToolSet = { ...tools };\n\n  // Mode-based filtering\n  if (mode === \"hybrid\") {\n    delete filtered.fillForm;\n  } else {\n    // DOM mode (default)\n    delete filtered.click;\n    delete filtered.type;\n    delete filtered.dragAndDrop;\n    delete filtered.clickAndHold;\n    delete filtered.fillFormVision;\n  }\n\n  if (excludeTools) {\n    for (const toolName of excludeTools) {\n      delete filtered[toolName];\n    }\n  }\n\n  return filtered;\n}\n\nexport function createAgentTools(v3: V3, options?: V3AgentToolOptions) {\n  const executionModel = options?.executionModel;\n  const mode = options?.mode ?? \"dom\";\n  const provider = options?.provider;\n  const excludeTools = options?.excludeTools;\n  const variables = options?.variables;\n  const toolTimeout = options?.toolTimeout;\n\n  const allTools: ToolSet = {\n    act: actTool(v3, executionModel, variables, toolTimeout),\n    ariaTree: ariaTreeTool(v3, toolTimeout),\n    click: clickTool(v3, provider),\n    clickAndHold: clickAndHoldTool(v3, provider),\n    dragAndDrop: dragAndDropTool(v3, provider),\n    extract: extractTool(v3, executionModel, toolTimeout),\n    fillForm: fillFormTool(v3, executionModel, variables, toolTimeout),\n    fillFormVision: fillFormVisionTool(v3, provider, variables),\n    goto: gotoTool(v3),\n    keys: keysTool(v3),\n    navback: navBackTool(v3),\n    screenshot: screenshotTool(v3),\n    scroll: mode === \"hybrid\" ? scrollVisionTool(v3, provider) : scrollTool(v3),\n    think: thinkTool(),\n    type: typeTool(v3, provider, variables),\n    wait: waitTool(v3, mode),\n  };\n\n  if (process.env.BRAVE_API_KEY) {\n    allTools.search = searchTool(v3);\n  }\n\n  return filterTools(allTools, mode, excludeTools);\n}\n\nexport type AgentTools = ReturnType<typeof createAgentTools>;\n\n/**\n * Type map of all agent tools for strong typing of tool calls and results.\n * Note: `search` is optional as it's only available when BRAVE_API_KEY is configured.\n */\nexport type AgentToolTypesMap = {\n  act: ReturnType<typeof actTool>;\n  ariaTree: ReturnType<typeof ariaTreeTool>;\n  click: ReturnType<typeof clickTool>;\n  clickAndHold: ReturnType<typeof clickAndHoldTool>;\n  dragAndDrop: ReturnType<typeof dragAndDropTool>;\n  extract: ReturnType<typeof extractTool>;\n  fillForm: ReturnType<typeof fillFormTool>;\n  fillFormVision: ReturnType<typeof fillFormVisionTool>;\n  goto: ReturnType<typeof gotoTool>;\n  keys: ReturnType<typeof keysTool>;\n  navback: ReturnType<typeof navBackTool>;\n  screenshot: ReturnType<typeof screenshotTool>;\n  scroll: ReturnType<typeof scrollTool> | ReturnType<typeof scrollVisionTool>;\n  search?: ReturnType<typeof searchTool>;\n  think: ReturnType<typeof thinkTool>;\n  type: ReturnType<typeof typeTool>;\n  wait: ReturnType<typeof waitTool>;\n};\n\n/**\n * Inferred UI tools type for type-safe tool inputs and outputs.\n * Use with UIMessage for full type safety in UI contexts.\n */\nexport type AgentUITools = InferUITools<AgentToolTypesMap>;\n\n/**\n * Union type for all possible agent tool calls.\n * Provides type-safe access to tool call arguments.\n */\nexport type AgentToolCall = {\n  [K in keyof AgentToolTypesMap]: {\n    toolName: K;\n    toolCallId: string;\n    args: AgentUITools[K][\"input\"];\n  };\n}[keyof AgentToolTypesMap];\n\n/**\n * Union type for all possible agent tool results.\n * Provides type-safe access to tool result values.\n */\nexport type AgentToolResult = {\n  [K in keyof AgentToolTypesMap]: {\n    toolName: K;\n    toolCallId: string;\n    result: AgentUITools[K][\"output\"];\n  };\n}[keyof AgentToolTypesMap];\n"]}

package/dist/esm/lib/v3/agent/tools/screenshot.d.ts

@@ -1,6 +1,14 @@
 import type { V3 } from "../../v3.js";
 export declare const screenshotTool: (v3: V3) => import("ai").Tool<Record<string, never>, {
+    success: boolean;
     base64: string;
     timestamp: number;
     pageUrl: string;
+    error?: undefined;
+} | {
+    success: boolean;
+    error: string;
+    base64?: undefined;
+    timestamp?: undefined;
+    pageUrl?: undefined;
 }>;