@browserbasehq/orca 3.0.9-alpha-4 → 3.1.0-patch.1

package/dist/esm/lib/v3/agent/tools/clickAndHold.js

@@ -0,0 +1,68 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { processCoordinates } from "../utils/coordinateNormalization.js";
+import { ensureXPath } from "../utils/xpath.js";
+export const clickAndHoldTool = (v3, provider) => tool({
+    description: "Click and hold on an element using its coordinates",
+    inputSchema: z.object({
+        describe: z
+            .string()
+            .describe("Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description"),
+        duration: z
+            .number()
+            .describe("The duration to hold the element in milliseconds"),
+        coordinates: z
+            .array(z.number())
+            .describe("The (x, y) coordinates to click on"),
+    }),
+    execute: async ({ describe, coordinates, duration }) => {
+        try {
+            const page = await v3.context.awaitActivePage();
+            const processed = processCoordinates(coordinates[0], coordinates[1], provider, v3);
+            v3.logger({
+                category: "agent",
+                message: `Agent calling tool: clickAndHold`,
+                level: 1,
+                auxiliary: {
+                    arguments: {
+                        value: JSON.stringify({
+                            describe,
+                            duration,
+                        }),
+                        type: "object",
+                    },
+                },
+            });
+            // Only request XPath when caching is enabled to avoid unnecessary computation
+            const shouldCollectXpath = v3.isAgentReplayActive();
+            // Use dragAndDrop from same point to same point with delay to simulate click and hold
+            const [xpath] = await page.dragAndDrop(processed.x, processed.y, processed.x, processed.y, { delay: duration, returnXpath: shouldCollectXpath });
+            // Record as "act" step with proper Action for deterministic replay (only when caching)
+            if (shouldCollectXpath) {
+                const normalizedXpath = ensureXPath(xpath);
+                if (normalizedXpath) {
+                    const action = {
+                        selector: normalizedXpath,
+                        description: describe,
+                        method: "clickAndHold",
+                        arguments: [String(duration)],
+                    };
+                    v3.recordAgentReplayStep({
+                        type: "act",
+                        instruction: describe,
+                        actions: [action],
+                        actionDescription: describe,
+                    });
+                }
+            }
+            return { success: true, describe };
+        }
+        catch (error) {
+            return {
+                success: false,
+                error: `Error clicking and holding: ${error.message}`,
+            };
+        }
+    },
+});
+//# sourceMappingURL=clickAndHold.js.map

package/dist/esm/lib/v3/agent/tools/clickAndHold.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"clickAndHold.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/clickAndHold.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC5D,IAAI,CAAC;IACH,WAAW,EAAE,oDAAoD;IACjE,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CAAC,kDAAkD,CAAC;QAC/D,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,EAAE,EAAE;QACrD,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,kCAAkC;gBAC3C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;4BACR,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YAEpD,sFAAsF;YACtF,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACpC,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACrD,CAAC;YAEF,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,cAAc;wBACtB,SAAS,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;qBAC9B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QACrC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,+BAAgC,KAAe,CAAC,OAAO,EAAE;aACjE,CAAC;QACJ,CAAC;IACH,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\n\nexport const clickAndHoldTool = (v3: V3, provider?: string) =>\n  tool({\n    description: \"Click and hold on an element using its coordinates\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      duration: z\n        .number()\n        .describe(\"The duration to hold the element in milliseconds\"),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to click on\"),\n    }),\n    execute: async ({ describe, coordinates, duration }) => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: clickAndHold`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({\n                describe,\n                duration,\n              }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n\n        // Use dragAndDrop from same point to same point with delay to simulate click and hold\n        const [xpath] = await page.dragAndDrop(\n          processed.x,\n          processed.y,\n          processed.x,\n          processed.y,\n          { delay: duration, returnXpath: shouldCollectXpath },\n        );\n\n        // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"clickAndHold\",\n              arguments: [String(duration)],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return { success: true, describe };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error clicking and holding: ${(error as Error).message}`,\n        };\n      }\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/dragAndDrop.d.ts

@@ -0,0 +1,7 @@
+import type { V3 } from "../../v3.js";
+import type { DragAndDropToolResult } from "../../types/public/agent.js";
+export declare const dragAndDropTool: (v3: V3, provider?: string) => import("ai").Tool<{
+    describe: string;
+    startCoordinates: number[];
+    endCoordinates: number[];
+}, DragAndDropToolResult>;

package/dist/esm/lib/v3/agent/tools/dragAndDrop.js

@@ -0,0 +1,105 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { processCoordinates } from "../utils/coordinateNormalization.js";
+import { ensureXPath } from "../utils/xpath.js";
+import { waitAndCaptureScreenshot } from "../utils/screenshotHandler.js";
+export const dragAndDropTool = (v3, provider) => tool({
+    description: "Drag and drop an element using its coordinates (this is the most reliable way to drag and drop an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)",
+    inputSchema: z.object({
+        describe: z.string().describe("Describe the element to drag and drop"),
+        startCoordinates: z
+            .array(z.number())
+            .describe("The (x, y) coordinates to start the drag and drop from"),
+        endCoordinates: z
+            .array(z.number())
+            .describe("The (x, y) coordinates to end the drag and drop at"),
+    }),
+    execute: async ({ describe, startCoordinates, endCoordinates, }) => {
+        try {
+            const page = await v3.context.awaitActivePage();
+            const processedStart = processCoordinates(startCoordinates[0], startCoordinates[1], provider, v3);
+            const processedEnd = processCoordinates(endCoordinates[0], endCoordinates[1], provider, v3);
+            v3.logger({
+                category: "agent",
+                message: `Agent calling tool: dragAndDrop`,
+                level: 1,
+                auxiliary: {
+                    arguments: {
+                        value: JSON.stringify({
+                            describe,
+                        }),
+                        type: "object",
+                    },
+                },
+            });
+            // Only request XPath when caching is enabled to avoid unnecessary computation
+            const shouldCollectXpath = v3.isAgentReplayActive();
+            const [fromXpath, toXpath] = await page.dragAndDrop(processedStart.x, processedStart.y, processedEnd.x, processedEnd.y, { returnXpath: shouldCollectXpath });
+            const screenshotBase64 = await waitAndCaptureScreenshot(page);
+            // Record as "act" step with proper Action for deterministic replay (only when caching)
+            if (shouldCollectXpath) {
+                const normalizedFrom = ensureXPath(fromXpath);
+                const normalizedTo = ensureXPath(toXpath);
+                if (normalizedFrom && normalizedTo) {
+                    const action = {
+                        selector: normalizedFrom,
+                        description: describe,
+                        method: "dragAndDrop",
+                        arguments: [normalizedTo],
+                    };
+                    v3.recordAgentReplayStep({
+                        type: "act",
+                        instruction: describe,
+                        actions: [action],
+                        actionDescription: describe,
+                    });
+                }
+            }
+            return {
+                success: true,
+                describe,
+                screenshotBase64,
+            };
+        }
+        catch (error) {
+            return {
+                success: false,
+                error: `Error dragging: ${error.message}`,
+            };
+        }
+    },
+    toModelOutput: (result) => {
+        if (result.success) {
+            const content = [
+                {
+                    type: "text",
+                    text: JSON.stringify({
+                        success: result.success,
+                        describe: result.describe,
+                    }),
+                },
+            ];
+            if (result.screenshotBase64) {
+                content.push({
+                    type: "media",
+                    mediaType: "image/png",
+                    data: result.screenshotBase64,
+                });
+            }
+            return { type: "content", value: content };
+        }
+        return {
+            type: "content",
+            value: [
+                {
+                    type: "text",
+                    text: JSON.stringify({
+                        success: result.success,
+                        error: result.error,
+                    }),
+                },
+            ],
+        };
+    },
+});
+//# sourceMappingURL=dragAndDrop.js.map

package/dist/esm/lib/v3/agent/tools/dragAndDrop.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dragAndDrop.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/dragAndDrop.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC3D,IAAI,CAAC;IACH,WAAW,EACT,kNAAkN;IACpN,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uCAAuC,CAAC;QACtE,gBAAgB,EAAE,CAAC;aAChB,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,wDAAwD,CAAC;QACrE,cAAc,EAAE,CAAC;aACd,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oDAAoD,CAAC;KAClE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,gBAAgB,EAChB,cAAc,GACf,EAAkC,EAAE;QACnC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,cAAc,GAAG,kBAAkB,CACvC,gBAAgB,CAAC,CAAC,CAAC,EACnB,gBAAgB,CAAC,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;YACF,MAAM,YAAY,GAAG,kBAAkB,CACrC,cAAc,CAAC,CAAC,CAAC,EACjB,cAAc,CAAC,CAAC,CAAC,EACjB,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,iCAAiC;gBAC1C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACjD,cAAc,CAAC,CAAC,EAChB,cAAc,CAAC,CAAC,EAChB,YAAY,CAAC,CAAC,EACd,YAAY,CAAC,CAAC,EACd,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACpC,CAAC;YAEF,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE9D,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,cAAc,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;gBAC9C,MAAM,YAAY,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;gBAC1C,IAAI,cAAc,IAAI,YAAY,EAAE,CAAC;oBACnC,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,cAAc;wBACxB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,aAAa;wBACrB,SAAS,EAAE,CAAC,YAAY,CAAC;qBAC1B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAoB,KAAe,CAAC,OAAO,EAAE;aACrD,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;qBAC1B,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;QACD,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,KAAK,EAAE,MAAM,CAAC,KAAK;qBACpB,CAAC;iBACH;aACF;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  DragAndDropToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const dragAndDropTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Drag and drop an element using its coordinates (this is the most reliable way to drag and drop an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z.string().describe(\"Describe the element to drag and drop\"),\n      startCoordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to start the drag and drop from\"),\n      endCoordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to end the drag and drop at\"),\n    }),\n    execute: async ({\n      describe,\n      startCoordinates,\n      endCoordinates,\n    }): Promise<DragAndDropToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processedStart = processCoordinates(\n          startCoordinates[0],\n          startCoordinates[1],\n          provider,\n          v3,\n        );\n        const processedEnd = processCoordinates(\n          endCoordinates[0],\n          endCoordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: dragAndDrop`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({\n                describe,\n              }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const [fromXpath, toXpath] = await page.dragAndDrop(\n          processedStart.x,\n          processedStart.y,\n          processedEnd.x,\n          processedEnd.y,\n          { returnXpath: shouldCollectXpath },\n        );\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedFrom = ensureXPath(fromXpath);\n          const normalizedTo = ensureXPath(toXpath);\n          if (normalizedFrom && normalizedTo) {\n            const action: Action = {\n              selector: normalizedFrom,\n              description: describe,\n              method: \"dragAndDrop\",\n              arguments: [normalizedTo],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error dragging: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success) {\n        const content: ModelOutputContentItem[] = [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              describe: result.describe,\n            }),\n          },\n        ];\n        if (result.screenshotBase64) {\n          content.push({\n            type: \"media\",\n            mediaType: \"image/png\",\n            data: result.screenshotBase64,\n          });\n        }\n        return { type: \"content\", value: content };\n      }\n      return {\n        type: \"content\",\n        value: [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              error: result.error,\n            }),\n          },\n        ],\n      };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/extract.d.ts

@@ -0,0 +1,20 @@
+import type { V3 } from "../../v3.js";
+export declare const extractTool: (v3: V3, executionModel?: string) => import("ai").Tool<{
+    instruction: string;
+    schema?: {
+        [x: string]: unknown;
+        type?: string;
+        properties?: Record<string, unknown>;
+        items?: unknown;
+        enum?: string[];
+        format?: "email" | "uuid" | "url";
+    };
+}, {
+    success: boolean;
+    result: unknown;
+    error?: undefined;
+} | {
+    success: boolean;
+    error: string;
+    result?: undefined;
+}>;

package/dist/esm/lib/v3/agent/tools/extract.js

@@ -0,0 +1,93 @@
+import { tool } from "ai";
+import { z } from "zod";
+function jsonSchemaToZod(schema) {
+    switch (schema.type) {
+        case "object": {
+            const shape = {};
+            if (schema.properties) {
+                for (const [key, value] of Object.entries(schema.properties)) {
+                    shape[key] = jsonSchemaToZod(value);
+                }
+            }
+            return z.object(shape);
+        }
+        case "array":
+            return z.array(schema.items ? jsonSchemaToZod(schema.items) : z.any());
+        case "string": {
+            let s = z.string();
+            if (schema.format === "url")
+                s = s.url();
+            if (schema.format === "email")
+                s = s.email();
+            if (schema.format === "uuid")
+                s = s.uuid();
+            if (schema.enum && schema.enum.length > 0)
+                return z.enum(schema.enum);
+            return s;
+        }
+        case "number":
+        case "integer":
+            return z.number();
+        case "boolean":
+            return z.boolean();
+        case "null":
+            return z.null();
+        default:
+            return z.any();
+    }
+}
+export const extractTool = (v3, executionModel) => tool({
+    description: `Extract structured data from the current page based on a provided schema.
+    
+    USAGE GUIDELINES:
+    - Keep schemas MINIMAL - only include fields essential for the task
+    - IMPORTANT: only use this if explicitly asked for structured output. In most scenarios, you should use the aria tree tool over this.
+    - For URL fields, use format: "url"
+    
+    EXAMPLES:
+    1. Extract a single value:
+       instruction: "extract the product price"
+       schema: { type: "object", properties: { price: { type: "number" } } }
+    
+    2. Extract multiple fields:
+       instruction: "extract product name and price"
+       schema: { type: "object", properties: { name: { type: "string" }, price: { type: "number" } } }
+    
+    3. Extract arrays:
+       instruction: "extract all product names and prices"
+       schema: { type: "object", properties: { products: { type: "array", items: { type: "object", properties: { name: { type: "string" }, price: { type: "number" } } } } } }
+    
+    4. Extract a URL:
+       instruction: "extract the link"
+       schema: { type: "object", properties: { url: { type: "string", format: "url" } } }`,
+    inputSchema: z.object({
+        instruction: z.string(),
+        schema: z
+            .object({
+            type: z.string().optional(),
+            properties: z.record(z.string(), z.unknown()).optional(),
+            items: z.unknown().optional(),
+            enum: z.array(z.string()).optional(),
+            format: z.enum(["url", "email", "uuid"]).optional(),
+        })
+            .passthrough()
+            .optional()
+            .describe("JSON Schema object describing the structure to extract"),
+    }),
+    execute: async ({ instruction, schema }) => {
+        try {
+            const parsedSchema = schema
+                ? jsonSchemaToZod(schema)
+                : undefined;
+            const result = await v3.extract(instruction, parsedSchema, {
+                ...(executionModel ? { model: executionModel } : {}),
+            });
+            return { success: true, result };
+        }
+        catch (error) {
+            const err = error;
+            return { success: false, error: err?.message ?? String(error) };
+        }
+    },
+});
+//# sourceMappingURL=extract.js.map

package/dist/esm/lib/v3/agent/tools/extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAc,MAAM,KAAK,CAAC;AAWpC,SAAS,eAAe,CAAC,MAAkB;IACzC,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,KAAK,GAA+B,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;gBACtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC7D,KAAK,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;gBACtC,CAAC;YACH,CAAC;YACD,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;QACD,KAAK,OAAO;YACV,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QACzE,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;YACnB,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK;gBAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO;gBAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM;gBAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAC3C,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;gBACvC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAA6B,CAAC,CAAC;YACtD,OAAO,CAAC,CAAC;QACX,CAAC;QACD,KAAK,QAAQ,CAAC;QACd,KAAK,SAAS;YACZ,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;QACpB,KAAK,SAAS;YACZ,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;QACrB,KAAK,MAAM;YACT,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAClB;YACE,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,EAAM,EAAE,cAAuB,EAAE,EAAE,CAC7D,IAAI,CAAC;IACH,WAAW,EAAE;;;;;;;;;;;;;;;;;;;;;;0FAsByE;IACtF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;QACvB,MAAM,EAAE,CAAC;aACN,MAAM,CAAC;YACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC3B,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;YACxD,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;YAC7B,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;YACpC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;SACpD,CAAC;aACD,WAAW,EAAE;aACb,QAAQ,EAAE;aACV,QAAQ,CAAC,wDAAwD,CAAC;KACtE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE;QACzC,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,MAAM;gBACzB,CAAC,CAAC,eAAe,CAAC,MAAoB,CAAC;gBACvC,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,YAAY,EAAE;gBACzD,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACrD,CAAC,CAAC;YACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,GAAG,GAAG,KAAc,CAAC;YAC3B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAClE,CAAC;IACH,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z, ZodTypeAny } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\ninterface JsonSchema {\n  type?: string;\n  properties?: Record<string, JsonSchema>;\n  items?: JsonSchema;\n  enum?: string[];\n  format?: \"url\" | \"email\" | \"uuid\";\n}\n\nfunction jsonSchemaToZod(schema: JsonSchema): ZodTypeAny {\n  switch (schema.type) {\n    case \"object\": {\n      const shape: Record<string, ZodTypeAny> = {};\n      if (schema.properties) {\n        for (const [key, value] of Object.entries(schema.properties)) {\n          shape[key] = jsonSchemaToZod(value);\n        }\n      }\n      return z.object(shape);\n    }\n    case \"array\":\n      return z.array(schema.items ? jsonSchemaToZod(schema.items) : z.any());\n    case \"string\": {\n      let s = z.string();\n      if (schema.format === \"url\") s = s.url();\n      if (schema.format === \"email\") s = s.email();\n      if (schema.format === \"uuid\") s = s.uuid();\n      if (schema.enum && schema.enum.length > 0)\n        return z.enum(schema.enum as [string, ...string[]]);\n      return s;\n    }\n    case \"number\":\n    case \"integer\":\n      return z.number();\n    case \"boolean\":\n      return z.boolean();\n    case \"null\":\n      return z.null();\n    default:\n      return z.any();\n  }\n}\n\nexport const extractTool = (v3: V3, executionModel?: string) =>\n  tool({\n    description: `Extract structured data from the current page based on a provided schema.\n    \n    USAGE GUIDELINES:\n    - Keep schemas MINIMAL - only include fields essential for the task\n    - IMPORTANT: only use this if explicitly asked for structured output. In most scenarios, you should use the aria tree tool over this.\n    - For URL fields, use format: \"url\"\n    \n    EXAMPLES:\n    1. Extract a single value:\n       instruction: \"extract the product price\"\n       schema: { type: \"object\", properties: { price: { type: \"number\" } } }\n    \n    2. Extract multiple fields:\n       instruction: \"extract product name and price\"\n       schema: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } }\n    \n    3. Extract arrays:\n       instruction: \"extract all product names and prices\"\n       schema: { type: \"object\", properties: { products: { type: \"array\", items: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } } } } }\n    \n    4. Extract a URL:\n       instruction: \"extract the link\"\n       schema: { type: \"object\", properties: { url: { type: \"string\", format: \"url\" } } }`,\n    inputSchema: z.object({\n      instruction: z.string(),\n      schema: z\n        .object({\n          type: z.string().optional(),\n          properties: z.record(z.string(), z.unknown()).optional(),\n          items: z.unknown().optional(),\n          enum: z.array(z.string()).optional(),\n          format: z.enum([\"url\", \"email\", \"uuid\"]).optional(),\n        })\n        .passthrough()\n        .optional()\n        .describe(\"JSON Schema object describing the structure to extract\"),\n    }),\n    execute: async ({ instruction, schema }) => {\n      try {\n        const parsedSchema = schema\n          ? jsonSchemaToZod(schema as JsonSchema)\n          : undefined;\n        const result = await v3.extract(instruction, parsedSchema, {\n          ...(executionModel ? { model: executionModel } : {}),\n        });\n        return { success: true, result };\n      } catch (error) {\n        const err = error as Error;\n        return { success: false, error: err?.message ?? String(error) };\n      }\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/fillFormVision.d.ts

@@ -0,0 +1,12 @@
+import type { V3 } from "../../v3.js";
+import type { FillFormVisionToolResult } from "../../types/public/agent.js";
+export declare const fillFormVisionTool: (v3: V3, provider?: string) => import("ai").Tool<{
+    fields: {
+        action: string;
+        value: string;
+        coordinates: {
+            x: number;
+            y: number;
+        };
+    }[];
+}, FillFormVisionToolResult>;

package/dist/esm/lib/v3/agent/tools/fillFormVision.js

@@ -0,0 +1,143 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { processCoordinates } from "../utils/coordinateNormalization.js";
+import { ensureXPath } from "../utils/xpath.js";
+import { waitAndCaptureScreenshot } from "../utils/screenshotHandler.js";
+export const fillFormVisionTool = (v3, provider) => tool({
+    description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL
+
+CRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)
+IMPORTANT: Ensure the fields are visible within the current viewport
+
+WHY THIS TOOL EXISTS:
+- Forms are the #1 use case for multi-field input
+- Optimized specifically for input/textarea elements
+- 4-6x faster than individual typing actions
+
+Use fillFormVision: Pure form filling (inputs, textareas only)
+MANDATORY USE CASES (always use fillFormVision for these):
+- Registration forms: name, email, password fields
+- Contact forms: name, email, message fields
+- Checkout forms: address, payment info fields
+- Profile updates: multiple user data fields
+- Search filters: multiple criteria inputs`,
+    inputSchema: z.object({
+        fields: z
+            .array(z.object({
+            action: z
+                .string()
+                .describe("Description of the typing action, e.g. 'type foo into the bar field'"),
+            value: z.string().describe("Text to type into the target field"),
+            coordinates: z
+                .object({
+                x: z.number(),
+                y: z.number(),
+            })
+                .describe("Coordinates of the target field"),
+        }))
+            .min(2, "Provide at least two fields to fill"),
+    }),
+    execute: async ({ fields }) => {
+        try {
+            const page = await v3.context.awaitActivePage();
+            // Process coordinates for each field
+            const processedFields = fields.map((field) => {
+                const processed = processCoordinates(field.coordinates.x, field.coordinates.y, provider, v3);
+                return {
+                    ...field,
+                    coordinates: { x: processed.x, y: processed.y },
+                };
+            });
+            v3.logger({
+                category: "agent",
+                message: `Agent calling tool: fillFormVision`,
+                level: 1,
+                auxiliary: {
+                    arguments: {
+                        value: JSON.stringify({ fields, processedFields }),
+                        type: "object",
+                    },
+                },
+            });
+            // Only request XPath when caching is enabled to avoid unnecessary computation
+            const shouldCollectXpath = v3.isAgentReplayActive();
+            const actions = [];
+            for (const field of processedFields) {
+                // Click the field, only requesting XPath when caching is enabled
+                const xpath = await page.click(field.coordinates.x, field.coordinates.y, {
+                    returnXpath: shouldCollectXpath,
+                });
+                await page.type(field.value);
+                // Build Action with XPath for deterministic replay (only when caching)
+                if (shouldCollectXpath) {
+                    const normalizedXpath = ensureXPath(xpath);
+                    if (normalizedXpath) {
+                        actions.push({
+                            selector: normalizedXpath,
+                            description: field.action,
+                            method: "type",
+                            arguments: [field.value],
+                        });
+                    }
+                }
+                // Small delay between fields
+                await new Promise((resolve) => setTimeout(resolve, 100));
+            }
+            const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);
+            // Record as "act" step with proper Actions for deterministic replay (only when caching)
+            if (shouldCollectXpath && actions.length > 0) {
+                v3.recordAgentReplayStep({
+                    type: "act",
+                    instruction: `Fill ${fields.length} form fields`,
+                    actions,
+                    actionDescription: `Fill ${fields.length} form fields`,
+                });
+            }
+            return {
+                success: true,
+                playwrightArguments: processedFields,
+                screenshotBase64,
+            };
+        }
+        catch (error) {
+            return {
+                success: false,
+                error: `Error filling form: ${error.message}`,
+            };
+        }
+    },
+    toModelOutput: (result) => {
+        if (result.success) {
+            const content = [
+                {
+                    type: "text",
+                    text: JSON.stringify({
+                        success: result.success,
+                        fieldsCount: result.playwrightArguments?.length ?? 0,
+                    }),
+                },
+            ];
+            if (result.screenshotBase64) {
+                content.push({
+                    type: "media",
+                    mediaType: "image/png",
+                    data: result.screenshotBase64,
+                });
+            }
+            return { type: "content", value: content };
+        }
+        return {
+            type: "content",
+            value: [
+                {
+                    type: "text",
+                    text: JSON.stringify({
+                        success: result.success,
+                        error: result.error,
+                    }),
+                },
+            ],
+        };
+    },
+});
+//# sourceMappingURL=fillFormVision.js.map

package/dist/esm/lib/v3/agent/tools/fillFormVision.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fillFormVision.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillFormVision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC9D,IAAI,CAAC;IACH,WAAW,EAAE;;;;;;;;;;;;;;;;2CAgB0B;IACvC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,MAAM,EAAE,CAAC;aACN,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;YACP,MAAM,EAAE,CAAC;iBACN,MAAM,EAAE;iBACR,QAAQ,CACP,sEAAsE,CACvE;YACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,oCAAoC,CAAC;YAChE,WAAW,EAAE,CAAC;iBACX,MAAM,CAAC;gBACN,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;gBACb,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;aACd,CAAC;iBACD,QAAQ,CAAC,iCAAiC,CAAC;SAC/C,CAAC,CACH;aACA,GAAG,CAAC,CAAC,EAAE,qCAAqC,CAAC;KACjD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAqC,EAAE;QAC/D,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAEhD,qCAAqC;YACrC,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC3C,MAAM,SAAS,GAAG,kBAAkB,CAClC,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;gBACF,OAAO;oBACL,GAAG,KAAK;oBACR,WAAW,EAAE,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;iBAChD,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,oCAAoC;gBAC7C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;wBAClD,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,OAAO,GAAa,EAAE,CAAC;YAE7B,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;gBACpC,iEAAiE;gBACjE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAC5B,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB;oBACE,WAAW,EAAE,kBAAkB;iBAChC,CACF,CAAC;gBACF,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAE7B,uEAAuE;gBACvE,IAAI,kBAAkB,EAAE,CAAC;oBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;oBAC3C,IAAI,eAAe,EAAE,CAAC;wBACpB,OAAO,CAAC,IAAI,CAAC;4BACX,QAAQ,EAAE,eAAe;4BACzB,WAAW,EAAE,KAAK,CAAC,MAAM;4BACzB,MAAM,EAAE,MAAM;4BACd,SAAS,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;yBACzB,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAED,6BAA6B;gBAC7B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YAC3D,CAAC;YAED,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAEnE,wFAAwF;YACxF,IAAI,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7C,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,KAAK;oBACX,WAAW,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;oBAChD,OAAO;oBACP,iBAAiB,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;iBACvD,CAAC,CAAC;YACL,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,mBAAmB,EAAE,eAAe;gBACpC,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,uBAAwB,KAAe,CAAC,OAAO,EAAE;aACzD,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,WAAW,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,IAAI,CAAC;qBACrD,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;QACD,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,KAAK,EAAE,MAAM,CAAC,KAAK;qBACpB,CAAC;iBACH;aACF;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  FillFormVisionToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const fillFormVisionTool = (v3: V3, provider?: string) =>\n  tool({\n    description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL\n\nCRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)\nIMPORTANT: Ensure the fields are visible within the current viewport\n\nWHY THIS TOOL EXISTS:\n- Forms are the #1 use case for multi-field input\n- Optimized specifically for input/textarea elements\n- 4-6x faster than individual typing actions\n\nUse fillFormVision: Pure form filling (inputs, textareas only)\nMANDATORY USE CASES (always use fillFormVision for these):\n- Registration forms: name, email, password fields\n- Contact forms: name, email, message fields\n- Checkout forms: address, payment info fields\n- Profile updates: multiple user data fields\n- Search filters: multiple criteria inputs`,\n    inputSchema: z.object({\n      fields: z\n        .array(\n          z.object({\n            action: z\n              .string()\n              .describe(\n                \"Description of the typing action, e.g. 'type foo into the bar field'\",\n              ),\n            value: z.string().describe(\"Text to type into the target field\"),\n            coordinates: z\n              .object({\n                x: z.number(),\n                y: z.number(),\n              })\n              .describe(\"Coordinates of the target field\"),\n          }),\n        )\n        .min(2, \"Provide at least two fields to fill\"),\n    }),\n    execute: async ({ fields }): Promise<FillFormVisionToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n\n        // Process coordinates for each field\n        const processedFields = fields.map((field) => {\n          const processed = processCoordinates(\n            field.coordinates.x,\n            field.coordinates.y,\n            provider,\n            v3,\n          );\n          return {\n            ...field,\n            coordinates: { x: processed.x, y: processed.y },\n          };\n        });\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: fillFormVision`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ fields, processedFields }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const actions: Action[] = [];\n\n        for (const field of processedFields) {\n          // Click the field, only requesting XPath when caching is enabled\n          const xpath = await page.click(\n            field.coordinates.x,\n            field.coordinates.y,\n            {\n              returnXpath: shouldCollectXpath,\n            },\n          );\n          await page.type(field.value);\n\n          // Build Action with XPath for deterministic replay (only when caching)\n          if (shouldCollectXpath) {\n            const normalizedXpath = ensureXPath(xpath);\n            if (normalizedXpath) {\n              actions.push({\n                selector: normalizedXpath,\n                description: field.action,\n                method: \"type\",\n                arguments: [field.value],\n              });\n            }\n          }\n\n          // Small delay between fields\n          await new Promise((resolve) => setTimeout(resolve, 100));\n        }\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n        // Record as \"act\" step with proper Actions for deterministic replay (only when caching)\n        if (shouldCollectXpath && actions.length > 0) {\n          v3.recordAgentReplayStep({\n            type: \"act\",\n            instruction: `Fill ${fields.length} form fields`,\n            actions,\n            actionDescription: `Fill ${fields.length} form fields`,\n          });\n        }\n\n        return {\n          success: true,\n          playwrightArguments: processedFields,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error filling form: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success) {\n        const content: ModelOutputContentItem[] = [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              fieldsCount: result.playwrightArguments?.length ?? 0,\n            }),\n          },\n        ];\n        if (result.screenshotBase64) {\n          content.push({\n            type: \"media\",\n            mediaType: \"image/png\",\n            data: result.screenshotBase64,\n          });\n        }\n        return { type: \"content\", value: content };\n      }\n      return {\n        type: \"content\",\n        value: [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              error: result.error,\n            }),\n          },\n        ],\n      };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/fillform.d.ts

@@ -0,0 +1,12 @@
+import type { V3 } from "../../v3.js";
+import type { Action } from "../../types/public/methods.js";
+export declare const fillFormTool: (v3: V3, executionModel?: string) => import("ai").Tool<{
+    fields: {
+        action: string;
+        value: string;
+    }[];
+}, {
+    success: boolean;
+    actions: unknown[];
+    playwrightArguments: Action[];
+}>;

package/dist/esm/lib/v3/agent/tools/fillform.js

@@ -0,0 +1,56 @@
+import { tool } from "ai";
+import { z } from "zod";
+export const fillFormTool = (v3, executionModel) => tool({
+    description: `📝 FORM FILL - MULTI-FIELD INPUT TOOL\nFor any form with 2+ inputs/textareas. Faster than individual typing.`,
+    inputSchema: z.object({
+        fields: z
+            .array(z.object({
+            action: z
+                .string()
+                .describe('Description of typing action, e.g. "type foo into the email field"'),
+            value: z.string().describe("Text to type into the target"),
+        }))
+            .min(1, "Provide at least one field to fill"),
+    }),
+    execute: async ({ fields }) => {
+        v3.logger({
+            category: "agent",
+            message: `Agent calling tool: fillForm`,
+            level: 1,
+            auxiliary: {
+                arguments: {
+                    value: JSON.stringify(fields),
+                    type: "object",
+                },
+            },
+        });
+        const instruction = `Return observation results for the following actions: ${fields
+            .map((f) => f.action)
+            .join(", ")}`;
+        const observeOptions = executionModel
+            ? { model: executionModel }
+            : undefined;
+        const observeResults = await v3.observe(instruction, observeOptions);
+        const completed = [];
+        const replayableActions = [];
+        for (const res of observeResults) {
+            const actResult = await v3.act(res);
+            completed.push(actResult);
+            if (Array.isArray(actResult.actions)) {
+                replayableActions.push(...actResult.actions);
+            }
+        }
+        v3.recordAgentReplayStep({
+            type: "fillForm",
+            fields,
+            observeResults,
+            actions: replayableActions,
+        });
+        return {
+            success: true,
+            actions: completed,
+            playwrightArguments: replayableActions,
+        };
+    },
+});
+//# sourceMappingURL=fillform.js.map

package/dist/esm/lib/v3/agent/tools/fillform.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fillform.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillform.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,EAAM,EAAE,cAAuB,EAAE,EAAE,CAC9D,IAAI,CAAC;IACH,WAAW,EAAE,8GAA8G;IAC3H,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,MAAM,EAAE,CAAC;aACN,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;YACP,MAAM,EAAE,CAAC;iBACN,MAAM,EAAE;iBACR,QAAQ,CACP,oEAAoE,CACrE;YACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8BAA8B,CAAC;SAC3D,CAAC,CACH;aACA,GAAG,CAAC,CAAC,EAAE,oCAAoC,CAAC;KAChD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;QAC5B,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,8BAA8B;YACvC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;oBAC7B,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QACH,MAAM,WAAW,GAAG,yDAAyD,MAAM;aAChF,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;aACpB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAEhB,MAAM,cAAc,GAAG,cAAc;YACnC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE;YAC3B,CAAC,CAAC,SAAS,CAAC;QACd,MAAM,cAAc,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;QAErE,MAAM,SAAS,GAAG,EAAe,CAAC;QAClC,MAAM,iBAAiB,GAAa,EAAE,CAAC;QACvC,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;YACjC,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACpC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;gBACrC,iBAAiB,CAAC,IAAI,CAAC,GAAI,SAAS,CAAC,OAAoB,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QACD,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,UAAU;YAChB,MAAM;YACN,cAAc;YACd,OAAO,EAAE,iBAAiB;SAC3B,CAAC,CAAC;QACH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,SAAS;YAClB,mBAAmB,EAAE,iBAAiB;SACvC,CAAC;IACJ,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\n\nexport const fillFormTool = (v3: V3, executionModel?: string) =>\n  tool({\n    description: `📝 FORM FILL - MULTI-FIELD INPUT TOOL\\nFor any form with 2+ inputs/textareas. Faster than individual typing.`,\n    inputSchema: z.object({\n      fields: z\n        .array(\n          z.object({\n            action: z\n              .string()\n              .describe(\n                'Description of typing action, e.g. \"type foo into the email field\"',\n              ),\n            value: z.string().describe(\"Text to type into the target\"),\n          }),\n        )\n        .min(1, \"Provide at least one field to fill\"),\n    }),\n    execute: async ({ fields }) => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: fillForm`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: JSON.stringify(fields),\n            type: \"object\",\n          },\n        },\n      });\n      const instruction = `Return observation results for the following actions: ${fields\n        .map((f) => f.action)\n        .join(\", \")}`;\n\n      const observeOptions = executionModel\n        ? { model: executionModel }\n        : undefined;\n      const observeResults = await v3.observe(instruction, observeOptions);\n\n      const completed = [] as unknown[];\n      const replayableActions: Action[] = [];\n      for (const res of observeResults) {\n        const actResult = await v3.act(res);\n        completed.push(actResult);\n        if (Array.isArray(actResult.actions)) {\n          replayableActions.push(...(actResult.actions as Action[]));\n        }\n      }\n      v3.recordAgentReplayStep({\n        type: \"fillForm\",\n        fields,\n        observeResults,\n        actions: replayableActions,\n      });\n      return {\n        success: true,\n        actions: completed,\n        playwrightArguments: replayableActions,\n      };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/goto.d.ts

@@ -0,0 +1,12 @@
+import type { V3 } from "../../v3.js";
+export declare const gotoTool: (v3: V3) => import("ai").Tool<{
+    url: string;
+}, {
+    success: boolean;
+    url: string;
+    error?: undefined;
+} | {
+    success: boolean;
+    error: any;
+    url?: undefined;
+}>;

package/dist/esm/lib/v3/agent/tools/goto.js

@@ -0,0 +1,31 @@
+import { tool } from "ai";
+import { z } from "zod";
+export const gotoTool = (v3) => tool({
+    description: "Navigate to a specific URL",
+    inputSchema: z.object({
+        url: z.string().describe("The URL to navigate to"),
+    }),
+    execute: async ({ url }) => {
+        try {
+            v3.logger({
+                category: "agent",
+                message: `Agent calling tool: goto`,
+                level: 1,
+                auxiliary: {
+                    arguments: {
+                        value: url,
+                        type: "string",
+                    },
+                },
+            });
+            const page = await v3.context.awaitActivePage();
+            await page.goto(url, { waitUntil: "load" });
+            v3.recordAgentReplayStep({ type: "goto", url, waitUntil: "load" });
+            return { success: true, url };
+        }
+        catch (error) {
+            return { success: false, error: error?.message ?? String(error) };
+        }
+    },
+});
+//# sourceMappingURL=goto.js.map