@browserbasehq/orca 3.0.9-alpha-3 → 3.1.0-headers

package/dist/esm/lib/v3/agent/tools/click.js

@@ -0,0 +1,96 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { processCoordinates } from "../utils/coordinateNormalization.js";
+import { ensureXPath } from "../utils/xpath.js";
+import { waitAndCaptureScreenshot } from "../utils/screenshotHandler.js";
+export const clickTool = (v3, provider) => tool({
+    description: "Click on an element using its coordinates (this is the most reliable way to click on an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)",
+    inputSchema: z.object({
+        describe: z
+            .string()
+            .describe("Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description"),
+        coordinates: z
+            .array(z.number())
+            .describe("The (x, y) coordinates to click on"),
+    }),
+    execute: async ({ describe, coordinates }) => {
+        try {
+            const page = await v3.context.awaitActivePage();
+            const processed = processCoordinates(coordinates[0], coordinates[1], provider, v3);
+            v3.logger({
+                category: "agent",
+                message: `Agent calling tool: click`,
+                level: 1,
+                auxiliary: {
+                    arguments: {
+                        value: JSON.stringify({ describe }),
+                        type: "object",
+                    },
+                },
+            });
+            // Only request XPath when caching is enabled to avoid unnecessary computation
+            const shouldCollectXpath = v3.isAgentReplayActive();
+            const xpath = await page.click(processed.x, processed.y, {
+                returnXpath: shouldCollectXpath,
+            });
+            const screenshotBase64 = await waitAndCaptureScreenshot(page);
+            // Record as an "act" step with proper Action for deterministic replay (only when caching)
+            if (shouldCollectXpath) {
+                const normalizedXpath = ensureXPath(xpath);
+                if (normalizedXpath) {
+                    const action = {
+                        selector: normalizedXpath,
+                        description: describe,
+                        method: "click",
+                        arguments: [],
+                    };
+                    v3.recordAgentReplayStep({
+                        type: "act",
+                        instruction: describe,
+                        actions: [action],
+                        actionDescription: describe,
+                    });
+                }
+            }
+            return {
+                success: true,
+                describe,
+                coordinates: [processed.x, processed.y],
+                screenshotBase64,
+            };
+        }
+        catch (error) {
+            return {
+                success: false,
+                error: `Error clicking: ${error.message}`,
+            };
+        }
+    },
+    toModelOutput: (result) => {
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
+        }
+        const content = [
+            {
+                type: "text",
+                text: JSON.stringify({
+                    success: result.success,
+                    describe: result.describe,
+                    coordinates: result.coordinates,
+                }),
+            },
+        ];
+        if (result.screenshotBase64) {
+            content.push({
+                type: "media",
+                mediaType: "image/png",
+                data: result.screenshotBase64,
+            });
+        }
+        return { type: "content", value: content };
+    },
+});
+//# sourceMappingURL=click.js.map

package/dist/esm/lib/v3/agent/tools/click.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"click.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/click.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CACrD,IAAI,CAAC;IACH,WAAW,EACT,wMAAwM;IAC1M,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAA4B,EAAE;QACrE,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,2BAA2B;gBACpC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;wBACnC,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;gBACvD,WAAW,EAAE,kBAAkB;aAChC,CAAC,CAAC;YAEH,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE9D,0FAA0F;YAC1F,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,OAAO;wBACf,SAAS,EAAE,EAAE;qBACd,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;gBACvC,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAoB,KAAe,CAAC,OAAO,EAAE;aACrD,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,WAAW,EAAE,MAAM,CAAC,WAAW;iBAChC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  ClickToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const clickTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Click on an element using its coordinates (this is the most reliable way to click on an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to click on\"),\n    }),\n    execute: async ({ describe, coordinates }): Promise<ClickToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: click`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const xpath = await page.click(processed.x, processed.y, {\n          returnXpath: shouldCollectXpath,\n        });\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"click\",\n              arguments: [],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          coordinates: [processed.x, processed.y],\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error clicking: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n            coordinates: result.coordinates,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/clickAndHold.d.ts

@@ -0,0 +1,14 @@
+import type { V3 } from "../../v3.js";
+export declare const clickAndHoldTool: (v3: V3, provider?: string) => import("ai").Tool<{
+    describe: string;
+    duration: number;
+    coordinates: number[];
+}, {
+    success: boolean;
+    describe: string;
+    error?: undefined;
+} | {
+    success: boolean;
+    error: string;
+    describe?: undefined;
+}>;

package/dist/esm/lib/v3/agent/tools/clickAndHold.js

@@ -0,0 +1,68 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { processCoordinates } from "../utils/coordinateNormalization.js";
+import { ensureXPath } from "../utils/xpath.js";
+export const clickAndHoldTool = (v3, provider) => tool({
+    description: "Click and hold on an element using its coordinates",
+    inputSchema: z.object({
+        describe: z
+            .string()
+            .describe("Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description"),
+        duration: z
+            .number()
+            .describe("The duration to hold the element in milliseconds"),
+        coordinates: z
+            .array(z.number())
+            .describe("The (x, y) coordinates to click on"),
+    }),
+    execute: async ({ describe, coordinates, duration }) => {
+        try {
+            const page = await v3.context.awaitActivePage();
+            const processed = processCoordinates(coordinates[0], coordinates[1], provider, v3);
+            v3.logger({
+                category: "agent",
+                message: `Agent calling tool: clickAndHold`,
+                level: 1,
+                auxiliary: {
+                    arguments: {
+                        value: JSON.stringify({
+                            describe,
+                            duration,
+                        }),
+                        type: "object",
+                    },
+                },
+            });
+            // Only request XPath when caching is enabled to avoid unnecessary computation
+            const shouldCollectXpath = v3.isAgentReplayActive();
+            // Use dragAndDrop from same point to same point with delay to simulate click and hold
+            const [xpath] = await page.dragAndDrop(processed.x, processed.y, processed.x, processed.y, { delay: duration, returnXpath: shouldCollectXpath });
+            // Record as "act" step with proper Action for deterministic replay (only when caching)
+            if (shouldCollectXpath) {
+                const normalizedXpath = ensureXPath(xpath);
+                if (normalizedXpath) {
+                    const action = {
+                        selector: normalizedXpath,
+                        description: describe,
+                        method: "clickAndHold",
+                        arguments: [String(duration)],
+                    };
+                    v3.recordAgentReplayStep({
+                        type: "act",
+                        instruction: describe,
+                        actions: [action],
+                        actionDescription: describe,
+                    });
+                }
+            }
+            return { success: true, describe };
+        }
+        catch (error) {
+            return {
+                success: false,
+                error: `Error clicking and holding: ${error.message}`,
+            };
+        }
+    },
+});
+//# sourceMappingURL=clickAndHold.js.map

package/dist/esm/lib/v3/agent/tools/clickAndHold.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"clickAndHold.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/clickAndHold.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC5D,IAAI,CAAC;IACH,WAAW,EAAE,oDAAoD;IACjE,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CAAC,kDAAkD,CAAC;QAC/D,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,EAAE,EAAE;QACrD,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,kCAAkC;gBAC3C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;4BACR,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YAEpD,sFAAsF;YACtF,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACpC,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACrD,CAAC;YAEF,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,cAAc;wBACtB,SAAS,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;qBAC9B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QACrC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,+BAAgC,KAAe,CAAC,OAAO,EAAE;aACjE,CAAC;QACJ,CAAC;IACH,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\n\nexport const clickAndHoldTool = (v3: V3, provider?: string) =>\n  tool({\n    description: \"Click and hold on an element using its coordinates\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      duration: z\n        .number()\n        .describe(\"The duration to hold the element in milliseconds\"),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to click on\"),\n    }),\n    execute: async ({ describe, coordinates, duration }) => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: clickAndHold`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({\n                describe,\n                duration,\n              }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n\n        // Use dragAndDrop from same point to same point with delay to simulate click and hold\n        const [xpath] = await page.dragAndDrop(\n          processed.x,\n          processed.y,\n          processed.x,\n          processed.y,\n          { delay: duration, returnXpath: shouldCollectXpath },\n        );\n\n        // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"clickAndHold\",\n              arguments: [String(duration)],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return { success: true, describe };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error clicking and holding: ${(error as Error).message}`,\n        };\n      }\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/dragAndDrop.d.ts

@@ -0,0 +1,7 @@
+import type { V3 } from "../../v3.js";
+import type { DragAndDropToolResult } from "../../types/public/agent.js";
+export declare const dragAndDropTool: (v3: V3, provider?: string) => import("ai").Tool<{
+    describe: string;
+    startCoordinates: number[];
+    endCoordinates: number[];
+}, DragAndDropToolResult>;

package/dist/esm/lib/v3/agent/tools/dragAndDrop.js

@@ -0,0 +1,97 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { processCoordinates } from "../utils/coordinateNormalization.js";
+import { ensureXPath } from "../utils/xpath.js";
+import { waitAndCaptureScreenshot } from "../utils/screenshotHandler.js";
+export const dragAndDropTool = (v3, provider) => tool({
+    description: "Drag and drop an element using its coordinates (this is the most reliable way to drag and drop an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)",
+    inputSchema: z.object({
+        describe: z.string().describe("Describe the element to drag and drop"),
+        startCoordinates: z
+            .array(z.number())
+            .describe("The (x, y) coordinates to start the drag and drop from"),
+        endCoordinates: z
+            .array(z.number())
+            .describe("The (x, y) coordinates to end the drag and drop at"),
+    }),
+    execute: async ({ describe, startCoordinates, endCoordinates, }) => {
+        try {
+            const page = await v3.context.awaitActivePage();
+            const processedStart = processCoordinates(startCoordinates[0], startCoordinates[1], provider, v3);
+            const processedEnd = processCoordinates(endCoordinates[0], endCoordinates[1], provider, v3);
+            v3.logger({
+                category: "agent",
+                message: `Agent calling tool: dragAndDrop`,
+                level: 1,
+                auxiliary: {
+                    arguments: {
+                        value: JSON.stringify({
+                            describe,
+                        }),
+                        type: "object",
+                    },
+                },
+            });
+            // Only request XPath when caching is enabled to avoid unnecessary computation
+            const shouldCollectXpath = v3.isAgentReplayActive();
+            const [fromXpath, toXpath] = await page.dragAndDrop(processedStart.x, processedStart.y, processedEnd.x, processedEnd.y, { returnXpath: shouldCollectXpath });
+            const screenshotBase64 = await waitAndCaptureScreenshot(page);
+            // Record as "act" step with proper Action for deterministic replay (only when caching)
+            if (shouldCollectXpath) {
+                const normalizedFrom = ensureXPath(fromXpath);
+                const normalizedTo = ensureXPath(toXpath);
+                if (normalizedFrom && normalizedTo) {
+                    const action = {
+                        selector: normalizedFrom,
+                        description: describe,
+                        method: "dragAndDrop",
+                        arguments: [normalizedTo],
+                    };
+                    v3.recordAgentReplayStep({
+                        type: "act",
+                        instruction: describe,
+                        actions: [action],
+                        actionDescription: describe,
+                    });
+                }
+            }
+            return {
+                success: true,
+                describe,
+                screenshotBase64,
+            };
+        }
+        catch (error) {
+            return {
+                success: false,
+                error: `Error dragging: ${error.message}`,
+            };
+        }
+    },
+    toModelOutput: (result) => {
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
+        }
+        const content = [
+            {
+                type: "text",
+                text: JSON.stringify({
+                    success: result.success,
+                    describe: result.describe,
+                }),
+            },
+        ];
+        if (result.screenshotBase64) {
+            content.push({
+                type: "media",
+                mediaType: "image/png",
+                data: result.screenshotBase64,
+            });
+        }
+        return { type: "content", value: content };
+    },
+});
+//# sourceMappingURL=dragAndDrop.js.map

package/dist/esm/lib/v3/agent/tools/dragAndDrop.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dragAndDrop.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/dragAndDrop.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC3D,IAAI,CAAC;IACH,WAAW,EACT,kNAAkN;IACpN,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uCAAuC,CAAC;QACtE,gBAAgB,EAAE,CAAC;aAChB,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,wDAAwD,CAAC;QACrE,cAAc,EAAE,CAAC;aACd,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oDAAoD,CAAC;KAClE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,gBAAgB,EAChB,cAAc,GACf,EAAkC,EAAE;QACnC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,cAAc,GAAG,kBAAkB,CACvC,gBAAgB,CAAC,CAAC,CAAC,EACnB,gBAAgB,CAAC,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;YACF,MAAM,YAAY,GAAG,kBAAkB,CACrC,cAAc,CAAC,CAAC,CAAC,EACjB,cAAc,CAAC,CAAC,CAAC,EACjB,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,iCAAiC;gBAC1C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACjD,cAAc,CAAC,CAAC,EAChB,cAAc,CAAC,CAAC,EAChB,YAAY,CAAC,CAAC,EACd,YAAY,CAAC,CAAC,EACd,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACpC,CAAC;YAEF,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE9D,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,cAAc,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;gBAC9C,MAAM,YAAY,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;gBAC1C,IAAI,cAAc,IAAI,YAAY,EAAE,CAAC;oBACnC,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,cAAc;wBACxB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,aAAa;wBACrB,SAAS,EAAE,CAAC,YAAY,CAAC;qBAC1B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAoB,KAAe,CAAC,OAAO,EAAE;aACrD,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;iBAC1B,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  DragAndDropToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const dragAndDropTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Drag and drop an element using its coordinates (this is the most reliable way to drag and drop an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z.string().describe(\"Describe the element to drag and drop\"),\n      startCoordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to start the drag and drop from\"),\n      endCoordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to end the drag and drop at\"),\n    }),\n    execute: async ({\n      describe,\n      startCoordinates,\n      endCoordinates,\n    }): Promise<DragAndDropToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processedStart = processCoordinates(\n          startCoordinates[0],\n          startCoordinates[1],\n          provider,\n          v3,\n        );\n        const processedEnd = processCoordinates(\n          endCoordinates[0],\n          endCoordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: dragAndDrop`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({\n                describe,\n              }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const [fromXpath, toXpath] = await page.dragAndDrop(\n          processedStart.x,\n          processedStart.y,\n          processedEnd.x,\n          processedEnd.y,\n          { returnXpath: shouldCollectXpath },\n        );\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedFrom = ensureXPath(fromXpath);\n          const normalizedTo = ensureXPath(toXpath);\n          if (normalizedFrom && normalizedTo) {\n            const action: Action = {\n              selector: normalizedFrom,\n              description: describe,\n              method: \"dragAndDrop\",\n              arguments: [normalizedTo],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error dragging: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/extract.d.ts

@@ -0,0 +1,21 @@
+import type { V3 } from "../../v3.js";
+import type { AgentModelConfig } from "../../types/public/agent.js";
+export declare const extractTool: (v3: V3, executionModel?: string | AgentModelConfig, toolTimeout?: number) => import("ai").Tool<{
+    instruction: string;
+    schema?: {
+        [x: string]: unknown;
+        type?: string;
+        properties?: Record<string, unknown>;
+        items?: unknown;
+        enum?: string[];
+        format?: "email" | "uuid" | "url";
+    };
+}, {
+    success: boolean;
+    result: unknown;
+    error?: undefined;
+} | {
+    success: boolean;
+    error: any;
+    result?: undefined;
+}>;

package/dist/esm/lib/v3/agent/tools/extract.js

@@ -0,0 +1,106 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { TimeoutError } from "../../types/public/sdkErrors.js";
+function jsonSchemaToZod(schema) {
+    switch (schema.type) {
+        case "object": {
+            const shape = {};
+            if (schema.properties) {
+                for (const [key, value] of Object.entries(schema.properties)) {
+                    shape[key] = jsonSchemaToZod(value);
+                }
+            }
+            return z.object(shape);
+        }
+        case "array":
+            return z.array(schema.items ? jsonSchemaToZod(schema.items) : z.any());
+        case "string": {
+            let s = z.string();
+            if (schema.format === "url")
+                s = s.url();
+            if (schema.format === "email")
+                s = s.email();
+            if (schema.format === "uuid")
+                s = s.uuid();
+            if (schema.enum && schema.enum.length > 0)
+                return z.enum(schema.enum);
+            return s;
+        }
+        case "number":
+        case "integer":
+            return z.number();
+        case "boolean":
+            return z.boolean();
+        case "null":
+            return z.null();
+        default:
+            return z.any();
+    }
+}
+export const extractTool = (v3, executionModel, toolTimeout) => tool({
+    description: `Extract structured data from the current page based on a provided schema.
+    
+    USAGE GUIDELINES:
+    - Keep schemas MINIMAL - only include fields essential for the task
+    - IMPORTANT: only use this if explicitly asked for structured output. In most scenarios, you should use the aria tree tool over this.
+    - For URL fields, use format: "url"
+    
+    EXAMPLES:
+    1. Extract a single value:
+       instruction: "extract the product price"
+       schema: { type: "object", properties: { price: { type: "number" } } }
+    
+    2. Extract multiple fields:
+       instruction: "extract product name and price"
+       schema: { type: "object", properties: { name: { type: "string" }, price: { type: "number" } } }
+    
+    3. Extract arrays:
+       instruction: "extract all product names and prices"
+       schema: { type: "object", properties: { products: { type: "array", items: { type: "object", properties: { name: { type: "string" }, price: { type: "number" } } } } } }
+    
+    4. Extract a URL:
+       instruction: "extract the link"
+       schema: { type: "object", properties: { url: { type: "string", format: "url" } } }`,
+    inputSchema: z.object({
+        instruction: z.string(),
+        schema: z
+            .object({
+            type: z.string().optional(),
+            properties: z.record(z.string(), z.unknown()).optional(),
+            items: z.unknown().optional(),
+            enum: z.array(z.string()).optional(),
+            format: z.enum(["url", "email", "uuid"]).optional(),
+        })
+            .passthrough()
+            .optional()
+            .describe("JSON Schema object describing the structure to extract"),
+    }),
+    execute: async ({ instruction, schema }) => {
+        try {
+            const parsedSchema = schema
+                ? jsonSchemaToZod(schema)
+                : undefined;
+            const result = await v3.extract(instruction, parsedSchema, {
+                ...(executionModel ? { model: executionModel } : {}),
+                timeout: toolTimeout,
+            });
+            return { success: true, result };
+        }
+        catch (error) {
+            if (error instanceof TimeoutError) {
+                const timeoutMessage = `TimeoutError: extract() timed out — try using a smaller or simpler schema`;
+                v3.logger({
+                    category: "agent",
+                    message: timeoutMessage,
+                    level: 0,
+                });
+                return {
+                    success: false,
+                    error: timeoutMessage,
+                };
+            }
+            return { success: false, error: error?.message ?? String(error) };
+        }
+    },
+});
+//# sourceMappingURL=extract.js.map

package/dist/esm/lib/v3/agent/tools/extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAc,MAAM,KAAK,CAAC;AAGpC,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAU/D,SAAS,eAAe,CAAC,MAAkB;IACzC,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,KAAK,GAA+B,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;gBACtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC7D,KAAK,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;gBACtC,CAAC;YACH,CAAC;YACD,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;QACD,KAAK,OAAO;YACV,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QACzE,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;YACnB,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK;gBAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO;gBAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM;gBAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAC3C,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;gBACvC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAA6B,CAAC,CAAC;YACtD,OAAO,CAAC,CAAC;QACX,CAAC;QACD,KAAK,QAAQ,CAAC;QACd,KAAK,SAAS;YACZ,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;QACpB,KAAK,SAAS;YACZ,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;QACrB,KAAK,MAAM;YACT,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAClB;YACE,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,EAAM,EACN,cAA0C,EAC1C,WAAoB,EACpB,EAAE,CACF,IAAI,CAAC;IACH,WAAW,EAAE;;;;;;;;;;;;;;;;;;;;;;0FAsByE;IACtF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;QACvB,MAAM,EAAE,CAAC;aACN,MAAM,CAAC;YACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC3B,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;YACxD,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;YAC7B,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;YACpC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;SACpD,CAAC;aACD,WAAW,EAAE;aACb,QAAQ,EAAE;aACV,QAAQ,CAAC,wDAAwD,CAAC;KACtE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE;QACzC,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,MAAM;gBACzB,CAAC,CAAC,eAAe,CAAC,MAAoB,CAAC;gBACvC,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,YAAY,EAAE;gBACzD,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpD,OAAO,EAAE,WAAW;aACrB,CAAC,CAAC;YACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBAClC,MAAM,cAAc,GAAG,2EAA2E,CAAC;gBACnG,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,cAAc;oBACvB,KAAK,EAAE,CAAC;iBACT,CAAC,CAAC;gBACH,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,cAAc;iBACtB,CAAC;YACJ,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QACpE,CAAC;IACH,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z, ZodTypeAny } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { AgentModelConfig } from \"../../types/public/agent.js\";\nimport { TimeoutError } from \"../../types/public/sdkErrors.js\";\n\ninterface JsonSchema {\n  type?: string;\n  properties?: Record<string, JsonSchema>;\n  items?: JsonSchema;\n  enum?: string[];\n  format?: \"url\" | \"email\" | \"uuid\";\n}\n\nfunction jsonSchemaToZod(schema: JsonSchema): ZodTypeAny {\n  switch (schema.type) {\n    case \"object\": {\n      const shape: Record<string, ZodTypeAny> = {};\n      if (schema.properties) {\n        for (const [key, value] of Object.entries(schema.properties)) {\n          shape[key] = jsonSchemaToZod(value);\n        }\n      }\n      return z.object(shape);\n    }\n    case \"array\":\n      return z.array(schema.items ? jsonSchemaToZod(schema.items) : z.any());\n    case \"string\": {\n      let s = z.string();\n      if (schema.format === \"url\") s = s.url();\n      if (schema.format === \"email\") s = s.email();\n      if (schema.format === \"uuid\") s = s.uuid();\n      if (schema.enum && schema.enum.length > 0)\n        return z.enum(schema.enum as [string, ...string[]]);\n      return s;\n    }\n    case \"number\":\n    case \"integer\":\n      return z.number();\n    case \"boolean\":\n      return z.boolean();\n    case \"null\":\n      return z.null();\n    default:\n      return z.any();\n  }\n}\n\nexport const extractTool = (\n  v3: V3,\n  executionModel?: string | AgentModelConfig,\n  toolTimeout?: number,\n) =>\n  tool({\n    description: `Extract structured data from the current page based on a provided schema.\n    \n    USAGE GUIDELINES:\n    - Keep schemas MINIMAL - only include fields essential for the task\n    - IMPORTANT: only use this if explicitly asked for structured output. In most scenarios, you should use the aria tree tool over this.\n    - For URL fields, use format: \"url\"\n    \n    EXAMPLES:\n    1. Extract a single value:\n       instruction: \"extract the product price\"\n       schema: { type: \"object\", properties: { price: { type: \"number\" } } }\n    \n    2. Extract multiple fields:\n       instruction: \"extract product name and price\"\n       schema: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } }\n    \n    3. Extract arrays:\n       instruction: \"extract all product names and prices\"\n       schema: { type: \"object\", properties: { products: { type: \"array\", items: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } } } } }\n    \n    4. Extract a URL:\n       instruction: \"extract the link\"\n       schema: { type: \"object\", properties: { url: { type: \"string\", format: \"url\" } } }`,\n    inputSchema: z.object({\n      instruction: z.string(),\n      schema: z\n        .object({\n          type: z.string().optional(),\n          properties: z.record(z.string(), z.unknown()).optional(),\n          items: z.unknown().optional(),\n          enum: z.array(z.string()).optional(),\n          format: z.enum([\"url\", \"email\", \"uuid\"]).optional(),\n        })\n        .passthrough()\n        .optional()\n        .describe(\"JSON Schema object describing the structure to extract\"),\n    }),\n    execute: async ({ instruction, schema }) => {\n      try {\n        const parsedSchema = schema\n          ? jsonSchemaToZod(schema as JsonSchema)\n          : undefined;\n        const result = await v3.extract(instruction, parsedSchema, {\n          ...(executionModel ? { model: executionModel } : {}),\n          timeout: toolTimeout,\n        });\n        return { success: true, result };\n      } catch (error) {\n        if (error instanceof TimeoutError) {\n          const timeoutMessage = `TimeoutError: extract() timed out — try using a smaller or simpler schema`;\n          v3.logger({\n            category: \"agent\",\n            message: timeoutMessage,\n            level: 0,\n          });\n          return {\n            success: false,\n            error: timeoutMessage,\n          };\n        }\n        return { success: false, error: error?.message ?? String(error) };\n      }\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/fillFormVision.d.ts

@@ -0,0 +1,12 @@
+import type { V3 } from "../../v3.js";
+import type { FillFormVisionToolResult, Variables } from "../../types/public/agent.js";
+export declare const fillFormVisionTool: (v3: V3, provider?: string, variables?: Variables) => import("ai").Tool<{
+    fields: {
+        action: string;
+        value: string;
+        coordinates: {
+            x: number;
+            y: number;
+        };
+    }[];
+}, FillFormVisionToolResult>;

package/dist/esm/lib/v3/agent/tools/fillFormVision.js

@@ -0,0 +1,154 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { processCoordinates } from "../utils/coordinateNormalization.js";
+import { ensureXPath } from "../utils/xpath.js";
+import { waitAndCaptureScreenshot } from "../utils/screenshotHandler.js";
+import { substituteVariables } from "../utils/variables.js";
+export const fillFormVisionTool = (v3, provider, variables) => {
+    const hasVariables = variables && Object.keys(variables).length > 0;
+    const valueDescription = hasVariables
+        ? `Text to type into the target field. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(", ")}`
+        : "Text to type into the target field";
+    return tool({
+        description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL
+
+CRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)
+IMPORTANT: Ensure the fields are visible within the current viewport
+
+WHY THIS TOOL EXISTS:
+- Forms are the #1 use case for multi-field input
+- Optimized specifically for input/textarea elements
+- 4-6x faster than individual typing actions
+
+Use fillFormVision: Pure form filling (inputs, textareas only)
+MANDATORY USE CASES (always use fillFormVision for these):
+- Registration forms: name, email, password fields
+- Contact forms: name, email, message fields
+- Checkout forms: address, payment info fields
+- Profile updates: multiple user data fields
+- Search filters: multiple criteria inputs`,
+        inputSchema: z.object({
+            fields: z
+                .array(z.object({
+                action: z
+                    .string()
+                    .describe("Description of the typing action, e.g. 'type foo into the bar field'"),
+                value: z.string().describe(valueDescription),
+                coordinates: z
+                    .object({
+                    x: z.number(),
+                    y: z.number(),
+                })
+                    .describe("Coordinates of the target field"),
+            }))
+                .min(2, "Provide at least two fields to fill"),
+        }),
+        execute: async ({ fields }) => {
+            try {
+                const page = await v3.context.awaitActivePage();
+                // Process coordinates and substitute variables for each field
+                // Keep original values (with %tokens%) for logging/caching, substituted values for typing
+                const processedFields = fields.map((field) => {
+                    const processed = processCoordinates(field.coordinates.x, field.coordinates.y, provider, v3);
+                    return {
+                        ...field,
+                        originalValue: field.value, // Keep original with %tokens% for cache
+                        value: substituteVariables(field.value, variables),
+                        coordinates: { x: processed.x, y: processed.y },
+                    };
+                });
+                v3.logger({
+                    category: "agent",
+                    message: `Agent calling tool: fillFormVision`,
+                    level: 1,
+                    auxiliary: {
+                        arguments: {
+                            value: JSON.stringify({ fields }), // Don't log substituted values
+                            type: "object",
+                        },
+                    },
+                });
+                // Only request XPath when caching is enabled to avoid unnecessary computation
+                const shouldCollectXpath = v3.isAgentReplayActive();
+                const actions = [];
+                for (const field of processedFields) {
+                    // Click the field, only requesting XPath when caching is enabled
+                    const xpath = await page.click(field.coordinates.x, field.coordinates.y, {
+                        returnXpath: shouldCollectXpath,
+                    });
+                    await page.type(field.value);
+                    // Build Action with XPath for deterministic replay (only when caching)
+                    // Use originalValue (with %tokens%) so cache stores references, not sensitive values
+                    if (shouldCollectXpath) {
+                        const normalizedXpath = ensureXPath(xpath);
+                        if (normalizedXpath) {
+                            actions.push({
+                                selector: normalizedXpath,
+                                description: field.action,
+                                method: "type",
+                                arguments: [field.originalValue],
+                            });
+                        }
+                    }
+                    // Small delay between fields
+                    await new Promise((resolve) => setTimeout(resolve, 100));
+                }
+                const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);
+                // Record as "act" step with proper Actions for deterministic replay (only when caching)
+                if (shouldCollectXpath && actions.length > 0) {
+                    v3.recordAgentReplayStep({
+                        type: "act",
+                        instruction: `Fill ${fields.length} form fields`,
+                        actions,
+                        actionDescription: `Fill ${fields.length} form fields`,
+                    });
+                }
+                return {
+                    success: true,
+                    playwrightArguments: processedFields,
+                    screenshotBase64,
+                };
+            }
+            catch (error) {
+                return {
+                    success: false,
+                    error: `Error filling form: ${error.message}`,
+                };
+            }
+        },
+        toModelOutput: (result) => {
+            if (result.success === false || result.error !== undefined) {
+                return {
+                    type: "content",
+                    value: [
+                        {
+                            type: "text",
+                            text: JSON.stringify({
+                                success: result.success,
+                                error: result.error,
+                            }),
+                        },
+                    ],
+                };
+            }
+            const content = [
+                {
+                    type: "text",
+                    text: JSON.stringify({
+                        success: result.success,
+                        fieldsCount: result.playwrightArguments?.length ?? 0,
+                    }),
+                },
+            ];
+            if (result.screenshotBase64) {
+                content.push({
+                    type: "media",
+                    mediaType: "image/png",
+                    data: result.screenshotBase64,
+                });
+            }
+            return { type: "content", value: content };
+        },
+    });
+};
+//# sourceMappingURL=fillFormVision.js.map