@browserbasehq/orca 3.1.0-patch.3 → 3.2.0-preview.1

package/dist/esm/lib/v3/agent/tools/ariaTree.js

@@ -1,31 +1,69 @@
 import { tool } from "ai";
 import { z } from "zod";
-export const ariaTreeTool = (v3) => tool({
+import { TimeoutError } from "../../types/public/sdkErrors.js";
+export const ariaTreeTool = (v3, toolTimeout) => tool({
     description: "gets the accessibility (ARIA) hybrid tree text for the current page. use this to understand structure and content.",
     inputSchema: z.object({}),
     execute: async () => {
-        v3.logger({
-            category: "agent",
-            message: `Agent calling tool: ariaTree`,
-            level: 1,
-        });
-        const page = await v3.context.awaitActivePage();
-        const { pageText } = (await v3.extract());
-        const pageUrl = page.url();
-        let content = pageText;
-        const MAX_TOKENS = 70000; // rough cap, assume ~4 chars per token for conservative truncation
-        const estimatedTokens = Math.ceil(content.length / 4);
-        if (estimatedTokens > MAX_TOKENS) {
-            const maxChars = MAX_TOKENS * 4;
-            content =
-                content.substring(0, maxChars) +
-                    "\n\n[CONTENT TRUNCATED: Exceeded 70,000 token limit]";
+        try {
+            v3.logger({
+                category: "agent",
+                message: `Agent calling tool: ariaTree`,
+                level: 1,
+            });
+            const page = await v3.context.awaitActivePage();
+            const extractOptions = toolTimeout
+                ? { timeout: toolTimeout }
+                : undefined;
+            const { pageText } = (await v3.extract(extractOptions));
+            const pageUrl = page.url();
+            let content = pageText;
+            const MAX_TOKENS = 70000; // rough cap, assume ~4 chars per token for conservative truncation
+            const estimatedTokens = Math.ceil(content.length / 4);
+            if (estimatedTokens > MAX_TOKENS) {
+                const maxChars = MAX_TOKENS * 4;
+                content =
+                    content.substring(0, maxChars) +
+                        "\n\n[CONTENT TRUNCATED: Exceeded 70,000 token limit]";
+            }
+            return { success: true, content, pageUrl };
         }
-        return { content, pageUrl };
+        catch (error) {
+            if (error instanceof TimeoutError) {
+                const timeoutMessage = `TimeoutError: ariaTree() timed out — the page may be too large`;
+                v3.logger({
+                    category: "agent",
+                    message: timeoutMessage,
+                    level: 0,
+                });
+                return {
+                    content: "",
+                    error: timeoutMessage,
+                    success: false,
+                    pageUrl: "",
+                };
+            }
+            return {
+                content: "",
+                error: error?.message ?? String(error),
+                success: false,
+                pageUrl: "",
+            };
+        }
+    },
+    toModelOutput: (result) => {
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
+        }
+        return {
+            type: "content",
+            value: [
+                { type: "text", text: `Accessibility Tree:\n${result.content}` },
+            ],
+        };
     },
-    toModelOutput: (result) => ({
-        type: "content",
-        value: [{ type: "text", text: `Accessibility Tree:\n${result.content}` }],
-    }),
 });
 //# sourceMappingURL=ariaTree.js.map

package/dist/esm/lib/v3/agent/tools/ariaTree.js.map

@@ -1 +1 @@
-{"version":3,"file":"ariaTree.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/ariaTree.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,EAAM,EAAE,EAAE,CACrC,IAAI,CAAC;IACH,WAAW,EACT,oHAAoH;IACtH,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;IACzB,OAAO,EAAE,KAAK,IAAI,EAAE;QAClB,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,8BAA8B;YACvC,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAChD,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,CAAyB,CAAC;QAClE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE3B,IAAI,OAAO,GAAG,QAAQ,CAAC;QACvB,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,mEAAmE;QAC7F,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACtD,IAAI,eAAe,GAAG,UAAU,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,UAAU,GAAG,CAAC,CAAC;YAChC,OAAO;gBACL,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,QAAQ,CAAC;oBAC9B,sDAAsD,CAAC;QAC3D,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IAC9B,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC1B,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,wBAAwB,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;KAC1E,CAAC;CACH,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport const ariaTreeTool = (v3: V3) =>\n  tool({\n    description:\n      \"gets the accessibility (ARIA) hybrid tree text for the current page. use this to understand structure and content.\",\n    inputSchema: z.object({}),\n    execute: async () => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: ariaTree`,\n        level: 1,\n      });\n      const page = await v3.context.awaitActivePage();\n      const { pageText } = (await v3.extract()) as { pageText: string };\n      const pageUrl = page.url();\n\n      let content = pageText;\n      const MAX_TOKENS = 70000; // rough cap, assume ~4 chars per token for conservative truncation\n      const estimatedTokens = Math.ceil(content.length / 4);\n      if (estimatedTokens > MAX_TOKENS) {\n        const maxChars = MAX_TOKENS * 4;\n        content =\n          content.substring(0, maxChars) +\n          \"\\n\\n[CONTENT TRUNCATED: Exceeded 70,000 token limit]\";\n      }\n\n      return { content, pageUrl };\n    },\n    toModelOutput: (result) => ({\n      type: \"content\",\n      value: [{ type: \"text\", text: `Accessibility Tree:\\n${result.content}` }],\n    }),\n  });\n"]}
+{"version":3,"file":"ariaTree.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/ariaTree.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAE/D,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,EAAM,EAAE,WAAoB,EAAE,EAAE,CAC3D,IAAI,CAAC;IACH,WAAW,EACT,oHAAoH;IACtH,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;IACzB,OAAO,EAAE,KAAK,IAAI,EAAE;QAClB,IAAI,CAAC;YACH,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,8BAA8B;gBACvC,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,cAAc,GAAG,WAAW;gBAChC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE;gBAC1B,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,CAErD,CAAC;YACF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAE3B,IAAI,OAAO,GAAG,QAAQ,CAAC;YACvB,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,mEAAmE;YAC7F,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACtD,IAAI,eAAe,GAAG,UAAU,EAAE,CAAC;gBACjC,MAAM,QAAQ,GAAG,UAAU,GAAG,CAAC,CAAC;gBAChC,OAAO;oBACL,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,QAAQ,CAAC;wBAC9B,sDAAsD,CAAC;YAC3D,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBAClC,MAAM,cAAc,GAAG,gEAAgE,CAAC;gBACxF,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,cAAc;oBACvB,KAAK,EAAE,CAAC;iBACT,CAAC,CAAC;gBACH,OAAO;oBACL,OAAO,EAAE,EAAE;oBACX,KAAK,EAAE,cAAc;oBACrB,OAAO,EAAE,KAAK;oBACd,OAAO,EAAE,EAAE;iBACZ,CAAC;YACJ,CAAC;YACD,OAAO;gBACL,OAAO,EAAE,EAAE;gBACX,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC;gBACtC,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,EAAE;aACZ,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE;gBACL,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,wBAAwB,MAAM,CAAC,OAAO,EAAE,EAAE;aACjE;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport { TimeoutError } from \"../../types/public/sdkErrors.js\";\n\nexport const ariaTreeTool = (v3: V3, toolTimeout?: number) =>\n  tool({\n    description:\n      \"gets the accessibility (ARIA) hybrid tree text for the current page. use this to understand structure and content.\",\n    inputSchema: z.object({}),\n    execute: async () => {\n      try {\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: ariaTree`,\n          level: 1,\n        });\n        const page = await v3.context.awaitActivePage();\n        const extractOptions = toolTimeout\n          ? { timeout: toolTimeout }\n          : undefined;\n        const { pageText } = (await v3.extract(extractOptions)) as {\n          pageText: string;\n        };\n        const pageUrl = page.url();\n\n        let content = pageText;\n        const MAX_TOKENS = 70000; // rough cap, assume ~4 chars per token for conservative truncation\n        const estimatedTokens = Math.ceil(content.length / 4);\n        if (estimatedTokens > MAX_TOKENS) {\n          const maxChars = MAX_TOKENS * 4;\n          content =\n            content.substring(0, maxChars) +\n            \"\\n\\n[CONTENT TRUNCATED: Exceeded 70,000 token limit]\";\n        }\n\n        return { success: true, content, pageUrl };\n      } catch (error) {\n        if (error instanceof TimeoutError) {\n          const timeoutMessage = `TimeoutError: ariaTree() timed out — the page may be too large`;\n          v3.logger({\n            category: \"agent\",\n            message: timeoutMessage,\n            level: 0,\n          });\n          return {\n            content: \"\",\n            error: timeoutMessage,\n            success: false,\n            pageUrl: \"\",\n          };\n        }\n        return {\n          content: \"\",\n          error: error?.message ?? String(error),\n          success: false,\n          pageUrl: \"\",\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      return {\n        type: \"content\",\n        value: [\n          { type: \"text\", text: `Accessibility Tree:\\n${result.content}` },\n        ],\n      };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/click.js

@@ -67,38 +67,30 @@ export const clickTool = (v3, provider) => tool({
         }
     },
     toModelOutput: (result) => {
-        if (result.success) {
-            const content = [
-                {
-                    type: "text",
-                    text: JSON.stringify({
-                        success: result.success,
-                        describe: result.describe,
-                        coordinates: result.coordinates,
-                    }),
-                },
-            ];
-            if (result.screenshotBase64) {
-                content.push({
-                    type: "media",
-                    mediaType: "image/png",
-                    data: result.screenshotBase64,
-                });
-            }
-            return { type: "content", value: content };
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
         }
-        return {
-            type: "content",
-            value: [
-                {
-                    type: "text",
-                    text: JSON.stringify({
-                        success: result.success,
-                        error: result.error,
-                    }),
-                },
-            ],
-        };
+        const content = [
+            {
+                type: "text",
+                text: JSON.stringify({
+                    success: result.success,
+                    describe: result.describe,
+                    coordinates: result.coordinates,
+                }),
+            },
+        ];
+        if (result.screenshotBase64) {
+            content.push({
+                type: "media",
+                mediaType: "image/png",
+                data: result.screenshotBase64,
+            });
+        }
+        return { type: "content", value: content };
     },
 });
 //# sourceMappingURL=click.js.map

package/dist/esm/lib/v3/agent/tools/click.js.map

@@ -1 +1 @@
-{"version":3,"file":"click.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/click.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CACrD,IAAI,CAAC;IACH,WAAW,EACT,wMAAwM;IAC1M,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAA4B,EAAE;QACrE,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,2BAA2B;gBACpC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;wBACnC,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;gBACvD,WAAW,EAAE,kBAAkB;aAChC,CAAC,CAAC;YAEH,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE9D,0FAA0F;YAC1F,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,OAAO;wBACf,SAAS,EAAE,EAAE;qBACd,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;gBACvC,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAoB,KAAe,CAAC,OAAO,EAAE;aACrD,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,WAAW,EAAE,MAAM,CAAC,WAAW;qBAChC,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;QACD,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,KAAK,EAAE,MAAM,CAAC,KAAK;qBACpB,CAAC;iBACH;aACF;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  ClickToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const clickTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Click on an element using its coordinates (this is the most reliable way to click on an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to click on\"),\n    }),\n    execute: async ({ describe, coordinates }): Promise<ClickToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: click`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const xpath = await page.click(processed.x, processed.y, {\n          returnXpath: shouldCollectXpath,\n        });\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"click\",\n              arguments: [],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          coordinates: [processed.x, processed.y],\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error clicking: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success) {\n        const content: ModelOutputContentItem[] = [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              describe: result.describe,\n              coordinates: result.coordinates,\n            }),\n          },\n        ];\n        if (result.screenshotBase64) {\n          content.push({\n            type: \"media\",\n            mediaType: \"image/png\",\n            data: result.screenshotBase64,\n          });\n        }\n        return { type: \"content\", value: content };\n      }\n      return {\n        type: \"content\",\n        value: [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              error: result.error,\n            }),\n          },\n        ],\n      };\n    },\n  });\n"]}
+{"version":3,"file":"click.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/click.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CACrD,IAAI,CAAC;IACH,WAAW,EACT,wMAAwM;IAC1M,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAA4B,EAAE;QACrE,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,2BAA2B;gBACpC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;wBACnC,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;gBACvD,WAAW,EAAE,kBAAkB;aAChC,CAAC,CAAC;YAEH,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE9D,0FAA0F;YAC1F,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,OAAO;wBACf,SAAS,EAAE,EAAE;qBACd,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;gBACvC,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAoB,KAAe,CAAC,OAAO,EAAE;aACrD,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,WAAW,EAAE,MAAM,CAAC,WAAW;iBAChC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  ClickToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const clickTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Click on an element using its coordinates (this is the most reliable way to click on an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to click on\"),\n    }),\n    execute: async ({ describe, coordinates }): Promise<ClickToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: click`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const xpath = await page.click(processed.x, processed.y, {\n          returnXpath: shouldCollectXpath,\n        });\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"click\",\n              arguments: [],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          coordinates: [processed.x, processed.y],\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error clicking: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n            coordinates: result.coordinates,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/dragAndDrop.js

@@ -69,37 +69,29 @@ export const dragAndDropTool = (v3, provider) => tool({
         }
     },
     toModelOutput: (result) => {
-        if (result.success) {
-            const content = [
-                {
-                    type: "text",
-                    text: JSON.stringify({
-                        success: result.success,
-                        describe: result.describe,
-                    }),
-                },
-            ];
-            if (result.screenshotBase64) {
-                content.push({
-                    type: "media",
-                    mediaType: "image/png",
-                    data: result.screenshotBase64,
-                });
-            }
-            return { type: "content", value: content };
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
         }
-        return {
-            type: "content",
-            value: [
-                {
-                    type: "text",
-                    text: JSON.stringify({
-                        success: result.success,
-                        error: result.error,
-                    }),
-                },
-            ],
-        };
+        const content = [
+            {
+                type: "text",
+                text: JSON.stringify({
+                    success: result.success,
+                    describe: result.describe,
+                }),
+            },
+        ];
+        if (result.screenshotBase64) {
+            content.push({
+                type: "media",
+                mediaType: "image/png",
+                data: result.screenshotBase64,
+            });
+        }
+        return { type: "content", value: content };
     },
 });
 //# sourceMappingURL=dragAndDrop.js.map

package/dist/esm/lib/v3/agent/tools/dragAndDrop.js.map

@@ -1 +1 @@
-{"version":3,"file":"dragAndDrop.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/dragAndDrop.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC3D,IAAI,CAAC;IACH,WAAW,EACT,kNAAkN;IACpN,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uCAAuC,CAAC;QACtE,gBAAgB,EAAE,CAAC;aAChB,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,wDAAwD,CAAC;QACrE,cAAc,EAAE,CAAC;aACd,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oDAAoD,CAAC;KAClE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,gBAAgB,EAChB,cAAc,GACf,EAAkC,EAAE;QACnC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,cAAc,GAAG,kBAAkB,CACvC,gBAAgB,CAAC,CAAC,CAAC,EACnB,gBAAgB,CAAC,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;YACF,MAAM,YAAY,GAAG,kBAAkB,CACrC,cAAc,CAAC,CAAC,CAAC,EACjB,cAAc,CAAC,CAAC,CAAC,EACjB,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,iCAAiC;gBAC1C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACjD,cAAc,CAAC,CAAC,EAChB,cAAc,CAAC,CAAC,EAChB,YAAY,CAAC,CAAC,EACd,YAAY,CAAC,CAAC,EACd,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACpC,CAAC;YAEF,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE9D,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,cAAc,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;gBAC9C,MAAM,YAAY,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;gBAC1C,IAAI,cAAc,IAAI,YAAY,EAAE,CAAC;oBACnC,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,cAAc;wBACxB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,aAAa;wBACrB,SAAS,EAAE,CAAC,YAAY,CAAC;qBAC1B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAoB,KAAe,CAAC,OAAO,EAAE;aACrD,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;qBAC1B,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;QACD,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,KAAK,EAAE,MAAM,CAAC,KAAK;qBACpB,CAAC;iBACH;aACF;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  DragAndDropToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const dragAndDropTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Drag and drop an element using its coordinates (this is the most reliable way to drag and drop an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z.string().describe(\"Describe the element to drag and drop\"),\n      startCoordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to start the drag and drop from\"),\n      endCoordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to end the drag and drop at\"),\n    }),\n    execute: async ({\n      describe,\n      startCoordinates,\n      endCoordinates,\n    }): Promise<DragAndDropToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processedStart = processCoordinates(\n          startCoordinates[0],\n          startCoordinates[1],\n          provider,\n          v3,\n        );\n        const processedEnd = processCoordinates(\n          endCoordinates[0],\n          endCoordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: dragAndDrop`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({\n                describe,\n              }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const [fromXpath, toXpath] = await page.dragAndDrop(\n          processedStart.x,\n          processedStart.y,\n          processedEnd.x,\n          processedEnd.y,\n          { returnXpath: shouldCollectXpath },\n        );\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedFrom = ensureXPath(fromXpath);\n          const normalizedTo = ensureXPath(toXpath);\n          if (normalizedFrom && normalizedTo) {\n            const action: Action = {\n              selector: normalizedFrom,\n              description: describe,\n              method: \"dragAndDrop\",\n              arguments: [normalizedTo],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error dragging: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success) {\n        const content: ModelOutputContentItem[] = [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              describe: result.describe,\n            }),\n          },\n        ];\n        if (result.screenshotBase64) {\n          content.push({\n            type: \"media\",\n            mediaType: \"image/png\",\n            data: result.screenshotBase64,\n          });\n        }\n        return { type: \"content\", value: content };\n      }\n      return {\n        type: \"content\",\n        value: [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              error: result.error,\n            }),\n          },\n        ],\n      };\n    },\n  });\n"]}
+{"version":3,"file":"dragAndDrop.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/dragAndDrop.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC3D,IAAI,CAAC;IACH,WAAW,EACT,kNAAkN;IACpN,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uCAAuC,CAAC;QACtE,gBAAgB,EAAE,CAAC;aAChB,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,wDAAwD,CAAC;QACrE,cAAc,EAAE,CAAC;aACd,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oDAAoD,CAAC;KAClE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,gBAAgB,EAChB,cAAc,GACf,EAAkC,EAAE;QACnC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,cAAc,GAAG,kBAAkB,CACvC,gBAAgB,CAAC,CAAC,CAAC,EACnB,gBAAgB,CAAC,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;YACF,MAAM,YAAY,GAAG,kBAAkB,CACrC,cAAc,CAAC,CAAC,CAAC,EACjB,cAAc,CAAC,CAAC,CAAC,EACjB,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,iCAAiC;gBAC1C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACjD,cAAc,CAAC,CAAC,EAChB,cAAc,CAAC,CAAC,EAChB,YAAY,CAAC,CAAC,EACd,YAAY,CAAC,CAAC,EACd,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACpC,CAAC;YAEF,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE9D,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,cAAc,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;gBAC9C,MAAM,YAAY,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;gBAC1C,IAAI,cAAc,IAAI,YAAY,EAAE,CAAC;oBACnC,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,cAAc;wBACxB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,aAAa;wBACrB,SAAS,EAAE,CAAC,YAAY,CAAC;qBAC1B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAoB,KAAe,CAAC,OAAO,EAAE;aACrD,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;iBAC1B,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  DragAndDropToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const dragAndDropTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Drag and drop an element using its coordinates (this is the most reliable way to drag and drop an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z.string().describe(\"Describe the element to drag and drop\"),\n      startCoordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to start the drag and drop from\"),\n      endCoordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to end the drag and drop at\"),\n    }),\n    execute: async ({\n      describe,\n      startCoordinates,\n      endCoordinates,\n    }): Promise<DragAndDropToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processedStart = processCoordinates(\n          startCoordinates[0],\n          startCoordinates[1],\n          provider,\n          v3,\n        );\n        const processedEnd = processCoordinates(\n          endCoordinates[0],\n          endCoordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: dragAndDrop`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({\n                describe,\n              }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const [fromXpath, toXpath] = await page.dragAndDrop(\n          processedStart.x,\n          processedStart.y,\n          processedEnd.x,\n          processedEnd.y,\n          { returnXpath: shouldCollectXpath },\n        );\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedFrom = ensureXPath(fromXpath);\n          const normalizedTo = ensureXPath(toXpath);\n          if (normalizedFrom && normalizedTo) {\n            const action: Action = {\n              selector: normalizedFrom,\n              description: describe,\n              method: \"dragAndDrop\",\n              arguments: [normalizedTo],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error dragging: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/extract.d.ts

@@ -1,6 +1,6 @@
 import type { V3 } from "../../v3.js";
 import type { AgentModelConfig } from "../../types/public/agent.js";
-export declare const extractTool: (v3: V3, executionModel?: string | AgentModelConfig) => import("ai").Tool<{
+export declare const extractTool: (v3: V3, executionModel?: string | AgentModelConfig, toolTimeout?: number) => import("ai").Tool<{
     instruction: string;
     schema?: {
         [x: string]: unknown;
@@ -16,6 +16,6 @@ export declare const extractTool: (v3: V3, executionModel?: string | AgentModelC
     error?: undefined;
 } | {
     success: boolean;
-    error: string;
+    error: any;
     result?: undefined;
 }>;

package/dist/esm/lib/v3/agent/tools/extract.js

@@ -1,5 +1,6 @@
 import { tool } from "ai";
 import { z } from "zod";
+import { TimeoutError } from "../../types/public/sdkErrors.js";
 function jsonSchemaToZod(schema) {
     switch (schema.type) {
         case "object": {
@@ -36,7 +37,7 @@ function jsonSchemaToZod(schema) {
             return z.any();
     }
 }
-export const extractTool = (v3, executionModel) => tool({
+export const extractTool = (v3, executionModel, toolTimeout) => tool({
     description: `Extract structured data from the current page based on a provided schema.
     
     USAGE GUIDELINES:
@@ -81,12 +82,24 @@ export const extractTool = (v3, executionModel) => tool({
                 : undefined;
             const result = await v3.extract(instruction, parsedSchema, {
                 ...(executionModel ? { model: executionModel } : {}),
+                timeout: toolTimeout,
             });
             return { success: true, result };
         }
         catch (error) {
-            const err = error;
-            return { success: false, error: err?.message ?? String(error) };
+            if (error instanceof TimeoutError) {
+                const timeoutMessage = `TimeoutError: extract() timed out — try using a smaller or simpler schema`;
+                v3.logger({
+                    category: "agent",
+                    message: timeoutMessage,
+                    level: 0,
+                });
+                return {
+                    success: false,
+                    error: timeoutMessage,
+                };
+            }
+            return { success: false, error: error?.message ?? String(error) };
         }
     },
 });

package/dist/esm/lib/v3/agent/tools/extract.js.map

@@ -1 +1 @@
-{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAc,MAAM,KAAK,CAAC;AAYpC,SAAS,eAAe,CAAC,MAAkB;IACzC,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,KAAK,GAA+B,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;gBACtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC7D,KAAK,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;gBACtC,CAAC;YACH,CAAC;YACD,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;QACD,KAAK,OAAO;YACV,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QACzE,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;YACnB,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK;gBAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO;gBAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM;gBAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAC3C,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;gBACvC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAA6B,CAAC,CAAC;YACtD,OAAO,CAAC,CAAC;QACX,CAAC;QACD,KAAK,QAAQ,CAAC;QACd,KAAK,SAAS;YACZ,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;QACpB,KAAK,SAAS;YACZ,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;QACrB,KAAK,MAAM;YACT,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAClB;YACE,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,EAAM,EACN,cAA0C,EAC1C,EAAE,CACF,IAAI,CAAC;IACH,WAAW,EAAE;;;;;;;;;;;;;;;;;;;;;;0FAsByE;IACtF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;QACvB,MAAM,EAAE,CAAC;aACN,MAAM,CAAC;YACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC3B,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;YACxD,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;YAC7B,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;YACpC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;SACpD,CAAC;aACD,WAAW,EAAE;aACb,QAAQ,EAAE;aACV,QAAQ,CAAC,wDAAwD,CAAC;KACtE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE;QACzC,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,MAAM;gBACzB,CAAC,CAAC,eAAe,CAAC,MAAoB,CAAC;gBACvC,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,YAAY,EAAE;gBACzD,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACrD,CAAC,CAAC;YACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,GAAG,GAAG,KAAc,CAAC;YAC3B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAClE,CAAC;IACH,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z, ZodTypeAny } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { AgentModelConfig } from \"../../types/public/agent.js\";\n\ninterface JsonSchema {\n  type?: string;\n  properties?: Record<string, JsonSchema>;\n  items?: JsonSchema;\n  enum?: string[];\n  format?: \"url\" | \"email\" | \"uuid\";\n}\n\nfunction jsonSchemaToZod(schema: JsonSchema): ZodTypeAny {\n  switch (schema.type) {\n    case \"object\": {\n      const shape: Record<string, ZodTypeAny> = {};\n      if (schema.properties) {\n        for (const [key, value] of Object.entries(schema.properties)) {\n          shape[key] = jsonSchemaToZod(value);\n        }\n      }\n      return z.object(shape);\n    }\n    case \"array\":\n      return z.array(schema.items ? jsonSchemaToZod(schema.items) : z.any());\n    case \"string\": {\n      let s = z.string();\n      if (schema.format === \"url\") s = s.url();\n      if (schema.format === \"email\") s = s.email();\n      if (schema.format === \"uuid\") s = s.uuid();\n      if (schema.enum && schema.enum.length > 0)\n        return z.enum(schema.enum as [string, ...string[]]);\n      return s;\n    }\n    case \"number\":\n    case \"integer\":\n      return z.number();\n    case \"boolean\":\n      return z.boolean();\n    case \"null\":\n      return z.null();\n    default:\n      return z.any();\n  }\n}\n\nexport const extractTool = (\n  v3: V3,\n  executionModel?: string | AgentModelConfig,\n) =>\n  tool({\n    description: `Extract structured data from the current page based on a provided schema.\n    \n    USAGE GUIDELINES:\n    - Keep schemas MINIMAL - only include fields essential for the task\n    - IMPORTANT: only use this if explicitly asked for structured output. In most scenarios, you should use the aria tree tool over this.\n    - For URL fields, use format: \"url\"\n    \n    EXAMPLES:\n    1. Extract a single value:\n       instruction: \"extract the product price\"\n       schema: { type: \"object\", properties: { price: { type: \"number\" } } }\n    \n    2. Extract multiple fields:\n       instruction: \"extract product name and price\"\n       schema: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } }\n    \n    3. Extract arrays:\n       instruction: \"extract all product names and prices\"\n       schema: { type: \"object\", properties: { products: { type: \"array\", items: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } } } } }\n    \n    4. Extract a URL:\n       instruction: \"extract the link\"\n       schema: { type: \"object\", properties: { url: { type: \"string\", format: \"url\" } } }`,\n    inputSchema: z.object({\n      instruction: z.string(),\n      schema: z\n        .object({\n          type: z.string().optional(),\n          properties: z.record(z.string(), z.unknown()).optional(),\n          items: z.unknown().optional(),\n          enum: z.array(z.string()).optional(),\n          format: z.enum([\"url\", \"email\", \"uuid\"]).optional(),\n        })\n        .passthrough()\n        .optional()\n        .describe(\"JSON Schema object describing the structure to extract\"),\n    }),\n    execute: async ({ instruction, schema }) => {\n      try {\n        const parsedSchema = schema\n          ? jsonSchemaToZod(schema as JsonSchema)\n          : undefined;\n        const result = await v3.extract(instruction, parsedSchema, {\n          ...(executionModel ? { model: executionModel } : {}),\n        });\n        return { success: true, result };\n      } catch (error) {\n        const err = error as Error;\n        return { success: false, error: err?.message ?? String(error) };\n      }\n    },\n  });\n"]}
+{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAc,MAAM,KAAK,CAAC;AAGpC,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAU/D,SAAS,eAAe,CAAC,MAAkB;IACzC,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,KAAK,GAA+B,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;gBACtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC7D,KAAK,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;gBACtC,CAAC;YACH,CAAC;YACD,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;QACD,KAAK,OAAO;YACV,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QACzE,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;YACnB,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK;gBAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO;gBAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM;gBAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAC3C,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;gBACvC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAA6B,CAAC,CAAC;YACtD,OAAO,CAAC,CAAC;QACX,CAAC;QACD,KAAK,QAAQ,CAAC;QACd,KAAK,SAAS;YACZ,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;QACpB,KAAK,SAAS;YACZ,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;QACrB,KAAK,MAAM;YACT,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAClB;YACE,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,EAAM,EACN,cAA0C,EAC1C,WAAoB,EACpB,EAAE,CACF,IAAI,CAAC;IACH,WAAW,EAAE;;;;;;;;;;;;;;;;;;;;;;0FAsByE;IACtF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;QACvB,MAAM,EAAE,CAAC;aACN,MAAM,CAAC;YACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC3B,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;YACxD,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;YAC7B,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;YACpC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;SACpD,CAAC;aACD,WAAW,EAAE;aACb,QAAQ,EAAE;aACV,QAAQ,CAAC,wDAAwD,CAAC;KACtE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE;QACzC,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,MAAM;gBACzB,CAAC,CAAC,eAAe,CAAC,MAAoB,CAAC;gBACvC,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,YAAY,EAAE;gBACzD,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpD,OAAO,EAAE,WAAW;aACrB,CAAC,CAAC;YACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBAClC,MAAM,cAAc,GAAG,2EAA2E,CAAC;gBACnG,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,cAAc;oBACvB,KAAK,EAAE,CAAC;iBACT,CAAC,CAAC;gBACH,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,cAAc;iBACtB,CAAC;YACJ,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QACpE,CAAC;IACH,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z, ZodTypeAny } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { AgentModelConfig } from \"../../types/public/agent.js\";\nimport { TimeoutError } from \"../../types/public/sdkErrors.js\";\n\ninterface JsonSchema {\n  type?: string;\n  properties?: Record<string, JsonSchema>;\n  items?: JsonSchema;\n  enum?: string[];\n  format?: \"url\" | \"email\" | \"uuid\";\n}\n\nfunction jsonSchemaToZod(schema: JsonSchema): ZodTypeAny {\n  switch (schema.type) {\n    case \"object\": {\n      const shape: Record<string, ZodTypeAny> = {};\n      if (schema.properties) {\n        for (const [key, value] of Object.entries(schema.properties)) {\n          shape[key] = jsonSchemaToZod(value);\n        }\n      }\n      return z.object(shape);\n    }\n    case \"array\":\n      return z.array(schema.items ? jsonSchemaToZod(schema.items) : z.any());\n    case \"string\": {\n      let s = z.string();\n      if (schema.format === \"url\") s = s.url();\n      if (schema.format === \"email\") s = s.email();\n      if (schema.format === \"uuid\") s = s.uuid();\n      if (schema.enum && schema.enum.length > 0)\n        return z.enum(schema.enum as [string, ...string[]]);\n      return s;\n    }\n    case \"number\":\n    case \"integer\":\n      return z.number();\n    case \"boolean\":\n      return z.boolean();\n    case \"null\":\n      return z.null();\n    default:\n      return z.any();\n  }\n}\n\nexport const extractTool = (\n  v3: V3,\n  executionModel?: string | AgentModelConfig,\n  toolTimeout?: number,\n) =>\n  tool({\n    description: `Extract structured data from the current page based on a provided schema.\n    \n    USAGE GUIDELINES:\n    - Keep schemas MINIMAL - only include fields essential for the task\n    - IMPORTANT: only use this if explicitly asked for structured output. In most scenarios, you should use the aria tree tool over this.\n    - For URL fields, use format: \"url\"\n    \n    EXAMPLES:\n    1. Extract a single value:\n       instruction: \"extract the product price\"\n       schema: { type: \"object\", properties: { price: { type: \"number\" } } }\n    \n    2. Extract multiple fields:\n       instruction: \"extract product name and price\"\n       schema: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } }\n    \n    3. Extract arrays:\n       instruction: \"extract all product names and prices\"\n       schema: { type: \"object\", properties: { products: { type: \"array\", items: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } } } } }\n    \n    4. Extract a URL:\n       instruction: \"extract the link\"\n       schema: { type: \"object\", properties: { url: { type: \"string\", format: \"url\" } } }`,\n    inputSchema: z.object({\n      instruction: z.string(),\n      schema: z\n        .object({\n          type: z.string().optional(),\n          properties: z.record(z.string(), z.unknown()).optional(),\n          items: z.unknown().optional(),\n          enum: z.array(z.string()).optional(),\n          format: z.enum([\"url\", \"email\", \"uuid\"]).optional(),\n        })\n        .passthrough()\n        .optional()\n        .describe(\"JSON Schema object describing the structure to extract\"),\n    }),\n    execute: async ({ instruction, schema }) => {\n      try {\n        const parsedSchema = schema\n          ? jsonSchemaToZod(schema as JsonSchema)\n          : undefined;\n        const result = await v3.extract(instruction, parsedSchema, {\n          ...(executionModel ? { model: executionModel } : {}),\n          timeout: toolTimeout,\n        });\n        return { success: true, result };\n      } catch (error) {\n        if (error instanceof TimeoutError) {\n          const timeoutMessage = `TimeoutError: extract() timed out — try using a smaller or simpler schema`;\n          v3.logger({\n            category: \"agent\",\n            message: timeoutMessage,\n            level: 0,\n          });\n          return {\n            success: false,\n            error: timeoutMessage,\n          };\n        }\n        return { success: false, error: error?.message ?? String(error) };\n      }\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/fillFormVision.d.ts

@@ -1,6 +1,6 @@
 import type { V3 } from "../../v3.js";
-import type { FillFormVisionToolResult } from "../../types/public/agent.js";
-export declare const fillFormVisionTool: (v3: V3, provider?: string) => import("ai").Tool<{
+import type { FillFormVisionToolResult, Variables } from "../../types/public/agent.js";
+export declare const fillFormVisionTool: (v3: V3, provider?: string, variables?: Variables) => import("ai").Tool<{
     fields: {
         action: string;
         value: string;