@browserbasehq/orca 3.1.0-patch.4 → 3.2.0-middleware.2

package/dist/cjs/lib/v3/agent/tools/browserbaseSearch.d.ts

@@ -0,0 +1,13 @@
+import type { V3 } from "../../v3.js";
+export interface SearchResult {
+    title: string;
+    url: string;
+    publishedDate?: string;
+}
+export declare const searchTool: (v3: V3, apiKey: string) => import("ai").Tool<{
+    query: string;
+}, {
+    timestamp: number;
+    results: SearchResult[];
+    error?: string;
+}>;

package/dist/cjs/lib/v3/agent/tools/browserbaseSearch.js

@@ -0,0 +1,70 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.searchTool = void 0;
+const ai_1 = require("ai");
+const zod_1 = require("zod");
+async function performBrowserbaseSearch(v3, query, apiKey, numResults = 5) {
+    try {
+        const response = await fetch("https://api.browserbase.com/v1/search", {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "x-bb-api-key": apiKey,
+            },
+            body: JSON.stringify({ query, numResults }),
+        });
+        if (!response.ok) {
+            return {
+                results: [],
+                error: `Browserbase Search API error: ${response.status} ${response.statusText}`,
+            };
+        }
+        const data = (await response.json());
+        const results = (data?.results ?? []).map(({ title, url, publishedDate }) => ({
+            title: title,
+            url: url,
+            ...(publishedDate && { publishedDate }),
+        }));
+        return { results };
+    }
+    catch (error) {
+        v3.logger({
+            category: "agent",
+            message: `Search error: ${error.message}`,
+            level: 0,
+        });
+        return {
+            results: [],
+            error: `Error performing search: ${error.message}`,
+        };
+    }
+}
+const searchTool = (v3, apiKey) => (0, ai_1.tool)({
+    description: "Perform a web search and returns results. Use this tool when you need information from the web or when you are unsure of the exact URL you want to navigate to. This can be used to find the ideal entry point, resulting in a task that is easier to complete due to starting further in the process.",
+    inputSchema: zod_1.z.object({
+        query: zod_1.z.string().describe("The search query to look for on the web"),
+    }),
+    execute: async ({ query }) => {
+        v3.logger({
+            category: "agent",
+            message: `Agent calling tool: search`,
+            level: 1,
+            auxiliary: {
+                arguments: {
+                    value: JSON.stringify({ query }),
+                    type: "object",
+                },
+            },
+        });
+        const result = await performBrowserbaseSearch(v3, query, apiKey);
+        v3.recordAgentReplayStep({
+            type: "search",
+            instruction: query,
+            playwrightArguments: { query },
+            message: result.error ?? `Found ${result.results.length} results`,
+        });
+        return { ...result, timestamp: Date.now() };
+    },
+});
+exports.searchTool = searchTool;
+//# sourceMappingURL=browserbaseSearch.js.map

package/dist/cjs/lib/v3/agent/tools/browserbaseSearch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"browserbaseSearch.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/browserbaseSearch.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAmBxB,KAAK,UAAU,wBAAwB,CACrC,EAAM,EACN,KAAa,EACb,MAAc,EACd,aAAqB,CAAC;IAEtB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,uCAAuC,EAAE;YACpE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,cAAc,EAAE,MAAM;aACvB;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;SAC5C,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,OAAO;gBACL,OAAO,EAAE,EAAE;gBACX,KAAK,EAAE,iCAAiC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE;aACjF,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAA2B,CAAC;QAC/D,MAAM,OAAO,GAAmB,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CACvD,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAC;YAClC,KAAK,EAAE,KAAK;YACZ,GAAG,EAAE,GAAG;YACR,GAAG,CAAC,aAAa,IAAI,EAAE,aAAa,EAAE,CAAC;SACxC,CAAC,CACH,CAAC;QAEF,OAAO,EAAE,OAAO,EAAE,CAAC;IACrB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,iBAAiB,KAAK,CAAC,OAAO,EAAE;YACzC,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,OAAO;YACL,OAAO,EAAE,EAAE;YACX,KAAK,EAAE,4BAA4B,KAAK,CAAC,OAAO,EAAE;SACnD,CAAC;IACJ,CAAC;AACH,CAAC;AAEM,MAAM,UAAU,GAAG,CAAC,EAAM,EAAE,MAAc,EAAE,EAAE,CACnD,IAAA,SAAI,EAAC;IACH,WAAW,EACT,wSAAwS;IAC1S,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;KACtE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;QAC3B,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC;oBAChC,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,wBAAwB,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QAEjE,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,KAAK;YAClB,mBAAmB,EAAE,EAAE,KAAK,EAAE;YAC9B,OAAO,EAAE,MAAM,CAAC,KAAK,IAAI,SAAS,MAAM,CAAC,OAAO,CAAC,MAAM,UAAU;SAClE,CAAC,CAAC;QAEH,OAAO,EAAE,GAAG,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAC9C,CAAC;CACF,CAAC,CAAC;AA/BQ,QAAA,UAAU,cA+BlB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport interface SearchResult {\n  title: string;\n  url: string;\n  publishedDate?: string;\n}\n\ninterface BrowserbaseRawResult {\n  title?: string;\n  url?: string;\n  publishedDate?: string;\n}\n\ninterface BrowserbaseApiResponse {\n  results?: BrowserbaseRawResult[];\n}\n\nasync function performBrowserbaseSearch(\n  v3: V3,\n  query: string,\n  apiKey: string,\n  numResults: number = 5,\n): Promise<{ results: SearchResult[]; error?: string }> {\n  try {\n    const response = await fetch(\"https://api.browserbase.com/v1/search\", {\n      method: \"POST\",\n      headers: {\n        \"Content-Type\": \"application/json\",\n        \"x-bb-api-key\": apiKey,\n      },\n      body: JSON.stringify({ query, numResults }),\n    });\n\n    if (!response.ok) {\n      return {\n        results: [],\n        error: `Browserbase Search API error: ${response.status} ${response.statusText}`,\n      };\n    }\n\n    const data = (await response.json()) as BrowserbaseApiResponse;\n    const results: SearchResult[] = (data?.results ?? []).map(\n      ({ title, url, publishedDate }) => ({\n        title: title,\n        url: url,\n        ...(publishedDate && { publishedDate }),\n      }),\n    );\n\n    return { results };\n  } catch (error) {\n    v3.logger({\n      category: \"agent\",\n      message: `Search error: ${error.message}`,\n      level: 0,\n    });\n    return {\n      results: [],\n      error: `Error performing search: ${error.message}`,\n    };\n  }\n}\n\nexport const searchTool = (v3: V3, apiKey: string) =>\n  tool({\n    description:\n      \"Perform a web search and returns results. Use this tool when you need information from the web or when you are unsure of the exact URL you want to navigate to. This can be used to find the ideal entry point, resulting in a task that is easier to complete due to starting further in the process.\",\n    inputSchema: z.object({\n      query: z.string().describe(\"The search query to look for on the web\"),\n    }),\n    execute: async ({ query }) => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: search`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: JSON.stringify({ query }),\n            type: \"object\",\n          },\n        },\n      });\n\n      const result = await performBrowserbaseSearch(v3, query, apiKey);\n\n      v3.recordAgentReplayStep({\n        type: \"search\",\n        instruction: query,\n        playwrightArguments: { query },\n        message: result.error ?? `Found ${result.results.length} results`,\n      });\n\n      return { ...result, timestamp: Date.now() };\n    },\n  });\n"]}

package/dist/cjs/lib/v3/agent/tools/click.js

@@ -70,38 +70,30 @@ const clickTool = (v3, provider) => (0, ai_1.tool)({
         }
     },
     toModelOutput: (result) => {
-        if (result.success) {
-            const content = [
-                {
-                    type: "text",
-                    text: JSON.stringify({
-                        success: result.success,
-                        describe: result.describe,
-                        coordinates: result.coordinates,
-                    }),
-                },
-            ];
-            if (result.screenshotBase64) {
-                content.push({
-                    type: "media",
-                    mediaType: "image/png",
-                    data: result.screenshotBase64,
-                });
-            }
-            return { type: "content", value: content };
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
         }
-        return {
-            type: "content",
-            value: [
-                {
-                    type: "text",
-                    text: JSON.stringify({
-                        success: result.success,
-                        error: result.error,
-                    }),
-                },
-            ],
-        };
+        const content = [
+            {
+                type: "text",
+                text: JSON.stringify({
+                    success: result.success,
+                    describe: result.describe,
+                    coordinates: result.coordinates,
+                }),
+            },
+        ];
+        if (result.screenshotBase64) {
+            content.push({
+                type: "media",
+                mediaType: "image/png",
+                data: result.screenshotBase64,
+            });
+        }
+        return { type: "content", value: content };
     },
 });
 exports.clickTool = clickTool;

package/dist/cjs/lib/v3/agent/tools/click.js.map

@@ -1 +1 @@
-{"version":3,"file":"click.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/click.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAOxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AAElE,MAAM,SAAS,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CACrD,IAAA,SAAI,EAAC;IACH,WAAW,EACT,wMAAwM;IAC1M,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,OAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,WAAW,EAAE,OAAC;aACX,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAA4B,EAAE;QACrE,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,IAAA,+CAAkB,EAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,2BAA2B;gBACpC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;wBACnC,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;gBACvD,WAAW,EAAE,kBAAkB;aAChC,CAAC,CAAC;YAEH,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,CAAC,CAAC;YAE9D,0FAA0F;YAC1F,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,OAAO;wBACf,SAAS,EAAE,EAAE;qBACd,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;gBACvC,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAoB,KAAe,CAAC,OAAO,EAAE;aACrD,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,WAAW,EAAE,MAAM,CAAC,WAAW;qBAChC,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;QACD,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,KAAK,EAAE,MAAM,CAAC,KAAK;qBACpB,CAAC;iBACH;aACF;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AA9GQ,QAAA,SAAS,aA8GjB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  ClickToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const clickTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Click on an element using its coordinates (this is the most reliable way to click on an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to click on\"),\n    }),\n    execute: async ({ describe, coordinates }): Promise<ClickToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: click`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const xpath = await page.click(processed.x, processed.y, {\n          returnXpath: shouldCollectXpath,\n        });\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"click\",\n              arguments: [],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          coordinates: [processed.x, processed.y],\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error clicking: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success) {\n        const content: ModelOutputContentItem[] = [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              describe: result.describe,\n              coordinates: result.coordinates,\n            }),\n          },\n        ];\n        if (result.screenshotBase64) {\n          content.push({\n            type: \"media\",\n            mediaType: \"image/png\",\n            data: result.screenshotBase64,\n          });\n        }\n        return { type: \"content\", value: content };\n      }\n      return {\n        type: \"content\",\n        value: [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              error: result.error,\n            }),\n          },\n        ],\n      };\n    },\n  });\n"]}
+{"version":3,"file":"click.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/click.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAOxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AAElE,MAAM,SAAS,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CACrD,IAAA,SAAI,EAAC;IACH,WAAW,EACT,wMAAwM;IAC1M,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,OAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,WAAW,EAAE,OAAC;aACX,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAA4B,EAAE;QACrE,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,IAAA,+CAAkB,EAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,2BAA2B;gBACpC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;wBACnC,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;gBACvD,WAAW,EAAE,kBAAkB;aAChC,CAAC,CAAC;YAEH,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,CAAC,CAAC;YAE9D,0FAA0F;YAC1F,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,OAAO;wBACf,SAAS,EAAE,EAAE;qBACd,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;gBACvC,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAmB,KAAK,CAAC,OAAO,EAAE;aAC1C,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,WAAW,EAAE,MAAM,CAAC,WAAW;iBAChC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC;AAvGQ,QAAA,SAAS,aAuGjB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  ClickToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const clickTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Click on an element using its coordinates (this is the most reliable way to click on an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to click on\"),\n    }),\n    execute: async ({ describe, coordinates }): Promise<ClickToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: click`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const xpath = await page.click(processed.x, processed.y, {\n          returnXpath: shouldCollectXpath,\n        });\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"click\",\n              arguments: [],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          coordinates: [processed.x, processed.y],\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error clicking: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n            coordinates: result.coordinates,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}

package/dist/cjs/lib/v3/agent/tools/clickAndHold.js.map

@@ -1 +1 @@
-{"version":3,"file":"clickAndHold.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/clickAndHold.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAGxB,oFAAyE;AACzE,gDAAgD;AAEzC,MAAM,gBAAgB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC5D,IAAA,SAAI,EAAC;IACH,WAAW,EAAE,oDAAoD;IACjE,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,OAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,QAAQ,EAAE,OAAC;aACR,MAAM,EAAE;aACR,QAAQ,CAAC,kDAAkD,CAAC;QAC/D,WAAW,EAAE,OAAC;aACX,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,EAAE,EAAE;QACrD,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,IAAA,+CAAkB,EAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,kCAAkC;gBAC3C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;4BACR,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YAEpD,sFAAsF;YACtF,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACpC,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACrD,CAAC;YAEF,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,cAAc;wBACtB,SAAS,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;qBAC9B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QACrC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,+BAAgC,KAAe,CAAC,OAAO,EAAE;aACjE,CAAC;QACJ,CAAC;IACH,CAAC;CACF,CAAC,CAAC;AAhFQ,QAAA,gBAAgB,oBAgFxB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\n\nexport const clickAndHoldTool = (v3: V3, provider?: string) =>\n  tool({\n    description: \"Click and hold on an element using its coordinates\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      duration: z\n        .number()\n        .describe(\"The duration to hold the element in milliseconds\"),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to click on\"),\n    }),\n    execute: async ({ describe, coordinates, duration }) => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: clickAndHold`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({\n                describe,\n                duration,\n              }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n\n        // Use dragAndDrop from same point to same point with delay to simulate click and hold\n        const [xpath] = await page.dragAndDrop(\n          processed.x,\n          processed.y,\n          processed.x,\n          processed.y,\n          { delay: duration, returnXpath: shouldCollectXpath },\n        );\n\n        // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"clickAndHold\",\n              arguments: [String(duration)],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return { success: true, describe };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error clicking and holding: ${(error as Error).message}`,\n        };\n      }\n    },\n  });\n"]}
+{"version":3,"file":"clickAndHold.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/clickAndHold.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAGxB,oFAAyE;AACzE,gDAAgD;AAEzC,MAAM,gBAAgB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC5D,IAAA,SAAI,EAAC;IACH,WAAW,EAAE,oDAAoD;IACjE,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,OAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,QAAQ,EAAE,OAAC;aACR,MAAM,EAAE;aACR,QAAQ,CAAC,kDAAkD,CAAC;QAC/D,WAAW,EAAE,OAAC;aACX,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,EAAE,EAAE;QACrD,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,IAAA,+CAAkB,EAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,kCAAkC;gBAC3C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;4BACR,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YAEpD,sFAAsF;YACtF,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACpC,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,SAAS,CAAC,CAAC,EACX,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACrD,CAAC;YAEF,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,cAAc;wBACtB,SAAS,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;qBAC9B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QACrC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,+BAA+B,KAAK,CAAC,OAAO,EAAE;aACtD,CAAC;QACJ,CAAC;IACH,CAAC;CACF,CAAC,CAAC;AAhFQ,QAAA,gBAAgB,oBAgFxB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\n\nexport const clickAndHoldTool = (v3: V3, provider?: string) =>\n  tool({\n    description: \"Click and hold on an element using its coordinates\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      duration: z\n        .number()\n        .describe(\"The duration to hold the element in milliseconds\"),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to click on\"),\n    }),\n    execute: async ({ describe, coordinates, duration }) => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: clickAndHold`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({\n                describe,\n                duration,\n              }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n\n        // Use dragAndDrop from same point to same point with delay to simulate click and hold\n        const [xpath] = await page.dragAndDrop(\n          processed.x,\n          processed.y,\n          processed.x,\n          processed.y,\n          { delay: duration, returnXpath: shouldCollectXpath },\n        );\n\n        // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"clickAndHold\",\n              arguments: [String(duration)],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return { success: true, describe };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error clicking and holding: ${error.message}`,\n        };\n      }\n    },\n  });\n"]}

package/dist/cjs/lib/v3/agent/tools/dragAndDrop.js

@@ -72,37 +72,29 @@ const dragAndDropTool = (v3, provider) => (0, ai_1.tool)({
         }
     },
     toModelOutput: (result) => {
-        if (result.success) {
-            const content = [
-                {
-                    type: "text",
-                    text: JSON.stringify({
-                        success: result.success,
-                        describe: result.describe,
-                    }),
-                },
-            ];
-            if (result.screenshotBase64) {
-                content.push({
-                    type: "media",
-                    mediaType: "image/png",
-                    data: result.screenshotBase64,
-                });
-            }
-            return { type: "content", value: content };
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
         }
-        return {
-            type: "content",
-            value: [
-                {
-                    type: "text",
-                    text: JSON.stringify({
-                        success: result.success,
-                        error: result.error,
-                    }),
-                },
-            ],
-        };
+        const content = [
+            {
+                type: "text",
+                text: JSON.stringify({
+                    success: result.success,
+                    describe: result.describe,
+                }),
+            },
+        ];
+        if (result.screenshotBase64) {
+            content.push({
+                type: "media",
+                mediaType: "image/png",
+                data: result.screenshotBase64,
+            });
+        }
+        return { type: "content", value: content };
     },
 });
 exports.dragAndDropTool = dragAndDropTool;

package/dist/cjs/lib/v3/agent/tools/dragAndDrop.js.map

@@ -1 +1 @@
-{"version":3,"file":"dragAndDrop.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/dragAndDrop.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAOxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AAElE,MAAM,eAAe,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC3D,IAAA,SAAI,EAAC;IACH,WAAW,EACT,kNAAkN;IACpN,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uCAAuC,CAAC;QACtE,gBAAgB,EAAE,OAAC;aAChB,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,wDAAwD,CAAC;QACrE,cAAc,EAAE,OAAC;aACd,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oDAAoD,CAAC;KAClE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,gBAAgB,EAChB,cAAc,GACf,EAAkC,EAAE;QACnC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,cAAc,GAAG,IAAA,+CAAkB,EACvC,gBAAgB,CAAC,CAAC,CAAC,EACnB,gBAAgB,CAAC,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;YACF,MAAM,YAAY,GAAG,IAAA,+CAAkB,EACrC,cAAc,CAAC,CAAC,CAAC,EACjB,cAAc,CAAC,CAAC,CAAC,EACjB,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,iCAAiC;gBAC1C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACjD,cAAc,CAAC,CAAC,EAChB,cAAc,CAAC,CAAC,EAChB,YAAY,CAAC,CAAC,EACd,YAAY,CAAC,CAAC,EACd,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACpC,CAAC;YAEF,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,CAAC,CAAC;YAE9D,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,cAAc,GAAG,IAAA,sBAAW,EAAC,SAAS,CAAC,CAAC;gBAC9C,MAAM,YAAY,GAAG,IAAA,sBAAW,EAAC,OAAO,CAAC,CAAC;gBAC1C,IAAI,cAAc,IAAI,YAAY,EAAE,CAAC;oBACnC,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,cAAc;wBACxB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,aAAa;wBACrB,SAAS,EAAE,CAAC,YAAY,CAAC;qBAC1B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAoB,KAAe,CAAC,OAAO,EAAE;aACrD,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;qBAC1B,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;QACD,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,KAAK,EAAE,MAAM,CAAC,KAAK;qBACpB,CAAC;iBACH;aACF;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AA5HQ,QAAA,eAAe,mBA4HvB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  DragAndDropToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const dragAndDropTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Drag and drop an element using its coordinates (this is the most reliable way to drag and drop an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z.string().describe(\"Describe the element to drag and drop\"),\n      startCoordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to start the drag and drop from\"),\n      endCoordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to end the drag and drop at\"),\n    }),\n    execute: async ({\n      describe,\n      startCoordinates,\n      endCoordinates,\n    }): Promise<DragAndDropToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processedStart = processCoordinates(\n          startCoordinates[0],\n          startCoordinates[1],\n          provider,\n          v3,\n        );\n        const processedEnd = processCoordinates(\n          endCoordinates[0],\n          endCoordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: dragAndDrop`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({\n                describe,\n              }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const [fromXpath, toXpath] = await page.dragAndDrop(\n          processedStart.x,\n          processedStart.y,\n          processedEnd.x,\n          processedEnd.y,\n          { returnXpath: shouldCollectXpath },\n        );\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedFrom = ensureXPath(fromXpath);\n          const normalizedTo = ensureXPath(toXpath);\n          if (normalizedFrom && normalizedTo) {\n            const action: Action = {\n              selector: normalizedFrom,\n              description: describe,\n              method: \"dragAndDrop\",\n              arguments: [normalizedTo],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error dragging: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success) {\n        const content: ModelOutputContentItem[] = [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              describe: result.describe,\n            }),\n          },\n        ];\n        if (result.screenshotBase64) {\n          content.push({\n            type: \"media\",\n            mediaType: \"image/png\",\n            data: result.screenshotBase64,\n          });\n        }\n        return { type: \"content\", value: content };\n      }\n      return {\n        type: \"content\",\n        value: [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              error: result.error,\n            }),\n          },\n        ],\n      };\n    },\n  });\n"]}
+{"version":3,"file":"dragAndDrop.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/dragAndDrop.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAOxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AAElE,MAAM,eAAe,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC3D,IAAA,SAAI,EAAC;IACH,WAAW,EACT,kNAAkN;IACpN,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uCAAuC,CAAC;QACtE,gBAAgB,EAAE,OAAC;aAChB,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,wDAAwD,CAAC;QACrE,cAAc,EAAE,OAAC;aACd,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oDAAoD,CAAC;KAClE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,gBAAgB,EAChB,cAAc,GACf,EAAkC,EAAE;QACnC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,cAAc,GAAG,IAAA,+CAAkB,EACvC,gBAAgB,CAAC,CAAC,CAAC,EACnB,gBAAgB,CAAC,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;YACF,MAAM,YAAY,GAAG,IAAA,+CAAkB,EACrC,cAAc,CAAC,CAAC,CAAC,EACjB,cAAc,CAAC,CAAC,CAAC,EACjB,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,iCAAiC;gBAC1C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACjD,cAAc,CAAC,CAAC,EAChB,cAAc,CAAC,CAAC,EAChB,YAAY,CAAC,CAAC,EACd,YAAY,CAAC,CAAC,EACd,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACpC,CAAC;YAEF,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,CAAC,CAAC;YAE9D,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,cAAc,GAAG,IAAA,sBAAW,EAAC,SAAS,CAAC,CAAC;gBAC9C,MAAM,YAAY,GAAG,IAAA,sBAAW,EAAC,OAAO,CAAC,CAAC;gBAC1C,IAAI,cAAc,IAAI,YAAY,EAAE,CAAC;oBACnC,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,cAAc;wBACxB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,aAAa;wBACrB,SAAS,EAAE,CAAC,YAAY,CAAC;qBAC1B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAmB,KAAK,CAAC,OAAO,EAAE;aAC1C,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;iBAC1B,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC;AArHQ,QAAA,eAAe,mBAqHvB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  DragAndDropToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const dragAndDropTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Drag and drop an element using its coordinates (this is the most reliable way to drag and drop an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z.string().describe(\"Describe the element to drag and drop\"),\n      startCoordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to start the drag and drop from\"),\n      endCoordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to end the drag and drop at\"),\n    }),\n    execute: async ({\n      describe,\n      startCoordinates,\n      endCoordinates,\n    }): Promise<DragAndDropToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processedStart = processCoordinates(\n          startCoordinates[0],\n          startCoordinates[1],\n          provider,\n          v3,\n        );\n        const processedEnd = processCoordinates(\n          endCoordinates[0],\n          endCoordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: dragAndDrop`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({\n                describe,\n              }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const [fromXpath, toXpath] = await page.dragAndDrop(\n          processedStart.x,\n          processedStart.y,\n          processedEnd.x,\n          processedEnd.y,\n          { returnXpath: shouldCollectXpath },\n        );\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedFrom = ensureXPath(fromXpath);\n          const normalizedTo = ensureXPath(toXpath);\n          if (normalizedFrom && normalizedTo) {\n            const action: Action = {\n              selector: normalizedFrom,\n              description: describe,\n              method: \"dragAndDrop\",\n              arguments: [normalizedTo],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error dragging: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}

package/dist/cjs/lib/v3/agent/tools/extract.d.ts

@@ -1,6 +1,6 @@
 import type { V3 } from "../../v3.js";
 import type { AgentModelConfig } from "../../types/public/agent.js";
-export declare const extractTool: (v3: V3, executionModel?: string | AgentModelConfig) => import("ai").Tool<{
+export declare const extractTool: (v3: V3, executionModel?: string | AgentModelConfig, toolTimeout?: number) => import("ai").Tool<{
     instruction: string;
     schema?: {
         [x: string]: unknown;
@@ -16,6 +16,6 @@ export declare const extractTool: (v3: V3, executionModel?: string | AgentModelC
     error?: undefined;
 } | {
     success: boolean;
-    error: string;
+    error: any;
     result?: undefined;
 }>;

package/dist/cjs/lib/v3/agent/tools/extract.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.extractTool = void 0;
 const ai_1 = require("ai");
 const zod_1 = require("zod");
+const sdkErrors_js_1 = require("../../types/public/sdkErrors.js");
 function jsonSchemaToZod(schema) {
     switch (schema.type) {
         case "object": {
@@ -39,7 +40,7 @@ function jsonSchemaToZod(schema) {
             return zod_1.z.any();
     }
 }
-const extractTool = (v3, executionModel) => (0, ai_1.tool)({
+const extractTool = (v3, executionModel, toolTimeout) => (0, ai_1.tool)({
     description: `Extract structured data from the current page based on a provided schema.
     
     USAGE GUIDELINES:
@@ -84,12 +85,15 @@ const extractTool = (v3, executionModel) => (0, ai_1.tool)({
                 : undefined;
             const result = await v3.extract(instruction, parsedSchema, {
                 ...(executionModel ? { model: executionModel } : {}),
+                timeout: toolTimeout,
             });
             return { success: true, result };
         }
         catch (error) {
-            const err = error;
-            return { success: false, error: err?.message ?? String(error) };
+            if (error instanceof sdkErrors_js_1.TimeoutError) {
+                throw error;
+            }
+            return { success: false, error: error?.message ?? String(error) };
         }
     },
 });

package/dist/cjs/lib/v3/agent/tools/extract.js.map

@@ -1 +1 @@
-{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/extract.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAoC;AAYpC,SAAS,eAAe,CAAC,MAAkB;IACzC,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,KAAK,GAA+B,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;gBACtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC7D,KAAK,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;gBACtC,CAAC;YACH,CAAC;YACD,OAAO,OAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;QACD,KAAK,OAAO;YACV,OAAO,OAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QACzE,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,IAAI,CAAC,GAAG,OAAC,CAAC,MAAM,EAAE,CAAC;YACnB,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK;gBAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO;gBAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM;gBAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAC3C,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;gBACvC,OAAO,OAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAA6B,CAAC,CAAC;YACtD,OAAO,CAAC,CAAC;QACX,CAAC;QACD,KAAK,QAAQ,CAAC;QACd,KAAK,SAAS;YACZ,OAAO,OAAC,CAAC,MAAM,EAAE,CAAC;QACpB,KAAK,SAAS;YACZ,OAAO,OAAC,CAAC,OAAO,EAAE,CAAC;QACrB,KAAK,MAAM;YACT,OAAO,OAAC,CAAC,IAAI,EAAE,CAAC;QAClB;YACE,OAAO,OAAC,CAAC,GAAG,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAEM,MAAM,WAAW,GAAG,CACzB,EAAM,EACN,cAA0C,EAC1C,EAAE,CACF,IAAA,SAAI,EAAC;IACH,WAAW,EAAE;;;;;;;;;;;;;;;;;;;;;;0FAsByE;IACtF,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,WAAW,EAAE,OAAC,CAAC,MAAM,EAAE;QACvB,MAAM,EAAE,OAAC;aACN,MAAM,CAAC;YACN,IAAI,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC3B,UAAU,EAAE,OAAC,CAAC,MAAM,CAAC,OAAC,CAAC,MAAM,EAAE,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;YACxD,KAAK,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;YAC7B,IAAI,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;YACpC,MAAM,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;SACpD,CAAC;aACD,WAAW,EAAE;aACb,QAAQ,EAAE;aACV,QAAQ,CAAC,wDAAwD,CAAC;KACtE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE;QACzC,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,MAAM;gBACzB,CAAC,CAAC,eAAe,CAAC,MAAoB,CAAC;gBACvC,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,YAAY,EAAE;gBACzD,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACrD,CAAC,CAAC;YACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,GAAG,GAAG,KAAc,CAAC;YAC3B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAClE,CAAC;IACH,CAAC;CACF,CAAC,CAAC;AAxDQ,QAAA,WAAW,eAwDnB","sourcesContent":["import { tool } from \"ai\";\nimport { z, ZodTypeAny } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { AgentModelConfig } from \"../../types/public/agent.js\";\n\ninterface JsonSchema {\n  type?: string;\n  properties?: Record<string, JsonSchema>;\n  items?: JsonSchema;\n  enum?: string[];\n  format?: \"url\" | \"email\" | \"uuid\";\n}\n\nfunction jsonSchemaToZod(schema: JsonSchema): ZodTypeAny {\n  switch (schema.type) {\n    case \"object\": {\n      const shape: Record<string, ZodTypeAny> = {};\n      if (schema.properties) {\n        for (const [key, value] of Object.entries(schema.properties)) {\n          shape[key] = jsonSchemaToZod(value);\n        }\n      }\n      return z.object(shape);\n    }\n    case \"array\":\n      return z.array(schema.items ? jsonSchemaToZod(schema.items) : z.any());\n    case \"string\": {\n      let s = z.string();\n      if (schema.format === \"url\") s = s.url();\n      if (schema.format === \"email\") s = s.email();\n      if (schema.format === \"uuid\") s = s.uuid();\n      if (schema.enum && schema.enum.length > 0)\n        return z.enum(schema.enum as [string, ...string[]]);\n      return s;\n    }\n    case \"number\":\n    case \"integer\":\n      return z.number();\n    case \"boolean\":\n      return z.boolean();\n    case \"null\":\n      return z.null();\n    default:\n      return z.any();\n  }\n}\n\nexport const extractTool = (\n  v3: V3,\n  executionModel?: string | AgentModelConfig,\n) =>\n  tool({\n    description: `Extract structured data from the current page based on a provided schema.\n    \n    USAGE GUIDELINES:\n    - Keep schemas MINIMAL - only include fields essential for the task\n    - IMPORTANT: only use this if explicitly asked for structured output. In most scenarios, you should use the aria tree tool over this.\n    - For URL fields, use format: \"url\"\n    \n    EXAMPLES:\n    1. Extract a single value:\n       instruction: \"extract the product price\"\n       schema: { type: \"object\", properties: { price: { type: \"number\" } } }\n    \n    2. Extract multiple fields:\n       instruction: \"extract product name and price\"\n       schema: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } }\n    \n    3. Extract arrays:\n       instruction: \"extract all product names and prices\"\n       schema: { type: \"object\", properties: { products: { type: \"array\", items: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } } } } }\n    \n    4. Extract a URL:\n       instruction: \"extract the link\"\n       schema: { type: \"object\", properties: { url: { type: \"string\", format: \"url\" } } }`,\n    inputSchema: z.object({\n      instruction: z.string(),\n      schema: z\n        .object({\n          type: z.string().optional(),\n          properties: z.record(z.string(), z.unknown()).optional(),\n          items: z.unknown().optional(),\n          enum: z.array(z.string()).optional(),\n          format: z.enum([\"url\", \"email\", \"uuid\"]).optional(),\n        })\n        .passthrough()\n        .optional()\n        .describe(\"JSON Schema object describing the structure to extract\"),\n    }),\n    execute: async ({ instruction, schema }) => {\n      try {\n        const parsedSchema = schema\n          ? jsonSchemaToZod(schema as JsonSchema)\n          : undefined;\n        const result = await v3.extract(instruction, parsedSchema, {\n          ...(executionModel ? { model: executionModel } : {}),\n        });\n        return { success: true, result };\n      } catch (error) {\n        const err = error as Error;\n        return { success: false, error: err?.message ?? String(error) };\n      }\n    },\n  });\n"]}
+{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/extract.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAoC;AAGpC,kEAA+D;AAU/D,SAAS,eAAe,CAAC,MAAkB;IACzC,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,KAAK,GAA+B,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;gBACtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC7D,KAAK,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;gBACtC,CAAC;YACH,CAAC;YACD,OAAO,OAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;QACD,KAAK,OAAO;YACV,OAAO,OAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QACzE,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,IAAI,CAAC,GAAG,OAAC,CAAC,MAAM,EAAE,CAAC;YACnB,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK;gBAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO;gBAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM;gBAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAC3C,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;gBACvC,OAAO,OAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAA6B,CAAC,CAAC;YACtD,OAAO,CAAC,CAAC;QACX,CAAC;QACD,KAAK,QAAQ,CAAC;QACd,KAAK,SAAS;YACZ,OAAO,OAAC,CAAC,MAAM,EAAE,CAAC;QACpB,KAAK,SAAS;YACZ,OAAO,OAAC,CAAC,OAAO,EAAE,CAAC;QACrB,KAAK,MAAM;YACT,OAAO,OAAC,CAAC,IAAI,EAAE,CAAC;QAClB;YACE,OAAO,OAAC,CAAC,GAAG,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAEM,MAAM,WAAW,GAAG,CACzB,EAAM,EACN,cAA0C,EAC1C,WAAoB,EACpB,EAAE,CACF,IAAA,SAAI,EAAC;IACH,WAAW,EAAE;;;;;;;;;;;;;;;;;;;;;;0FAsByE;IACtF,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;QACpB,WAAW,EAAE,OAAC,CAAC,MAAM,EAAE;QACvB,MAAM,EAAE,OAAC;aACN,MAAM,CAAC;YACN,IAAI,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC3B,UAAU,EAAE,OAAC,CAAC,MAAM,CAAC,OAAC,CAAC,MAAM,EAAE,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;YACxD,KAAK,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;YAC7B,IAAI,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;YACpC,MAAM,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;SACpD,CAAC;aACD,WAAW,EAAE;aACb,QAAQ,EAAE;aACV,QAAQ,CAAC,wDAAwD,CAAC;KACtE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE;QACzC,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,MAAM;gBACzB,CAAC,CAAC,eAAe,CAAC,MAAoB,CAAC;gBACvC,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,YAAY,EAAE;gBACzD,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpD,OAAO,EAAE,WAAW;aACrB,CAAC,CAAC;YACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,2BAAY,EAAE,CAAC;gBAClC,MAAM,KAAK,CAAC;YACd,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QACpE,CAAC;IACH,CAAC;CACF,CAAC,CAAC;AA5DQ,QAAA,WAAW,eA4DnB","sourcesContent":["import { tool } from \"ai\";\nimport { z, ZodTypeAny } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { AgentModelConfig } from \"../../types/public/agent.js\";\nimport { TimeoutError } from \"../../types/public/sdkErrors.js\";\n\ninterface JsonSchema {\n  type?: string;\n  properties?: Record<string, JsonSchema>;\n  items?: JsonSchema;\n  enum?: string[];\n  format?: \"url\" | \"email\" | \"uuid\";\n}\n\nfunction jsonSchemaToZod(schema: JsonSchema): ZodTypeAny {\n  switch (schema.type) {\n    case \"object\": {\n      const shape: Record<string, ZodTypeAny> = {};\n      if (schema.properties) {\n        for (const [key, value] of Object.entries(schema.properties)) {\n          shape[key] = jsonSchemaToZod(value);\n        }\n      }\n      return z.object(shape);\n    }\n    case \"array\":\n      return z.array(schema.items ? jsonSchemaToZod(schema.items) : z.any());\n    case \"string\": {\n      let s = z.string();\n      if (schema.format === \"url\") s = s.url();\n      if (schema.format === \"email\") s = s.email();\n      if (schema.format === \"uuid\") s = s.uuid();\n      if (schema.enum && schema.enum.length > 0)\n        return z.enum(schema.enum as [string, ...string[]]);\n      return s;\n    }\n    case \"number\":\n    case \"integer\":\n      return z.number();\n    case \"boolean\":\n      return z.boolean();\n    case \"null\":\n      return z.null();\n    default:\n      return z.any();\n  }\n}\n\nexport const extractTool = (\n  v3: V3,\n  executionModel?: string | AgentModelConfig,\n  toolTimeout?: number,\n) =>\n  tool({\n    description: `Extract structured data from the current page based on a provided schema.\n    \n    USAGE GUIDELINES:\n    - Keep schemas MINIMAL - only include fields essential for the task\n    - IMPORTANT: only use this if explicitly asked for structured output. In most scenarios, you should use the aria tree tool over this.\n    - For URL fields, use format: \"url\"\n    \n    EXAMPLES:\n    1. Extract a single value:\n       instruction: \"extract the product price\"\n       schema: { type: \"object\", properties: { price: { type: \"number\" } } }\n    \n    2. Extract multiple fields:\n       instruction: \"extract product name and price\"\n       schema: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } }\n    \n    3. Extract arrays:\n       instruction: \"extract all product names and prices\"\n       schema: { type: \"object\", properties: { products: { type: \"array\", items: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } } } } }\n    \n    4. Extract a URL:\n       instruction: \"extract the link\"\n       schema: { type: \"object\", properties: { url: { type: \"string\", format: \"url\" } } }`,\n    inputSchema: z.object({\n      instruction: z.string(),\n      schema: z\n        .object({\n          type: z.string().optional(),\n          properties: z.record(z.string(), z.unknown()).optional(),\n          items: z.unknown().optional(),\n          enum: z.array(z.string()).optional(),\n          format: z.enum([\"url\", \"email\", \"uuid\"]).optional(),\n        })\n        .passthrough()\n        .optional()\n        .describe(\"JSON Schema object describing the structure to extract\"),\n    }),\n    execute: async ({ instruction, schema }) => {\n      try {\n        const parsedSchema = schema\n          ? jsonSchemaToZod(schema as JsonSchema)\n          : undefined;\n        const result = await v3.extract(instruction, parsedSchema, {\n          ...(executionModel ? { model: executionModel } : {}),\n          timeout: toolTimeout,\n        });\n        return { success: true, result };\n      } catch (error) {\n        if (error instanceof TimeoutError) {\n          throw error;\n        }\n        return { success: false, error: error?.message ?? String(error) };\n      }\n    },\n  });\n"]}

package/dist/cjs/lib/v3/agent/tools/fillFormVision.js

@@ -120,37 +120,37 @@ MANDATORY USE CASES (always use fillFormVision for these):
             }
         },
         toModelOutput: (result) => {
-            if (result.success) {
-                const content = [
-                    {
-                        type: "text",
-                        text: JSON.stringify({
-                            success: result.success,
-                            fieldsCount: result.playwrightArguments?.length ?? 0,
-                        }),
-                    },
-                ];
-                if (result.screenshotBase64) {
-                    content.push({
-                        type: "media",
-                        mediaType: "image/png",
-                        data: result.screenshotBase64,
-                    });
-                }
-                return { type: "content", value: content };
+            if (result.success === false || result.error !== undefined) {
+                return {
+                    type: "content",
+                    value: [
+                        {
+                            type: "text",
+                            text: JSON.stringify({
+                                success: result.success,
+                                error: result.error,
+                            }),
+                        },
+                    ],
+                };
             }
-            return {
-                type: "content",
-                value: [
-                    {
-                        type: "text",
-                        text: JSON.stringify({
-                            success: result.success,
-                            error: result.error,
-                        }),
-                    },
-                ],
-            };
+            const content = [
+                {
+                    type: "text",
+                    text: JSON.stringify({
+                        success: result.success,
+                        fieldsCount: result.playwrightArguments?.length ?? 0,
+                    }),
+                },
+            ];
+            if (result.screenshotBase64) {
+                content.push({
+                    type: "media",
+                    mediaType: "image/png",
+                    data: result.screenshotBase64,
+                });
+            }
+            return { type: "content", value: content };
         },
     });
 };

package/dist/cjs/lib/v3/agent/tools/fillFormVision.js.map

@@ -1 +1 @@
-{"version":3,"file":"fillFormVision.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillFormVision.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAQxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AACzE,wDAA4D;AAErD,MAAM,kBAAkB,GAAG,CAChC,EAAM,EACN,QAAiB,EACjB,SAAqB,EACrB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,qGAAqG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAC1I,CAAC,CAAC,oCAAoC,CAAC;IAEzC,OAAO,IAAA,SAAI,EAAC;QACV,WAAW,EAAE;;;;;;;;;;;;;;;;2CAgB0B;QACvC,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,OAAC;iBACN,KAAK,CACJ,OAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,OAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,sEAAsE,CACvE;gBACH,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBAC5C,WAAW,EAAE,OAAC;qBACX,MAAM,CAAC;oBACN,CAAC,EAAE,OAAC,CAAC,MAAM,EAAE;oBACb,CAAC,EAAE,OAAC,CAAC,MAAM,EAAE;iBACd,CAAC;qBACD,QAAQ,CAAC,iCAAiC,CAAC;aAC/C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,qCAAqC,CAAC;SACjD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAqC,EAAE;YAC/D,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAEhD,8DAA8D;gBAC9D,0FAA0F;gBAC1F,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;oBAC3C,MAAM,SAAS,GAAG,IAAA,+CAAkB,EAClC,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;oBACF,OAAO;wBACL,GAAG,KAAK;wBACR,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE,wCAAwC;wBACpE,KAAK,EAAE,IAAA,kCAAmB,EAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC;wBAClD,WAAW,EAAE,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;qBAChD,CAAC;gBACJ,CAAC,CAAC,CAAC;gBAEH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,oCAAoC;oBAC7C,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,+BAA+B;4BAClE,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,OAAO,GAAa,EAAE,CAAC;gBAE7B,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;oBACpC,iEAAiE;oBACjE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAC5B,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB;wBACE,WAAW,EAAE,kBAAkB;qBAChC,CACF,CAAC;oBACF,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;oBAE7B,uEAAuE;oBACvE,qFAAqF;oBACrF,IAAI,kBAAkB,EAAE,CAAC;wBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;wBAC3C,IAAI,eAAe,EAAE,CAAC;4BACpB,OAAO,CAAC,IAAI,CAAC;gCACX,QAAQ,EAAE,eAAe;gCACzB,WAAW,EAAE,KAAK,CAAC,MAAM;gCACzB,MAAM,EAAE,MAAM;gCACd,SAAS,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC;6BACjC,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,6BAA6B;oBAC7B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;gBAC3D,CAAC;gBAED,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAEnE,wFAAwF;gBACxF,IAAI,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7C,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;wBAChD,OAAO;wBACP,iBAAiB,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;qBACvD,CAAC,CAAC;gBACL,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,mBAAmB,EAAE,eAAe;oBACpC,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,uBAAwB,KAAe,CAAC,OAAO,EAAE;iBACzD,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,MAAM,OAAO,GAA6B;oBACxC;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;4BACnB,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,WAAW,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,IAAI,CAAC;yBACrD,CAAC;qBACH;iBACF,CAAC;gBACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;oBAC5B,OAAO,CAAC,IAAI,CAAC;wBACX,IAAI,EAAE,OAAO;wBACb,SAAS,EAAE,WAAW;wBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;qBAC9B,CAAC,CAAC;gBACL,CAAC;gBACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;YAC7C,CAAC;YACD,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE;oBACL;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;4BACnB,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,KAAK,EAAE,MAAM,CAAC,KAAK;yBACpB,CAAC;qBACH;iBACF;aACF,CAAC;QACJ,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC;AA5KW,QAAA,kBAAkB,sBA4K7B","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  FillFormVisionToolResult,\n  ModelOutputContentItem,\n  Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const fillFormVisionTool = (\n  v3: V3,\n  provider?: string,\n  variables?: Variables,\n) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const valueDescription = hasVariables\n    ? `Text to type into the target field. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"Text to type into the target field\";\n\n  return tool({\n    description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL\n\nCRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)\nIMPORTANT: Ensure the fields are visible within the current viewport\n\nWHY THIS TOOL EXISTS:\n- Forms are the #1 use case for multi-field input\n- Optimized specifically for input/textarea elements\n- 4-6x faster than individual typing actions\n\nUse fillFormVision: Pure form filling (inputs, textareas only)\nMANDATORY USE CASES (always use fillFormVision for these):\n- Registration forms: name, email, password fields\n- Contact forms: name, email, message fields\n- Checkout forms: address, payment info fields\n- Profile updates: multiple user data fields\n- Search filters: multiple criteria inputs`,\n    inputSchema: z.object({\n      fields: z\n        .array(\n          z.object({\n            action: z\n              .string()\n              .describe(\n                \"Description of the typing action, e.g. 'type foo into the bar field'\",\n              ),\n            value: z.string().describe(valueDescription),\n            coordinates: z\n              .object({\n                x: z.number(),\n                y: z.number(),\n              })\n              .describe(\"Coordinates of the target field\"),\n          }),\n        )\n        .min(2, \"Provide at least two fields to fill\"),\n    }),\n    execute: async ({ fields }): Promise<FillFormVisionToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n\n        // Process coordinates and substitute variables for each field\n        // Keep original values (with %tokens%) for logging/caching, substituted values for typing\n        const processedFields = fields.map((field) => {\n          const processed = processCoordinates(\n            field.coordinates.x,\n            field.coordinates.y,\n            provider,\n            v3,\n          );\n          return {\n            ...field,\n            originalValue: field.value, // Keep original with %tokens% for cache\n            value: substituteVariables(field.value, variables),\n            coordinates: { x: processed.x, y: processed.y },\n          };\n        });\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: fillFormVision`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ fields }), // Don't log substituted values\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const actions: Action[] = [];\n\n        for (const field of processedFields) {\n          // Click the field, only requesting XPath when caching is enabled\n          const xpath = await page.click(\n            field.coordinates.x,\n            field.coordinates.y,\n            {\n              returnXpath: shouldCollectXpath,\n            },\n          );\n          await page.type(field.value);\n\n          // Build Action with XPath for deterministic replay (only when caching)\n          // Use originalValue (with %tokens%) so cache stores references, not sensitive values\n          if (shouldCollectXpath) {\n            const normalizedXpath = ensureXPath(xpath);\n            if (normalizedXpath) {\n              actions.push({\n                selector: normalizedXpath,\n                description: field.action,\n                method: \"type\",\n                arguments: [field.originalValue],\n              });\n            }\n          }\n\n          // Small delay between fields\n          await new Promise((resolve) => setTimeout(resolve, 100));\n        }\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n        // Record as \"act\" step with proper Actions for deterministic replay (only when caching)\n        if (shouldCollectXpath && actions.length > 0) {\n          v3.recordAgentReplayStep({\n            type: \"act\",\n            instruction: `Fill ${fields.length} form fields`,\n            actions,\n            actionDescription: `Fill ${fields.length} form fields`,\n          });\n        }\n\n        return {\n          success: true,\n          playwrightArguments: processedFields,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error filling form: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success) {\n        const content: ModelOutputContentItem[] = [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              fieldsCount: result.playwrightArguments?.length ?? 0,\n            }),\n          },\n        ];\n        if (result.screenshotBase64) {\n          content.push({\n            type: \"media\",\n            mediaType: \"image/png\",\n            data: result.screenshotBase64,\n          });\n        }\n        return { type: \"content\", value: content };\n      }\n      return {\n        type: \"content\",\n        value: [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              error: result.error,\n            }),\n          },\n        ],\n      };\n    },\n  });\n};\n"]}
+{"version":3,"file":"fillFormVision.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillFormVision.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAQxB,oFAAyE;AACzE,gDAAgD;AAChD,wEAAyE;AACzE,wDAA4D;AAErD,MAAM,kBAAkB,GAAG,CAChC,EAAM,EACN,QAAiB,EACjB,SAAqB,EACrB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,qGAAqG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAC1I,CAAC,CAAC,oCAAoC,CAAC;IAEzC,OAAO,IAAA,SAAI,EAAC;QACV,WAAW,EAAE;;;;;;;;;;;;;;;;2CAgB0B;QACvC,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,OAAC;iBACN,KAAK,CACJ,OAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,OAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,sEAAsE,CACvE;gBACH,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBAC5C,WAAW,EAAE,OAAC;qBACX,MAAM,CAAC;oBACN,CAAC,EAAE,OAAC,CAAC,MAAM,EAAE;oBACb,CAAC,EAAE,OAAC,CAAC,MAAM,EAAE;iBACd,CAAC;qBACD,QAAQ,CAAC,iCAAiC,CAAC;aAC/C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,qCAAqC,CAAC;SACjD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAqC,EAAE;YAC/D,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAEhD,8DAA8D;gBAC9D,0FAA0F;gBAC1F,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;oBAC3C,MAAM,SAAS,GAAG,IAAA,+CAAkB,EAClC,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;oBACF,OAAO;wBACL,GAAG,KAAK;wBACR,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE,wCAAwC;wBACpE,KAAK,EAAE,IAAA,kCAAmB,EAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC;wBAClD,WAAW,EAAE,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;qBAChD,CAAC;gBACJ,CAAC,CAAC,CAAC;gBAEH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,oCAAoC;oBAC7C,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,+BAA+B;4BAClE,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,OAAO,GAAa,EAAE,CAAC;gBAE7B,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;oBACpC,iEAAiE;oBACjE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAC5B,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB;wBACE,WAAW,EAAE,kBAAkB;qBAChC,CACF,CAAC;oBACF,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;oBAE7B,uEAAuE;oBACvE,qFAAqF;oBACrF,IAAI,kBAAkB,EAAE,CAAC;wBACvB,MAAM,eAAe,GAAG,IAAA,sBAAW,EAAC,KAAK,CAAC,CAAC;wBAC3C,IAAI,eAAe,EAAE,CAAC;4BACpB,OAAO,CAAC,IAAI,CAAC;gCACX,QAAQ,EAAE,eAAe;gCACzB,WAAW,EAAE,KAAK,CAAC,MAAM;gCACzB,MAAM,EAAE,MAAM;gCACd,SAAS,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC;6BACjC,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,6BAA6B;oBAC7B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;gBAC3D,CAAC;gBAED,MAAM,gBAAgB,GAAG,MAAM,IAAA,+CAAwB,EAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAEnE,wFAAwF;gBACxF,IAAI,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7C,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;wBAChD,OAAO;wBACP,iBAAiB,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;qBACvD,CAAC,CAAC;gBACL,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,mBAAmB,EAAE,eAAe;oBACpC,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,uBAAuB,KAAK,CAAC,OAAO,EAAE;iBAC9C,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE;wBACL;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gCACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gCACvB,KAAK,EAAE,MAAM,CAAC,KAAK;6BACpB,CAAC;yBACH;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,WAAW,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,IAAI,CAAC;qBACrD,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC;AA7KW,QAAA,kBAAkB,sBA6K7B","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  FillFormVisionToolResult,\n  ModelOutputContentItem,\n  Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const fillFormVisionTool = (\n  v3: V3,\n  provider?: string,\n  variables?: Variables,\n) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const valueDescription = hasVariables\n    ? `Text to type into the target field. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"Text to type into the target field\";\n\n  return tool({\n    description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL\n\nCRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)\nIMPORTANT: Ensure the fields are visible within the current viewport\n\nWHY THIS TOOL EXISTS:\n- Forms are the #1 use case for multi-field input\n- Optimized specifically for input/textarea elements\n- 4-6x faster than individual typing actions\n\nUse fillFormVision: Pure form filling (inputs, textareas only)\nMANDATORY USE CASES (always use fillFormVision for these):\n- Registration forms: name, email, password fields\n- Contact forms: name, email, message fields\n- Checkout forms: address, payment info fields\n- Profile updates: multiple user data fields\n- Search filters: multiple criteria inputs`,\n    inputSchema: z.object({\n      fields: z\n        .array(\n          z.object({\n            action: z\n              .string()\n              .describe(\n                \"Description of the typing action, e.g. 'type foo into the bar field'\",\n              ),\n            value: z.string().describe(valueDescription),\n            coordinates: z\n              .object({\n                x: z.number(),\n                y: z.number(),\n              })\n              .describe(\"Coordinates of the target field\"),\n          }),\n        )\n        .min(2, \"Provide at least two fields to fill\"),\n    }),\n    execute: async ({ fields }): Promise<FillFormVisionToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n\n        // Process coordinates and substitute variables for each field\n        // Keep original values (with %tokens%) for logging/caching, substituted values for typing\n        const processedFields = fields.map((field) => {\n          const processed = processCoordinates(\n            field.coordinates.x,\n            field.coordinates.y,\n            provider,\n            v3,\n          );\n          return {\n            ...field,\n            originalValue: field.value, // Keep original with %tokens% for cache\n            value: substituteVariables(field.value, variables),\n            coordinates: { x: processed.x, y: processed.y },\n          };\n        });\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: fillFormVision`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ fields }), // Don't log substituted values\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const actions: Action[] = [];\n\n        for (const field of processedFields) {\n          // Click the field, only requesting XPath when caching is enabled\n          const xpath = await page.click(\n            field.coordinates.x,\n            field.coordinates.y,\n            {\n              returnXpath: shouldCollectXpath,\n            },\n          );\n          await page.type(field.value);\n\n          // Build Action with XPath for deterministic replay (only when caching)\n          // Use originalValue (with %tokens%) so cache stores references, not sensitive values\n          if (shouldCollectXpath) {\n            const normalizedXpath = ensureXPath(xpath);\n            if (normalizedXpath) {\n              actions.push({\n                selector: normalizedXpath,\n                description: field.action,\n                method: \"type\",\n                arguments: [field.originalValue],\n              });\n            }\n          }\n\n          // Small delay between fields\n          await new Promise((resolve) => setTimeout(resolve, 100));\n        }\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n        // Record as \"act\" step with proper Actions for deterministic replay (only when caching)\n        if (shouldCollectXpath && actions.length > 0) {\n          v3.recordAgentReplayStep({\n            type: \"act\",\n            instruction: `Fill ${fields.length} form fields`,\n            actions,\n            actionDescription: `Fill ${fields.length} form fields`,\n          });\n        }\n\n        return {\n          success: true,\n          playwrightArguments: processedFields,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error filling form: ${error.message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [\n            {\n              type: \"text\",\n              text: JSON.stringify({\n                success: result.success,\n                error: result.error,\n              }),\n            },\n          ],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            fieldsCount: result.playwrightArguments?.length ?? 0,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n};\n"]}

package/dist/cjs/lib/v3/agent/tools/fillform.d.ts

@@ -1,13 +1,18 @@
 import type { V3 } from "../../v3.js";
 import type { Action } from "../../types/public/methods.js";
 import type { AgentModelConfig, Variables } from "../../types/public/agent.js";
-export declare const fillFormTool: (v3: V3, executionModel?: string | AgentModelConfig, variables?: Variables) => import("ai").Tool<{
+export declare const fillFormTool: (v3: V3, executionModel?: string | AgentModelConfig, variables?: Variables, toolTimeout?: number) => import("ai").Tool<{
     fields: {
         action: string;
-        value: string;
     }[];
 }, {
     success: boolean;
     actions: unknown[];
     playwrightArguments: Action[];
+    error?: undefined;
+} | {
+    success: boolean;
+    error: any;
+    actions?: undefined;
+    playwrightArguments?: undefined;
 }>;