npm - @browserbasehq/orca - Versions diffs - 3.1.0-patch.4 → 3.2.0-preview.2 - Mend

@browserbasehq/orca 3.1.0-patch.4 → 3.2.0-preview.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (297) hide show

package/dist/esm/lib/v3/agent/tools/browserbaseSearch.js ADDED Viewed

@@ -0,0 +1,66 @@
+import { tool } from "ai";
+import { z } from "zod";
+async function performBrowserbaseSearch(v3, query, apiKey, numResults = 5) {
+    try {
+        const response = await fetch("https://api.browserbase.com/v1/search", {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "x-bb-api-key": apiKey,
+            },
+            body: JSON.stringify({ query, numResults }),
+        });
+        if (!response.ok) {
+            return {
+                results: [],
+                error: `Browserbase Search API error: ${response.status} ${response.statusText}`,
+            };
+        }
+        const data = (await response.json());
+        const results = (data?.results ?? []).map(({ title, url, publishedDate }) => ({
+            title: title,
+            url: url,
+            ...(publishedDate && { publishedDate }),
+        }));
+        return { results };
+    }
+    catch (error) {
+        v3.logger({
+            category: "agent",
+            message: `Search error: ${error.message}`,
+            level: 0,
+        });
+        return {
+            results: [],
+            error: `Error performing search: ${error.message}`,
+        };
+    }
+}
+export const searchTool = (v3, apiKey) => tool({
+    description: "Perform a web search and returns results. Use this tool when you need information from the web or when you are unsure of the exact URL you want to navigate to. This can be used to find the ideal entry point, resulting in a task that is easier to complete due to starting further in the process.",
+    inputSchema: z.object({
+        query: z.string().describe("The search query to look for on the web"),
+    }),
+    execute: async ({ query }) => {
+        v3.logger({
+            category: "agent",
+            message: `Agent calling tool: search`,
+            level: 1,
+            auxiliary: {
+                arguments: {
+                    value: JSON.stringify({ query }),
+                    type: "object",
+                },
+            },
+        });
+        const result = await performBrowserbaseSearch(v3, query, apiKey);
+        v3.recordAgentReplayStep({
+            type: "search",
+            instruction: query,
+            playwrightArguments: { query },
+            message: result.error ?? `Found ${result.results.length} results`,
+        });
+        return { ...result, timestamp: Date.now() };
+    },
+});
+//# sourceMappingURL=browserbaseSearch.js.map

package/dist/esm/lib/v3/agent/tools/browserbaseSearch.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"browserbaseSearch.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/browserbaseSearch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAmBxB,KAAK,UAAU,wBAAwB,CACrC,EAAM,EACN,KAAa,EACb,MAAc,EACd,aAAqB,CAAC;IAEtB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,uCAAuC,EAAE;YACpE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,cAAc,EAAE,MAAM;aACvB;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;SAC5C,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,OAAO;gBACL,OAAO,EAAE,EAAE;gBACX,KAAK,EAAE,iCAAiC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE;aACjF,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAA2B,CAAC;QAC/D,MAAM,OAAO,GAAmB,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CACvD,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAC;YAClC,KAAK,EAAE,KAAK;YACZ,GAAG,EAAE,GAAG;YACR,GAAG,CAAC,aAAa,IAAI,EAAE,aAAa,EAAE,CAAC;SACxC,CAAC,CACH,CAAC;QAEF,OAAO,EAAE,OAAO,EAAE,CAAC;IACrB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,iBAAkB,KAAe,CAAC,OAAO,EAAE;YACpD,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,OAAO;YACL,OAAO,EAAE,EAAE;YACX,KAAK,EAAE,4BAA6B,KAAe,CAAC,OAAO,EAAE;SAC9D,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,EAAM,EAAE,MAAc,EAAE,EAAE,CACnD,IAAI,CAAC;IACH,WAAW,EACT,wSAAwS;IAC1S,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;KACtE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;QAC3B,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC;oBAChC,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,wBAAwB,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QAEjE,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,KAAK;YAClB,mBAAmB,EAAE,EAAE,KAAK,EAAE;YAC9B,OAAO,EAAE,MAAM,CAAC,KAAK,IAAI,SAAS,MAAM,CAAC,OAAO,CAAC,MAAM,UAAU;SAClE,CAAC,CAAC;QAEH,OAAO,EAAE,GAAG,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAC9C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport interface SearchResult {\n title: string;\n url: string;\n publishedDate?: string;\n}\n\ninterface BrowserbaseRawResult {\n title?: string;\n url?: string;\n publishedDate?: string;\n}\n\ninterface BrowserbaseApiResponse {\n results?: BrowserbaseRawResult[];\n}\n\nasync function performBrowserbaseSearch(\n v3: V3,\n query: string,\n apiKey: string,\n numResults: number = 5,\n): Promise<{ results: SearchResult[]; error?: string }> {\n try {\n const response = await fetch(\"https://api.browserbase.com/v1/search\", {\n method: \"POST\",\n headers: {\n \"Content-Type\": \"application/json\",\n \"x-bb-api-key\": apiKey,\n },\n body: JSON.stringify({ query, numResults }),\n });\n\n if (!response.ok) {\n return {\n results: [],\n error: `Browserbase Search API error: ${response.status} ${response.statusText}`,\n };\n }\n\n const data = (await response.json()) as BrowserbaseApiResponse;\n const results: SearchResult[] = (data?.results ?? []).map(\n ({ title, url, publishedDate }) => ({\n title: title,\n url: url,\n ...(publishedDate && { publishedDate }),\n }),\n );\n\n return { results };\n } catch (error) {\n v3.logger({\n category: \"agent\",\n message: `Search error: ${(error as Error).message}`,\n level: 0,\n });\n return {\n results: [],\n error: `Error performing search: ${(error as Error).message}`,\n };\n }\n}\n\nexport const searchTool = (v3: V3, apiKey: string) =>\n tool({\n description:\n \"Perform a web search and returns results. Use this tool when you need information from the web or when you are unsure of the exact URL you want to navigate to. This can be used to find the ideal entry point, resulting in a task that is easier to complete due to starting further in the process.\",\n inputSchema: z.object({\n query: z.string().describe(\"The search query to look for on the web\"),\n }),\n execute: async ({ query }) => {\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: search`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ query }),\n type: \"object\",\n },\n },\n });\n\n const result = await performBrowserbaseSearch(v3, query, apiKey);\n\n v3.recordAgentReplayStep({\n type: \"search\",\n instruction: query,\n playwrightArguments: { query },\n message: result.error ?? `Found ${result.results.length} results`,\n });\n\n return { ...result, timestamp: Date.now() };\n },\n });\n"]}

package/dist/esm/lib/v3/agent/tools/click.js CHANGED Viewed

@@ -67,38 +67,30 @@ export const clickTool = (v3, provider) => tool({
         }
     },
     toModelOutput: (result) => {
-        if (result.success) {
-            const content = [
-                {
-                    type: "text",
-                    text: JSON.stringify({
-                        success: result.success,
-                        describe: result.describe,
-                        coordinates: result.coordinates,
-                    }),
-                },
-            ];
-            if (result.screenshotBase64) {
-                content.push({
-                    type: "media",
-                    mediaType: "image/png",
-                    data: result.screenshotBase64,
-                });
-            }
-            return { type: "content", value: content };
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
         }
-        return {
-            type: "content",
-            value: [
-                {
-                    type: "text",
-                    text: JSON.stringify({
-                        success: result.success,
-                        error: result.error,
-                    }),
-                },
-            ],
-        };
+        const content = [
+            {
+                type: "text",
+                text: JSON.stringify({
+                    success: result.success,
+                    describe: result.describe,
+                    coordinates: result.coordinates,
+                }),
+            },
+        ];
+        if (result.screenshotBase64) {
+            content.push({
+                type: "media",
+                mediaType: "image/png",
+                data: result.screenshotBase64,
+            });
+        }
+        return { type: "content", value: content };
     },
 });
 //# sourceMappingURL=click.js.map

package/dist/esm/lib/v3/agent/tools/click.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"click.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/click.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CACrD,IAAI,CAAC;IACH,WAAW,EACT,wMAAwM;IAC1M,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAA4B,EAAE;QACrE,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,2BAA2B;gBACpC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;wBACnC,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;gBACvD,WAAW,EAAE,kBAAkB;aAChC,CAAC,CAAC;YAEH,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE9D,0FAA0F;YAC1F,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,OAAO;wBACf,SAAS,EAAE,EAAE;qBACd,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;gBACvC,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAoB,KAAe,CAAC,OAAO,EAAE;aACrD,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;~~YACnB~~,MAAM,OAAO,GAA6B;~~gBACxC~~;~~oBACE~~,IAAI,EAAE,MAAM;~~oBACZ~~,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;~~wBACnB~~,OAAO,EAAE,MAAM,CAAC,OAAO;~~wBACvB~~,QAAQ,EAAE,MAAM,CAAC,QAAQ;~~wBACzB~~,WAAW,EAAE,MAAM,CAAC,WAAW;~~qBAChC~~,CAAC;~~iBACH~~;~~aACF~~,CAAC;~~YACF~~,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;~~gBAC5B~~,OAAO,CAAC,IAAI,CAAC;~~oBACX~~,IAAI,EAAE,OAAO;~~oBACb~~,SAAS,EAAE,WAAW;~~oBACtB~~,IAAI,EAAE,MAAM,CAAC,gBAAgB;~~iBAC9B~~,CAAC,CAAC;~~YACL~~,CAAC;~~YACD~~,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;~~QAC7C~~,CAAC;QACD,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,KAAK,EAAE,MAAM,CAAC,KAAK;qBACpB,CAAC;iBACH;aACF;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n ClickToolResult,\n ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const clickTool = (v3: V3, provider?: string) =>\n tool({\n description:\n \"Click on an element using its coordinates (this is the most reliable way to click on an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n inputSchema: z.object({\n describe: z\n .string()\n .describe(\n \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n ),\n coordinates: z\n .array(z.number())\n .describe(\"The (x, y) coordinates to click on\"),\n }),\n execute: async ({ describe, coordinates }): Promise<ClickToolResult> => {\n try {\n const page = await v3.context.awaitActivePage();\n const processed = processCoordinates(\n coordinates[0],\n coordinates[1],\n provider,\n v3,\n );\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: click`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ describe }),\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n const xpath = await page.click(processed.x, processed.y, {\n returnXpath: shouldCollectXpath,\n });\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n if (shouldCollectXpath) {\n const normalizedXpath = ensureXPath(xpath);\n if (normalizedXpath) {\n const action: Action = {\n selector: normalizedXpath,\n description: describe,\n method: \"click\",\n arguments: [],\n };\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: describe,\n actions: [action],\n actionDescription: describe,\n });\n }\n }\n\n return {\n success: true,\n describe,\n coordinates: [processed.x, processed.y],\n screenshotBase64,\n };\n } catch (error) {\n return {\n success: false,\n error: `Error clicking: ${(error as Error).message}`,\n };\n }\n },\n toModelOutput: (result) => {\n if (result.success) {\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n describe: result.describe,\n coordinates: result.coordinates,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n }\n return {\n type: \"content\",\n value: [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n error: result.error,\n }),\n },\n ],\n };\n },\n });\n"]}
1	+ {"version":3,"file":"click.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/click.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CACrD,IAAI,CAAC;IACH,WAAW,EACT,wMAAwM;IAC1M,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,2HAA2H,CAC5H;QACH,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oCAAoC,CAAC;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,EAA4B,EAAE;QACrE,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,2BAA2B;gBACpC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;wBACnC,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;gBACvD,WAAW,EAAE,kBAAkB;aAChC,CAAC,CAAC;YAEH,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE9D,0FAA0F;YAC1F,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,OAAO;wBACf,SAAS,EAAE,EAAE;qBACd,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,WAAW,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;gBACvC,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAoB,KAAe,CAAC,OAAO,EAAE;aACrD,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,WAAW,EAAE,MAAM,CAAC,WAAW;iBAChC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n ClickToolResult,\n ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const clickTool = (v3: V3, provider?: string) =>\n tool({\n description:\n \"Click on an element using its coordinates (this is the most reliable way to click on an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n inputSchema: z.object({\n describe: z\n .string()\n .describe(\n \"Describe the element to click on in a short, specific phrase that mentions the element type and a good visual description\",\n ),\n coordinates: z\n .array(z.number())\n .describe(\"The (x, y) coordinates to click on\"),\n }),\n execute: async ({ describe, coordinates }): Promise<ClickToolResult> => {\n try {\n const page = await v3.context.awaitActivePage();\n const processed = processCoordinates(\n coordinates[0],\n coordinates[1],\n provider,\n v3,\n );\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: click`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ describe }),\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n const xpath = await page.click(processed.x, processed.y, {\n returnXpath: shouldCollectXpath,\n });\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n if (shouldCollectXpath) {\n const normalizedXpath = ensureXPath(xpath);\n if (normalizedXpath) {\n const action: Action = {\n selector: normalizedXpath,\n description: describe,\n method: \"click\",\n arguments: [],\n };\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: describe,\n actions: [action],\n actionDescription: describe,\n });\n }\n }\n\n return {\n success: true,\n describe,\n coordinates: [processed.x, processed.y],\n screenshotBase64,\n };\n } catch (error) {\n return {\n success: false,\n error: `Error clicking: ${(error as Error).message}`,\n };\n }\n },\n toModelOutput: (result) => {\n if (result.success === false \|\| result.error !== undefined) {\n return {\n type: \"content\",\n value: [{ type: \"text\", text: JSON.stringify(result) }],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n describe: result.describe,\n coordinates: result.coordinates,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n"]}

package/dist/esm/lib/v3/agent/tools/dragAndDrop.js CHANGED Viewed

@@ -69,37 +69,29 @@ export const dragAndDropTool = (v3, provider) => tool({
         }
     },
     toModelOutput: (result) => {
-        if (result.success) {
-            const content = [
-                {
-                    type: "text",
-                    text: JSON.stringify({
-                        success: result.success,
-                        describe: result.describe,
-                    }),
-                },
-            ];
-            if (result.screenshotBase64) {
-                content.push({
-                    type: "media",
-                    mediaType: "image/png",
-                    data: result.screenshotBase64,
-                });
-            }
-            return { type: "content", value: content };
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
         }
-        return {
-            type: "content",
-            value: [
-                {
-                    type: "text",
-                    text: JSON.stringify({
-                        success: result.success,
-                        error: result.error,
-                    }),
-                },
-            ],
-        };
+        const content = [
+            {
+                type: "text",
+                text: JSON.stringify({
+                    success: result.success,
+                    describe: result.describe,
+                }),
+            },
+        ];
+        if (result.screenshotBase64) {
+            content.push({
+                type: "media",
+                mediaType: "image/png",
+                data: result.screenshotBase64,
+            });
+        }
+        return { type: "content", value: content };
     },
 });
 //# sourceMappingURL=dragAndDrop.js.map

package/dist/esm/lib/v3/agent/tools/dragAndDrop.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"dragAndDrop.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/dragAndDrop.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC3D,IAAI,CAAC;IACH,WAAW,EACT,kNAAkN;IACpN,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uCAAuC,CAAC;QACtE,gBAAgB,EAAE,CAAC;aAChB,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,wDAAwD,CAAC;QACrE,cAAc,EAAE,CAAC;aACd,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oDAAoD,CAAC;KAClE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,gBAAgB,EAChB,cAAc,GACf,EAAkC,EAAE;QACnC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,cAAc,GAAG,kBAAkB,CACvC,gBAAgB,CAAC,CAAC,CAAC,EACnB,gBAAgB,CAAC,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;YACF,MAAM,YAAY,GAAG,kBAAkB,CACrC,cAAc,CAAC,CAAC,CAAC,EACjB,cAAc,CAAC,CAAC,CAAC,EACjB,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,iCAAiC;gBAC1C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACjD,cAAc,CAAC,CAAC,EAChB,cAAc,CAAC,CAAC,EAChB,YAAY,CAAC,CAAC,EACd,YAAY,CAAC,CAAC,EACd,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACpC,CAAC;YAEF,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE9D,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,cAAc,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;gBAC9C,MAAM,YAAY,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;gBAC1C,IAAI,cAAc,IAAI,YAAY,EAAE,CAAC;oBACnC,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,cAAc;wBACxB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,aAAa;wBACrB,SAAS,EAAE,CAAC,YAAY,CAAC;qBAC1B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAoB,KAAe,CAAC,OAAO,EAAE;aACrD,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;~~YACnB~~,MAAM,OAAO,GAA6B;~~gBACxC~~;~~oBACE~~,IAAI,EAAE,MAAM;~~oBACZ~~,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;~~wBACnB~~,OAAO,EAAE,MAAM,CAAC,OAAO;~~wBACvB~~,QAAQ,EAAE,MAAM,CAAC,QAAQ;~~qBAC1B~~,CAAC;~~iBACH~~;~~aACF~~,CAAC;~~YACF~~,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;~~gBAC5B~~,OAAO,CAAC,IAAI,CAAC;~~oBACX~~,IAAI,EAAE,OAAO;~~oBACb~~,SAAS,EAAE,WAAW;~~oBACtB~~,IAAI,EAAE,MAAM,CAAC,gBAAgB;~~iBAC9B~~,CAAC,CAAC;~~YACL~~,CAAC;~~YACD~~,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;~~QAC7C~~,CAAC;QACD,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,KAAK,EAAE,MAAM,CAAC,KAAK;qBACpB,CAAC;iBACH;aACF;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n DragAndDropToolResult,\n ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const dragAndDropTool = (v3: V3, provider?: string) =>\n tool({\n description:\n \"Drag and drop an element using its coordinates (this is the most reliable way to drag and drop an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n inputSchema: z.object({\n describe: z.string().describe(\"Describe the element to drag and drop\"),\n startCoordinates: z\n .array(z.number())\n .describe(\"The (x, y) coordinates to start the drag and drop from\"),\n endCoordinates: z\n .array(z.number())\n .describe(\"The (x, y) coordinates to end the drag and drop at\"),\n }),\n execute: async ({\n describe,\n startCoordinates,\n endCoordinates,\n }): Promise<DragAndDropToolResult> => {\n try {\n const page = await v3.context.awaitActivePage();\n const processedStart = processCoordinates(\n startCoordinates[0],\n startCoordinates[1],\n provider,\n v3,\n );\n const processedEnd = processCoordinates(\n endCoordinates[0],\n endCoordinates[1],\n provider,\n v3,\n );\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: dragAndDrop`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({\n describe,\n }),\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n const [fromXpath, toXpath] = await page.dragAndDrop(\n processedStart.x,\n processedStart.y,\n processedEnd.x,\n processedEnd.y,\n { returnXpath: shouldCollectXpath },\n );\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n if (shouldCollectXpath) {\n const normalizedFrom = ensureXPath(fromXpath);\n const normalizedTo = ensureXPath(toXpath);\n if (normalizedFrom && normalizedTo) {\n const action: Action = {\n selector: normalizedFrom,\n description: describe,\n method: \"dragAndDrop\",\n arguments: [normalizedTo],\n };\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: describe,\n actions: [action],\n actionDescription: describe,\n });\n }\n }\n\n return {\n success: true,\n describe,\n screenshotBase64,\n };\n } catch (error) {\n return {\n success: false,\n error: `Error dragging: ${(error as Error).message}`,\n };\n }\n },\n toModelOutput: (result) => {\n if (result.success) {\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n describe: result.describe,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n }\n return {\n type: \"content\",\n value: [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n error: result.error,\n }),\n },\n ],\n };\n },\n });\n"]}
1	+ {"version":3,"file":"dragAndDrop.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/dragAndDrop.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC3D,IAAI,CAAC;IACH,WAAW,EACT,kNAAkN;IACpN,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uCAAuC,CAAC;QACtE,gBAAgB,EAAE,CAAC;aAChB,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,wDAAwD,CAAC;QACrE,cAAc,EAAE,CAAC;aACd,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,oDAAoD,CAAC;KAClE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,gBAAgB,EAChB,cAAc,GACf,EAAkC,EAAE;QACnC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,cAAc,GAAG,kBAAkB,CACvC,gBAAgB,CAAC,CAAC,CAAC,EACnB,gBAAgB,CAAC,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;YACF,MAAM,YAAY,GAAG,kBAAkB,CACrC,cAAc,CAAC,CAAC,CAAC,EACjB,cAAc,CAAC,CAAC,CAAC,EACjB,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,iCAAiC;gBAC1C,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;4BACpB,QAAQ;yBACT,CAAC;wBACF,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CACjD,cAAc,CAAC,CAAC,EAChB,cAAc,CAAC,CAAC,EAChB,YAAY,CAAC,CAAC,EACd,YAAY,CAAC,CAAC,EACd,EAAE,WAAW,EAAE,kBAAkB,EAAE,CACpC,CAAC;YAEF,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE9D,uFAAuF;YACvF,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,cAAc,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;gBAC9C,MAAM,YAAY,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;gBAC1C,IAAI,cAAc,IAAI,YAAY,EAAE,CAAC;oBACnC,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,cAAc;wBACxB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,aAAa;wBACrB,SAAS,EAAE,CAAC,YAAY,CAAC;qBAC1B,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,mBAAoB,KAAe,CAAC,OAAO,EAAE;aACrD,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;iBAC1B,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n DragAndDropToolResult,\n ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const dragAndDropTool = (v3: V3, provider?: string) =>\n tool({\n description:\n \"Drag and drop an element using its coordinates (this is the most reliable way to drag and drop an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n inputSchema: z.object({\n describe: z.string().describe(\"Describe the element to drag and drop\"),\n startCoordinates: z\n .array(z.number())\n .describe(\"The (x, y) coordinates to start the drag and drop from\"),\n endCoordinates: z\n .array(z.number())\n .describe(\"The (x, y) coordinates to end the drag and drop at\"),\n }),\n execute: async ({\n describe,\n startCoordinates,\n endCoordinates,\n }): Promise<DragAndDropToolResult> => {\n try {\n const page = await v3.context.awaitActivePage();\n const processedStart = processCoordinates(\n startCoordinates[0],\n startCoordinates[1],\n provider,\n v3,\n );\n const processedEnd = processCoordinates(\n endCoordinates[0],\n endCoordinates[1],\n provider,\n v3,\n );\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: dragAndDrop`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({\n describe,\n }),\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n const [fromXpath, toXpath] = await page.dragAndDrop(\n processedStart.x,\n processedStart.y,\n processedEnd.x,\n processedEnd.y,\n { returnXpath: shouldCollectXpath },\n );\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n // Record as \"act\" step with proper Action for deterministic replay (only when caching)\n if (shouldCollectXpath) {\n const normalizedFrom = ensureXPath(fromXpath);\n const normalizedTo = ensureXPath(toXpath);\n if (normalizedFrom && normalizedTo) {\n const action: Action = {\n selector: normalizedFrom,\n description: describe,\n method: \"dragAndDrop\",\n arguments: [normalizedTo],\n };\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: describe,\n actions: [action],\n actionDescription: describe,\n });\n }\n }\n\n return {\n success: true,\n describe,\n screenshotBase64,\n };\n } catch (error) {\n return {\n success: false,\n error: `Error dragging: ${(error as Error).message}`,\n };\n }\n },\n toModelOutput: (result) => {\n if (result.success === false \|\| result.error !== undefined) {\n return {\n type: \"content\",\n value: [{ type: \"text\", text: JSON.stringify(result) }],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n describe: result.describe,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n"]}

package/dist/esm/lib/v3/agent/tools/extract.d.ts CHANGED Viewed

@@ -1,6 +1,6 @@
 import type { V3 } from "../../v3.js";
 import type { AgentModelConfig } from "../../types/public/agent.js";
-export declare const extractTool: (v3: V3, executionModel?: string | AgentModelConfig) => import("ai").Tool<{
+export declare const extractTool: (v3: V3, executionModel?: string | AgentModelConfig, toolTimeout?: number) => import("ai").Tool<{
     instruction: string;
     schema?: {
         [x: string]: unknown;
@@ -16,6 +16,6 @@ export declare const extractTool: (v3: V3, executionModel?: string | AgentModelC
     error?: undefined;
 } | {
     success: boolean;
-    error: string;
+    error: any;
     result?: undefined;
 }>;

package/dist/esm/lib/v3/agent/tools/extract.js CHANGED Viewed

@@ -1,5 +1,6 @@
 import { tool } from "ai";
 import { z } from "zod";
+import { TimeoutError } from "../../types/public/sdkErrors.js";
 function jsonSchemaToZod(schema) {
     switch (schema.type) {
         case "object": {
@@ -36,7 +37,7 @@ function jsonSchemaToZod(schema) {
             return z.any();
     }
 }
-export const extractTool = (v3, executionModel) => tool({
+export const extractTool = (v3, executionModel, toolTimeout) => tool({
     description: `Extract structured data from the current page based on a provided schema.
     USAGE GUIDELINES:
@@ -81,12 +82,24 @@ export const extractTool = (v3, executionModel) => tool({
                 : undefined;
             const result = await v3.extract(instruction, parsedSchema, {
                 ...(executionModel ? { model: executionModel } : {}),
+                timeout: toolTimeout,
             });
             return { success: true, result };
         }
         catch (error) {
-            const err = error;
-            return { success: false, error: err?.message ?? String(error) };
+            if (error instanceof TimeoutError) {
+                const timeoutMessage = `TimeoutError: extract() timed out — try using a smaller or simpler schema`;
+                v3.logger({
+                    category: "agent",
+                    message: timeoutMessage,
+                    level: 0,
+                });
+                return {
+                    success: false,
+                    error: timeoutMessage,
+                };
+            }
+            return { success: false, error: error?.message ?? String(error) };
         }
     },
 });

package/dist/esm/lib/v3/agent/tools/extract.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"extract.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAc,MAAM,KAAK,CAAC;~~AAYpC~~,SAAS,eAAe,CAAC,MAAkB;IACzC,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,KAAK,GAA+B,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;gBACtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC7D,KAAK,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;gBACtC,CAAC;YACH,CAAC;YACD,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;QACD,KAAK,OAAO;YACV,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QACzE,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;YACnB,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK;gBAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO;gBAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM;gBAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAC3C,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;gBACvC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAA6B,CAAC,CAAC;YACtD,OAAO,CAAC,CAAC;QACX,CAAC;QACD,KAAK,QAAQ,CAAC;QACd,KAAK,SAAS;YACZ,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;QACpB,KAAK,SAAS;YACZ,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;QACrB,KAAK,MAAM;YACT,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAClB;YACE,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,EAAM,EACN,cAA0C,EAC1C,EAAE,CACF,IAAI,CAAC;IACH,WAAW,EAAE;;;;;;;;;;;;;;;;;;;;;;0FAsByE;IACtF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;QACvB,MAAM,EAAE,CAAC;aACN,MAAM,CAAC;YACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC3B,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;YACxD,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;YAC7B,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;YACpC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;SACpD,CAAC;aACD,WAAW,EAAE;aACb,QAAQ,EAAE;aACV,QAAQ,CAAC,wDAAwD,CAAC;KACtE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE;QACzC,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,MAAM;gBACzB,CAAC,CAAC,eAAe,CAAC,MAAoB,CAAC;gBACvC,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,YAAY,EAAE;gBACzD,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;~~aACrD~~,CAAC,CAAC;YACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,~~GAAG~~,GAAG,~~KAAc~~,CAAC;~~YAC3B~~,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,~~GAAG~~,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;~~QAClE~~,CAAC;IACH,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z, ZodTypeAny } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { AgentModelConfig } from \"../../types/public/agent.js\";\n\ninterface JsonSchema {\n type?: string;\n properties?: Record<string, JsonSchema>;\n items?: JsonSchema;\n enum?: string[];\n format?: \"url\" \| \"email\" \| \"uuid\";\n}\n\nfunction jsonSchemaToZod(schema: JsonSchema): ZodTypeAny {\n switch (schema.type) {\n case \"object\": {\n const shape: Record<string, ZodTypeAny> = {};\n if (schema.properties) {\n for (const [key, value] of Object.entries(schema.properties)) {\n shape[key] = jsonSchemaToZod(value);\n }\n }\n return z.object(shape);\n }\n case \"array\":\n return z.array(schema.items ? jsonSchemaToZod(schema.items) : z.any());\n case \"string\": {\n let s = z.string();\n if (schema.format === \"url\") s = s.url();\n if (schema.format === \"email\") s = s.email();\n if (schema.format === \"uuid\") s = s.uuid();\n if (schema.enum && schema.enum.length > 0)\n return z.enum(schema.enum as [string, ...string[]]);\n return s;\n }\n case \"number\":\n case \"integer\":\n return z.number();\n case \"boolean\":\n return z.boolean();\n case \"null\":\n return z.null();\n default:\n return z.any();\n }\n}\n\nexport const extractTool = (\n v3: V3,\n executionModel?: string \| AgentModelConfig,\n) =>\n tool({\n description: `Extract structured data from the current page based on a provided schema.\n \n USAGE GUIDELINES:\n - Keep schemas MINIMAL - only include fields essential for the task\n - IMPORTANT: only use this if explicitly asked for structured output. In most scenarios, you should use the aria tree tool over this.\n - For URL fields, use format: \"url\"\n \n EXAMPLES:\n 1. Extract a single value:\n instruction: \"extract the product price\"\n schema: { type: \"object\", properties: { price: { type: \"number\" } } }\n \n 2. Extract multiple fields:\n instruction: \"extract product name and price\"\n schema: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } }\n \n 3. Extract arrays:\n instruction: \"extract all product names and prices\"\n schema: { type: \"object\", properties: { products: { type: \"array\", items: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } } } } }\n \n 4. Extract a URL:\n instruction: \"extract the link\"\n schema: { type: \"object\", properties: { url: { type: \"string\", format: \"url\" } } }`,\n inputSchema: z.object({\n instruction: z.string(),\n schema: z\n .object({\n type: z.string().optional(),\n properties: z.record(z.string(), z.unknown()).optional(),\n items: z.unknown().optional(),\n enum: z.array(z.string()).optional(),\n format: z.enum([\"url\", \"email\", \"uuid\"]).optional(),\n })\n .passthrough()\n .optional()\n .describe(\"JSON Schema object describing the structure to extract\"),\n }),\n execute: async ({ instruction, schema }) => {\n try {\n const parsedSchema = schema\n ? jsonSchemaToZod(schema as JsonSchema)\n : undefined;\n const result = await v3.extract(instruction, parsedSchema, {\n ...(executionModel ? { model: executionModel } : {}),\n });\n return { success: true, result };\n } catch (error) {\n const ~~err~~ = ~~error~~ as ~~Error~~;\n return { success: false, error: ~~err~~?.message ?? String(error) };\n }\n },\n });\n"]}
1	+ {"version":3,"file":"extract.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAc,MAAM,KAAK,CAAC;AAGpC,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAU/D,SAAS,eAAe,CAAC,MAAkB;IACzC,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,KAAK,GAA+B,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;gBACtB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC7D,KAAK,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;gBACtC,CAAC;YACH,CAAC;YACD,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;QACD,KAAK,OAAO;YACV,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QACzE,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;YACnB,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK;gBAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO;gBAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;YAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM;gBAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAC3C,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;gBACvC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAA6B,CAAC,CAAC;YACtD,OAAO,CAAC,CAAC;QACX,CAAC;QACD,KAAK,QAAQ,CAAC;QACd,KAAK,SAAS;YACZ,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;QACpB,KAAK,SAAS;YACZ,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;QACrB,KAAK,MAAM;YACT,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAClB;YACE,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,EAAM,EACN,cAA0C,EAC1C,WAAoB,EACpB,EAAE,CACF,IAAI,CAAC;IACH,WAAW,EAAE;;;;;;;;;;;;;;;;;;;;;;0FAsByE;IACtF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;QACvB,MAAM,EAAE,CAAC;aACN,MAAM,CAAC;YACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC3B,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;YACxD,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;YAC7B,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;YACpC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;SACpD,CAAC;aACD,WAAW,EAAE;aACb,QAAQ,EAAE;aACV,QAAQ,CAAC,wDAAwD,CAAC;KACtE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE;QACzC,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,MAAM;gBACzB,CAAC,CAAC,eAAe,CAAC,MAAoB,CAAC;gBACvC,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,YAAY,EAAE;gBACzD,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpD,OAAO,EAAE,WAAW;aACrB,CAAC,CAAC;YACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QACnC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBAClC,MAAM,cAAc,GAAG,2EAA2E,CAAC;gBACnG,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,cAAc;oBACvB,KAAK,EAAE,CAAC;iBACT,CAAC,CAAC;gBACH,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,cAAc;iBACtB,CAAC;YACJ,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QACpE,CAAC;IACH,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z, ZodTypeAny } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { AgentModelConfig } from \"../../types/public/agent.js\";\nimport { TimeoutError } from \"../../types/public/sdkErrors.js\";\n\ninterface JsonSchema {\n type?: string;\n properties?: Record<string, JsonSchema>;\n items?: JsonSchema;\n enum?: string[];\n format?: \"url\" \| \"email\" \| \"uuid\";\n}\n\nfunction jsonSchemaToZod(schema: JsonSchema): ZodTypeAny {\n switch (schema.type) {\n case \"object\": {\n const shape: Record<string, ZodTypeAny> = {};\n if (schema.properties) {\n for (const [key, value] of Object.entries(schema.properties)) {\n shape[key] = jsonSchemaToZod(value);\n }\n }\n return z.object(shape);\n }\n case \"array\":\n return z.array(schema.items ? jsonSchemaToZod(schema.items) : z.any());\n case \"string\": {\n let s = z.string();\n if (schema.format === \"url\") s = s.url();\n if (schema.format === \"email\") s = s.email();\n if (schema.format === \"uuid\") s = s.uuid();\n if (schema.enum && schema.enum.length > 0)\n return z.enum(schema.enum as [string, ...string[]]);\n return s;\n }\n case \"number\":\n case \"integer\":\n return z.number();\n case \"boolean\":\n return z.boolean();\n case \"null\":\n return z.null();\n default:\n return z.any();\n }\n}\n\nexport const extractTool = (\n v3: V3,\n executionModel?: string \| AgentModelConfig,\n toolTimeout?: number,\n) =>\n tool({\n description: `Extract structured data from the current page based on a provided schema.\n \n USAGE GUIDELINES:\n - Keep schemas MINIMAL - only include fields essential for the task\n - IMPORTANT: only use this if explicitly asked for structured output. In most scenarios, you should use the aria tree tool over this.\n - For URL fields, use format: \"url\"\n \n EXAMPLES:\n 1. Extract a single value:\n instruction: \"extract the product price\"\n schema: { type: \"object\", properties: { price: { type: \"number\" } } }\n \n 2. Extract multiple fields:\n instruction: \"extract product name and price\"\n schema: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } }\n \n 3. Extract arrays:\n instruction: \"extract all product names and prices\"\n schema: { type: \"object\", properties: { products: { type: \"array\", items: { type: \"object\", properties: { name: { type: \"string\" }, price: { type: \"number\" } } } } } }\n \n 4. Extract a URL:\n instruction: \"extract the link\"\n schema: { type: \"object\", properties: { url: { type: \"string\", format: \"url\" } } }`,\n inputSchema: z.object({\n instruction: z.string(),\n schema: z\n .object({\n type: z.string().optional(),\n properties: z.record(z.string(), z.unknown()).optional(),\n items: z.unknown().optional(),\n enum: z.array(z.string()).optional(),\n format: z.enum([\"url\", \"email\", \"uuid\"]).optional(),\n })\n .passthrough()\n .optional()\n .describe(\"JSON Schema object describing the structure to extract\"),\n }),\n execute: async ({ instruction, schema }) => {\n try {\n const parsedSchema = schema\n ? jsonSchemaToZod(schema as JsonSchema)\n : undefined;\n const result = await v3.extract(instruction, parsedSchema, {\n ...(executionModel ? { model: executionModel } : {}),\n timeout: toolTimeout,\n });\n return { success: true, result };\n } catch (error) {\n if (error instanceof TimeoutError) {\n const timeoutMessage = `TimeoutError: extract() timed out — try using a smaller or simpler schema`;\n v3.logger({\n category: \"agent\",\n message: timeoutMessage,\n level: 0,\n });\n return {\n success: false,\n error: timeoutMessage,\n };\n }\n return { success: false, error: error?.message ?? String(error) };\n }\n },\n });\n"]}

package/dist/esm/lib/v3/agent/tools/fillFormVision.js CHANGED Viewed

@@ -117,37 +117,37 @@ MANDATORY USE CASES (always use fillFormVision for these):
             }
         },
         toModelOutput: (result) => {
-            if (result.success) {
-                const content = [
-                    {
-                        type: "text",
-                        text: JSON.stringify({
-                            success: result.success,
-                            fieldsCount: result.playwrightArguments?.length ?? 0,
-                        }),
-                    },
-                ];
-                if (result.screenshotBase64) {
-                    content.push({
-                        type: "media",
-                        mediaType: "image/png",
-                        data: result.screenshotBase64,
-                    });
-                }
-                return { type: "content", value: content };
+            if (result.success === false || result.error !== undefined) {
+                return {
+                    type: "content",
+                    value: [
+                        {
+                            type: "text",
+                            text: JSON.stringify({
+                                success: result.success,
+                                error: result.error,
+                            }),
+                        },
+                    ],
+                };
             }
-            return {
-                type: "content",
-                value: [
-                    {
-                        type: "text",
-                        text: JSON.stringify({
-                            success: result.success,
-                            error: result.error,
-                        }),
-                    },
-                ],
-            };
+            const content = [
+                {
+                    type: "text",
+                    text: JSON.stringify({
+                        success: result.success,
+                        fieldsCount: result.playwrightArguments?.length ?? 0,
+                    }),
+                },
+            ];
+            if (result.screenshotBase64) {
+                content.push({
+                    type: "media",
+                    mediaType: "image/png",
+                    data: result.screenshotBase64,
+                });
+            }
+            return { type: "content", value: content };
         },
     });
 };

package/dist/esm/lib/v3/agent/tools/fillFormVision.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"fillFormVision.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillFormVision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAChC,EAAM,EACN,QAAiB,EACjB,SAAqB,EACrB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,qGAAqG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAC1I,CAAC,CAAC,oCAAoC,CAAC;IAEzC,OAAO,IAAI,CAAC;QACV,WAAW,EAAE;;;;;;;;;;;;;;;;2CAgB0B;QACvC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,CAAC;iBACN,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,CAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,sEAAsE,CACvE;gBACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBAC5C,WAAW,EAAE,CAAC;qBACX,MAAM,CAAC;oBACN,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;oBACb,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;iBACd,CAAC;qBACD,QAAQ,CAAC,iCAAiC,CAAC;aAC/C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,qCAAqC,CAAC;SACjD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAqC,EAAE;YAC/D,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAEhD,8DAA8D;gBAC9D,0FAA0F;gBAC1F,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;oBAC3C,MAAM,SAAS,GAAG,kBAAkB,CAClC,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;oBACF,OAAO;wBACL,GAAG,KAAK;wBACR,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE,wCAAwC;wBACpE,KAAK,EAAE,mBAAmB,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC;wBAClD,WAAW,EAAE,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;qBAChD,CAAC;gBACJ,CAAC,CAAC,CAAC;gBAEH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,oCAAoC;oBAC7C,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,+BAA+B;4BAClE,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,OAAO,GAAa,EAAE,CAAC;gBAE7B,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;oBACpC,iEAAiE;oBACjE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAC5B,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB;wBACE,WAAW,EAAE,kBAAkB;qBAChC,CACF,CAAC;oBACF,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;oBAE7B,uEAAuE;oBACvE,qFAAqF;oBACrF,IAAI,kBAAkB,EAAE,CAAC;wBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;wBAC3C,IAAI,eAAe,EAAE,CAAC;4BACpB,OAAO,CAAC,IAAI,CAAC;gCACX,QAAQ,EAAE,eAAe;gCACzB,WAAW,EAAE,KAAK,CAAC,MAAM;gCACzB,MAAM,EAAE,MAAM;gCACd,SAAS,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC;6BACjC,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,6BAA6B;oBAC7B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;gBAC3D,CAAC;gBAED,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAEnE,wFAAwF;gBACxF,IAAI,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7C,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;wBAChD,OAAO;wBACP,iBAAiB,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;qBACvD,CAAC,CAAC;gBACL,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,mBAAmB,EAAE,eAAe;oBACpC,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,uBAAwB,KAAe,CAAC,OAAO,EAAE;iBACzD,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,MAAM,OAAO,GAA6B;oBACxC;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;4BACnB,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,WAAW,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,IAAI,CAAC;yBACrD,CAAC;qBACH;iBACF,CAAC;gBACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;oBAC5B,OAAO,CAAC,IAAI,CAAC;wBACX,IAAI,EAAE,OAAO;wBACb,SAAS,EAAE,WAAW;wBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;qBAC9B,CAAC,CAAC;gBACL,CAAC;gBACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;YAC7C,CAAC;YACD,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE;oBACL;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;4BACnB,OAAO,EAAE,MAAM,CAAC,OAAO;4BACvB,KAAK,EAAE,MAAM,CAAC,KAAK;yBACpB,CAAC;qBACH;iBACF;aACF,CAAC;QACJ,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n FillFormVisionToolResult,\n ModelOutputContentItem,\n Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const fillFormVisionTool = (\n v3: V3,\n provider?: string,\n variables?: Variables,\n) => {\n const hasVariables = variables && Object.keys(variables).length > 0;\n const valueDescription = hasVariables\n ? `Text to type into the target field. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n : \"Text to type into the target field\";\n\n return tool({\n description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL\n\nCRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)\nIMPORTANT: Ensure the fields are visible within the current viewport\n\nWHY THIS TOOL EXISTS:\n- Forms are the #1 use case for multi-field input\n- Optimized specifically for input/textarea elements\n- 4-6x faster than individual typing actions\n\nUse fillFormVision: Pure form filling (inputs, textareas only)\nMANDATORY USE CASES (always use fillFormVision for these):\n- Registration forms: name, email, password fields\n- Contact forms: name, email, message fields\n- Checkout forms: address, payment info fields\n- Profile updates: multiple user data fields\n- Search filters: multiple criteria inputs`,\n inputSchema: z.object({\n fields: z\n .array(\n z.object({\n action: z\n .string()\n .describe(\n \"Description of the typing action, e.g. 'type foo into the bar field'\",\n ),\n value: z.string().describe(valueDescription),\n coordinates: z\n .object({\n x: z.number(),\n y: z.number(),\n })\n .describe(\"Coordinates of the target field\"),\n }),\n )\n .min(2, \"Provide at least two fields to fill\"),\n }),\n execute: async ({ fields }): Promise<FillFormVisionToolResult> => {\n try {\n const page = await v3.context.awaitActivePage();\n\n // Process coordinates and substitute variables for each field\n // Keep original values (with %tokens%) for logging/caching, substituted values for typing\n const processedFields = fields.map((field) => {\n const processed = processCoordinates(\n field.coordinates.x,\n field.coordinates.y,\n provider,\n v3,\n );\n return {\n ...field,\n originalValue: field.value, // Keep original with %tokens% for cache\n value: substituteVariables(field.value, variables),\n coordinates: { x: processed.x, y: processed.y },\n };\n });\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: fillFormVision`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ fields }), // Don't log substituted values\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n const actions: Action[] = [];\n\n for (const field of processedFields) {\n // Click the field, only requesting XPath when caching is enabled\n const xpath = await page.click(\n field.coordinates.x,\n field.coordinates.y,\n {\n returnXpath: shouldCollectXpath,\n },\n );\n await page.type(field.value);\n\n // Build Action with XPath for deterministic replay (only when caching)\n // Use originalValue (with %tokens%) so cache stores references, not sensitive values\n if (shouldCollectXpath) {\n const normalizedXpath = ensureXPath(xpath);\n if (normalizedXpath) {\n actions.push({\n selector: normalizedXpath,\n description: field.action,\n method: \"type\",\n arguments: [field.originalValue],\n });\n }\n }\n\n // Small delay between fields\n await new Promise((resolve) => setTimeout(resolve, 100));\n }\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n // Record as \"act\" step with proper Actions for deterministic replay (only when caching)\n if (shouldCollectXpath && actions.length > 0) {\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: `Fill ${fields.length} form fields`,\n actions,\n actionDescription: `Fill ${fields.length} form fields`,\n });\n }\n\n return {\n success: true,\n playwrightArguments: processedFields,\n screenshotBase64,\n };\n } catch (error) {\n return {\n success: false,\n error: `Error filling form: ${(error as Error).message}`,\n };\n }\n },\n toModelOutput: (result) => {\n if (result.success) {\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n fieldsCount: result.playwrightArguments?.length ?? 0,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n }\n return {\n type: \"content\",\n value: [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n error: result.error,\n }),\n },\n ],\n };\n },\n });\n};\n"]}
1	+ {"version":3,"file":"fillFormVision.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillFormVision.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAChC,EAAM,EACN,QAAiB,EACjB,SAAqB,EACrB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC,qGAAqG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QAC1I,CAAC,CAAC,oCAAoC,CAAC;IAEzC,OAAO,IAAI,CAAC;QACV,WAAW,EAAE;;;;;;;;;;;;;;;;2CAgB0B;QACvC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,CAAC;iBACN,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,CAAC;qBACN,MAAM,EAAE;qBACR,QAAQ,CACP,sEAAsE,CACvE;gBACH,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBAC5C,WAAW,EAAE,CAAC;qBACX,MAAM,CAAC;oBACN,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;oBACb,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE;iBACd,CAAC;qBACD,QAAQ,CAAC,iCAAiC,CAAC;aAC/C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,qCAAqC,CAAC;SACjD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAqC,EAAE;YAC/D,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAEhD,8DAA8D;gBAC9D,0FAA0F;gBAC1F,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;oBAC3C,MAAM,SAAS,GAAG,kBAAkB,CAClC,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,QAAQ,EACR,EAAE,CACH,CAAC;oBACF,OAAO;wBACL,GAAG,KAAK;wBACR,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE,wCAAwC;wBACpE,KAAK,EAAE,mBAAmB,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC;wBAClD,WAAW,EAAE,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;qBAChD,CAAC;gBACJ,CAAC,CAAC,CAAC;gBAEH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,oCAAoC;oBAC7C,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,+BAA+B;4BAClE,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,OAAO,GAAa,EAAE,CAAC;gBAE7B,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;oBACpC,iEAAiE;oBACjE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAC5B,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB,KAAK,CAAC,WAAW,CAAC,CAAC,EACnB;wBACE,WAAW,EAAE,kBAAkB;qBAChC,CACF,CAAC;oBACF,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;oBAE7B,uEAAuE;oBACvE,qFAAqF;oBACrF,IAAI,kBAAkB,EAAE,CAAC;wBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;wBAC3C,IAAI,eAAe,EAAE,CAAC;4BACpB,OAAO,CAAC,IAAI,CAAC;gCACX,QAAQ,EAAE,eAAe;gCACzB,WAAW,EAAE,KAAK,CAAC,MAAM;gCACzB,MAAM,EAAE,MAAM;gCACd,SAAS,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC;6BACjC,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,6BAA6B;oBAC7B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;gBAC3D,CAAC;gBAED,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAEnE,wFAAwF;gBACxF,IAAI,kBAAkB,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7C,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;wBAChD,OAAO;wBACP,iBAAiB,EAAE,QAAQ,MAAM,CAAC,MAAM,cAAc;qBACvD,CAAC,CAAC;gBACL,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,mBAAmB,EAAE,eAAe;oBACpC,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,uBAAwB,KAAe,CAAC,OAAO,EAAE;iBACzD,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE;wBACL;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gCACnB,OAAO,EAAE,MAAM,CAAC,OAAO;gCACvB,KAAK,EAAE,MAAM,CAAC,KAAK;6BACpB,CAAC;yBACH;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,WAAW,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,IAAI,CAAC;qBACrD,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n FillFormVisionToolResult,\n ModelOutputContentItem,\n Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const fillFormVisionTool = (\n v3: V3,\n provider?: string,\n variables?: Variables,\n) => {\n const hasVariables = variables && Object.keys(variables).length > 0;\n const valueDescription = hasVariables\n ? `Text to type into the target field. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n : \"Text to type into the target field\";\n\n return tool({\n description: `FORM FILL - SPECIALIZED MULTI-FIELD INPUT TOOL\n\nCRITICAL: Use this for ANY form with 2+ input fields (text inputs, textareas, etc.)\nIMPORTANT: Ensure the fields are visible within the current viewport\n\nWHY THIS TOOL EXISTS:\n- Forms are the #1 use case for multi-field input\n- Optimized specifically for input/textarea elements\n- 4-6x faster than individual typing actions\n\nUse fillFormVision: Pure form filling (inputs, textareas only)\nMANDATORY USE CASES (always use fillFormVision for these):\n- Registration forms: name, email, password fields\n- Contact forms: name, email, message fields\n- Checkout forms: address, payment info fields\n- Profile updates: multiple user data fields\n- Search filters: multiple criteria inputs`,\n inputSchema: z.object({\n fields: z\n .array(\n z.object({\n action: z\n .string()\n .describe(\n \"Description of the typing action, e.g. 'type foo into the bar field'\",\n ),\n value: z.string().describe(valueDescription),\n coordinates: z\n .object({\n x: z.number(),\n y: z.number(),\n })\n .describe(\"Coordinates of the target field\"),\n }),\n )\n .min(2, \"Provide at least two fields to fill\"),\n }),\n execute: async ({ fields }): Promise<FillFormVisionToolResult> => {\n try {\n const page = await v3.context.awaitActivePage();\n\n // Process coordinates and substitute variables for each field\n // Keep original values (with %tokens%) for logging/caching, substituted values for typing\n const processedFields = fields.map((field) => {\n const processed = processCoordinates(\n field.coordinates.x,\n field.coordinates.y,\n provider,\n v3,\n );\n return {\n ...field,\n originalValue: field.value, // Keep original with %tokens% for cache\n value: substituteVariables(field.value, variables),\n coordinates: { x: processed.x, y: processed.y },\n };\n });\n\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: fillFormVision`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify({ fields }), // Don't log substituted values\n type: \"object\",\n },\n },\n });\n\n // Only request XPath when caching is enabled to avoid unnecessary computation\n const shouldCollectXpath = v3.isAgentReplayActive();\n const actions: Action[] = [];\n\n for (const field of processedFields) {\n // Click the field, only requesting XPath when caching is enabled\n const xpath = await page.click(\n field.coordinates.x,\n field.coordinates.y,\n {\n returnXpath: shouldCollectXpath,\n },\n );\n await page.type(field.value);\n\n // Build Action with XPath for deterministic replay (only when caching)\n // Use originalValue (with %tokens%) so cache stores references, not sensitive values\n if (shouldCollectXpath) {\n const normalizedXpath = ensureXPath(xpath);\n if (normalizedXpath) {\n actions.push({\n selector: normalizedXpath,\n description: field.action,\n method: \"type\",\n arguments: [field.originalValue],\n });\n }\n }\n\n // Small delay between fields\n await new Promise((resolve) => setTimeout(resolve, 100));\n }\n\n const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n // Record as \"act\" step with proper Actions for deterministic replay (only when caching)\n if (shouldCollectXpath && actions.length > 0) {\n v3.recordAgentReplayStep({\n type: \"act\",\n instruction: `Fill ${fields.length} form fields`,\n actions,\n actionDescription: `Fill ${fields.length} form fields`,\n });\n }\n\n return {\n success: true,\n playwrightArguments: processedFields,\n screenshotBase64,\n };\n } catch (error) {\n return {\n success: false,\n error: `Error filling form: ${(error as Error).message}`,\n };\n }\n },\n toModelOutput: (result) => {\n if (result.success === false \|\| result.error !== undefined) {\n return {\n type: \"content\",\n value: [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n error: result.error,\n }),\n },\n ],\n };\n }\n\n const content: ModelOutputContentItem[] = [\n {\n type: \"text\",\n text: JSON.stringify({\n success: result.success,\n fieldsCount: result.playwrightArguments?.length ?? 0,\n }),\n },\n ];\n if (result.screenshotBase64) {\n content.push({\n type: \"media\",\n mediaType: \"image/png\",\n data: result.screenshotBase64,\n });\n }\n return { type: \"content\", value: content };\n },\n });\n};\n"]}

package/dist/esm/lib/v3/agent/tools/fillform.d.ts CHANGED Viewed

@@ -1,7 +1,7 @@
 import type { V3 } from "../../v3.js";
 import type { Action } from "../../types/public/methods.js";
 import type { AgentModelConfig, Variables } from "../../types/public/agent.js";
-export declare const fillFormTool: (v3: V3, executionModel?: string | AgentModelConfig, variables?: Variables) => import("ai").Tool<{
+export declare const fillFormTool: (v3: V3, executionModel?: string | AgentModelConfig, variables?: Variables, toolTimeout?: number) => import("ai").Tool<{
     fields: {
         action: string;
         value: string;
@@ -10,4 +10,10 @@ export declare const fillFormTool: (v3: V3, executionModel?: string | AgentModel
     success: boolean;
     actions: unknown[];
     playwrightArguments: Action[];
+    error?: undefined;
+} | {
+    success: boolean;
+    error: any;
+    actions?: undefined;
+    playwrightArguments?: undefined;
 }>;