@browserbasehq/orca 3.0.9-alpha-3 → 3.1.0-headers

package/dist/esm/lib/v3/agent/tools/scroll.js

@@ -0,0 +1,154 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { processCoordinates } from "../utils/coordinateNormalization.js";
+import { waitAndCaptureScreenshot } from "../utils/screenshotHandler.js";
+/**
+ * Simple scroll tool for DOM mode (non-grounding models).
+ * No coordinates - scrolls from viewport center.
+ */
+export const scrollTool = (v3) => tool({
+    description: "Scroll the page up or down by a percentage of the viewport height. Default is 80%, and what should be typically used for general page scrolling",
+    inputSchema: z.object({
+        direction: z.enum(["up", "down"]),
+        percentage: z.number().min(1).max(200).optional(),
+    }),
+    execute: async ({ direction, percentage = 80, }) => {
+        v3.logger({
+            category: "agent",
+            message: `Agent calling tool: scroll`,
+            level: 1,
+            auxiliary: {
+                arguments: {
+                    value: JSON.stringify({ direction, percentage }),
+                    type: "object",
+                },
+            },
+        });
+        const page = await v3.context.awaitActivePage();
+        const { w, h } = await page.mainFrame().evaluate("({ w: window.innerWidth, h: window.innerHeight })");
+        const scrollDistance = Math.round((h * percentage) / 100);
+        const cx = Math.floor(w / 2);
+        const cy = Math.floor(h / 2);
+        const deltaY = direction === "up" ? -scrollDistance : scrollDistance;
+        await page.scroll(cx, cy, 0, deltaY);
+        v3.recordAgentReplayStep({
+            type: "scroll",
+            deltaX: 0,
+            deltaY,
+            anchor: { x: cx, y: cy },
+        });
+        return {
+            success: true,
+            message: `Scrolled ${percentage}% ${direction} (${scrollDistance}px)`,
+            scrolledPixels: scrollDistance,
+        };
+    },
+    toModelOutput: (result) => {
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
+        }
+        return {
+            type: "json",
+            value: {
+                success: result.success,
+                message: result.message,
+                scrolledPixels: result.scrolledPixels,
+            },
+        };
+    },
+});
+/**
+ * Scroll tool for hybrid mode (grounding models).
+ * Supports optional coordinates for scrolling within nested scrollable elements.
+ */
+export const scrollVisionTool = (v3, provider) => tool({
+    description: `Scroll the page up or down. For general page scrolling, no coordinates needed. Only provide coordinates when scrolling inside a nested scrollable element (e.g., a dropdown menu, modal with overflow, or scrollable sidebar). Default is 80%, and what should be typically used for general page scrolling`,
+    inputSchema: z.object({
+        direction: z.enum(["up", "down"]),
+        coordinates: z
+            .array(z.number())
+            .optional()
+            .describe("Only use coordinates for scrolling inside a nested scrollable element - provide (x, y) within that element"),
+        percentage: z.number().min(1).max(200).optional(),
+    }),
+    execute: async ({ direction, coordinates, percentage = 80, }) => {
+        const page = await v3.context.awaitActivePage();
+        const { w, h } = await page.mainFrame().evaluate("({ w: window.innerWidth, h: window.innerHeight })");
+        // Process coordinates if provided, otherwise use viewport center
+        let cx;
+        let cy;
+        if (coordinates) {
+            const processed = processCoordinates(coordinates[0], coordinates[1], provider, v3);
+            cx = processed.x;
+            cy = processed.y;
+        }
+        else {
+            cx = Math.floor(w / 2);
+            cy = Math.floor(h / 2);
+        }
+        v3.logger({
+            category: "agent",
+            message: `Agent calling tool: scroll`,
+            level: 1,
+            auxiliary: {
+                arguments: {
+                    value: JSON.stringify({
+                        direction,
+                        coordinates,
+                        percentage,
+                        processed: { cx, cy },
+                    }),
+                    type: "object",
+                },
+            },
+        });
+        const scrollDistance = Math.round((h * percentage) / 100);
+        const deltaY = direction === "up" ? -scrollDistance : scrollDistance;
+        await page.scroll(cx, cy, 0, deltaY);
+        const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);
+        v3.recordAgentReplayStep({
+            type: "scroll",
+            deltaX: 0,
+            deltaY,
+            anchor: { x: cx, y: cy },
+        });
+        return {
+            success: true,
+            message: coordinates
+                ? `Scrolled ${percentage}% ${direction} at (${cx}, ${cy})`
+                : `Scrolled ${percentage}% ${direction}`,
+            scrolledPixels: scrollDistance,
+            screenshotBase64,
+        };
+    },
+    toModelOutput: (result) => {
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
+        }
+        const content = [
+            {
+                type: "text",
+                text: JSON.stringify({
+                    success: result.success,
+                    message: result.message,
+                    scrolledPixels: result.scrolledPixels,
+                }),
+            },
+        ];
+        if (result.screenshotBase64) {
+            content.push({
+                type: "media",
+                mediaType: "image/png",
+                data: result.screenshotBase64,
+            });
+        }
+        return { type: "content", value: content };
+    },
+});
+//# sourceMappingURL=scroll.js.map

package/dist/esm/lib/v3/agent/tools/scroll.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scroll.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/scroll.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE;;;GAGG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,EAAM,EAAE,EAAE,CACnC,IAAI,CAAC;IACH,WAAW,EACT,iJAAiJ;IACnJ,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,UAAU,GAAG,EAAE,GAChB,EAA6B,EAAE;QAC9B,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;oBAChD,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,YAAY,UAAU,KAAK,SAAS,KAAK,cAAc,KAAK;YACrE,cAAc,EAAE,cAAc;SAC/B,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE;gBACL,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;aACtC;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEL;;;GAGG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC5D,IAAI,CAAC;IACH,WAAW,EAAE,6SAA6S;IAC1T,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,EAAE;aACV,QAAQ,CACP,4GAA4G,CAC7G;QACH,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,WAAW,EACX,UAAU,GAAG,EAAE,GAChB,EAAmC,EAAE;QACpC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,iEAAiE;QACjE,IAAI,EAAU,CAAC;QACf,IAAI,EAAU,CAAC;QACf,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YACF,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;YACjB,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACvB,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzB,CAAC;QAED,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;wBACpB,SAAS;wBACT,WAAW;wBACX,UAAU;wBACV,SAAS,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE;qBACtB,CAAC;oBACF,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAEnE,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,WAAW;gBAClB,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,QAAQ,EAAE,KAAK,EAAE,GAAG;gBAC1D,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,EAAE;YAC1C,cAAc,EAAE,cAAc;YAC9B,gBAAgB;SACjB,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;iBACtC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type {\n  ScrollToolResult,\n  ScrollVisionToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\n/**\n * Simple scroll tool for DOM mode (non-grounding models).\n * No coordinates - scrolls from viewport center.\n */\nexport const scrollTool = (v3: V3) =>\n  tool({\n    description:\n      \"Scroll the page up or down by a percentage of the viewport height. Default is 80%, and what should be typically used for general page scrolling\",\n    inputSchema: z.object({\n      direction: z.enum([\"up\", \"down\"]),\n      percentage: z.number().min(1).max(200).optional(),\n    }),\n    execute: async ({\n      direction,\n      percentage = 80,\n    }): Promise<ScrollToolResult> => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: scroll`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: JSON.stringify({ direction, percentage }),\n            type: \"object\",\n          },\n        },\n      });\n\n      const page = await v3.context.awaitActivePage();\n\n      const { w, h } = await page.mainFrame().evaluate<{\n        w: number;\n        h: number;\n      }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n      const scrollDistance = Math.round((h * percentage) / 100);\n      const cx = Math.floor(w / 2);\n      const cy = Math.floor(h / 2);\n      const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n      await page.scroll(cx, cy, 0, deltaY);\n\n      v3.recordAgentReplayStep({\n        type: \"scroll\",\n        deltaX: 0,\n        deltaY,\n        anchor: { x: cx, y: cy },\n      });\n\n      return {\n        success: true,\n        message: `Scrolled ${percentage}% ${direction} (${scrollDistance}px)`,\n        scrolledPixels: scrollDistance,\n      };\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      return {\n        type: \"json\",\n        value: {\n          success: result.success,\n          message: result.message,\n          scrolledPixels: result.scrolledPixels,\n        },\n      };\n    },\n  });\n\n/**\n * Scroll tool for hybrid mode (grounding models).\n * Supports optional coordinates for scrolling within nested scrollable elements.\n */\nexport const scrollVisionTool = (v3: V3, provider?: string) =>\n  tool({\n    description: `Scroll the page up or down. For general page scrolling, no coordinates needed. Only provide coordinates when scrolling inside a nested scrollable element (e.g., a dropdown menu, modal with overflow, or scrollable sidebar). Default is 80%, and what should be typically used for general page scrolling`,\n    inputSchema: z.object({\n      direction: z.enum([\"up\", \"down\"]),\n      coordinates: z\n        .array(z.number())\n        .optional()\n        .describe(\n          \"Only use coordinates for scrolling inside a nested scrollable element - provide (x, y) within that element\",\n        ),\n      percentage: z.number().min(1).max(200).optional(),\n    }),\n    execute: async ({\n      direction,\n      coordinates,\n      percentage = 80,\n    }): Promise<ScrollVisionToolResult> => {\n      const page = await v3.context.awaitActivePage();\n\n      const { w, h } = await page.mainFrame().evaluate<{\n        w: number;\n        h: number;\n      }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n      // Process coordinates if provided, otherwise use viewport center\n      let cx: number;\n      let cy: number;\n      if (coordinates) {\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n        cx = processed.x;\n        cy = processed.y;\n      } else {\n        cx = Math.floor(w / 2);\n        cy = Math.floor(h / 2);\n      }\n\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: scroll`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: JSON.stringify({\n              direction,\n              coordinates,\n              percentage,\n              processed: { cx, cy },\n            }),\n            type: \"object\",\n          },\n        },\n      });\n\n      const scrollDistance = Math.round((h * percentage) / 100);\n      const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n      await page.scroll(cx, cy, 0, deltaY);\n\n      const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n      v3.recordAgentReplayStep({\n        type: \"scroll\",\n        deltaX: 0,\n        deltaY,\n        anchor: { x: cx, y: cy },\n      });\n\n      return {\n        success: true,\n        message: coordinates\n          ? `Scrolled ${percentage}% ${direction} at (${cx}, ${cy})`\n          : `Scrolled ${percentage}% ${direction}`,\n        scrolledPixels: scrollDistance,\n        screenshotBase64,\n      };\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            message: result.message,\n            scrolledPixels: result.scrolledPixels,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/search.d.ts

@@ -0,0 +1,15 @@
+import type { V3 } from "../../v3.js";
+export interface BraveSearchResult {
+    title: string;
+    url: string;
+    description?: string;
+}
+export declare const searchTool: (v3: V3) => import("ai").Tool<{
+    query: string;
+}, {
+    timestamp: number;
+    data?: {
+        results: BraveSearchResult[];
+    };
+    error?: string;
+}>;

package/dist/esm/lib/v3/agent/tools/search.js

@@ -0,0 +1,73 @@
+import { tool } from "ai";
+import { z } from "zod";
+async function performBraveSearch(query) {
+    try {
+        const encodedQuery = encodeURIComponent(query);
+        const response = await fetch(`https://api.search.brave.com/res/v1/web/search?q=${encodedQuery}`, {
+            method: "GET",
+            headers: {
+                Accept: "application/json",
+                "Accept-Encoding": "gzip",
+                "X-Subscription-Token": process.env.BRAVE_API_KEY,
+            },
+        });
+        if (!response.ok) {
+            return {
+                error: `Brave API error: ${response.status} ${response.statusText}`,
+                data: { results: [] },
+            };
+        }
+        const data = (await response.json());
+        const results = [];
+        if (data?.web?.results && Array.isArray(data.web.results)) {
+            for (const item of data.web.results.slice(0, 5)) {
+                if (item.title && item.url) {
+                    results.push({
+                        title: item.title,
+                        url: item.url,
+                        description: item.description,
+                    });
+                }
+            }
+        }
+        return { data: { results } };
+    }
+    catch (error) {
+        console.error("Search error", error);
+        return {
+            error: `Error performing search: ${error.message}`,
+            data: { results: [] },
+        };
+    }
+}
+export const searchTool = (v3) => tool({
+    description: "Perform a web search and returns results. Use this tool when you need information from the web or when you are unsure of the exact URL you want to navigate to. This can be used to find the ideal entry point, resulting in a task that is easier to complete due to starting further in the process.",
+    inputSchema: z.object({
+        query: z.string().describe("The search query to look for on the web"),
+    }),
+    execute: async ({ query }) => {
+        v3.logger({
+            category: "agent",
+            message: `Agent calling tool: search`,
+            level: 1,
+            auxiliary: {
+                arguments: {
+                    value: JSON.stringify({ query }),
+                    type: "object",
+                },
+            },
+        });
+        const result = await performBraveSearch(query);
+        v3.recordAgentReplayStep({
+            type: "search",
+            instruction: query,
+            playwrightArguments: { query },
+            message: result.error ?? `Found ${result.data?.results.length ?? 0} results`,
+        });
+        return {
+            ...result,
+            timestamp: Date.now(),
+        };
+    },
+});
+//# sourceMappingURL=search.js.map

package/dist/esm/lib/v3/agent/tools/search.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAgCxB,KAAK,UAAU,kBAAkB,CAAC,KAAa;IAC7C,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAC1B,oDAAoD,YAAY,EAAE,EAClE;YACE,MAAM,EAAE,KAAK;YACb,OAAO,EAAE;gBACP,MAAM,EAAE,kBAAkB;gBAC1B,iBAAiB,EAAE,MAAM;gBACzB,sBAAsB,EAAE,OAAO,CAAC,GAAG,CAAC,aAAc;aACnD;SACF,CACF,CAAC;QAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,OAAO;gBACL,KAAK,EAAE,oBAAoB,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE;gBACnE,IAAI,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;aACtB,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAqB,CAAC;QACzD,MAAM,OAAO,GAAwB,EAAE,CAAC;QAExC,IAAI,IAAI,EAAE,GAAG,EAAE,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1D,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;gBAChD,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;oBAC3B,OAAO,CAAC,IAAI,CAAC;wBACX,KAAK,EAAE,IAAI,CAAC,KAAK;wBACjB,GAAG,EAAE,IAAI,CAAC,GAAG;wBACb,WAAW,EAAE,IAAI,CAAC,WAAW;qBAC9B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;QACrC,OAAO;YACL,KAAK,EAAE,4BAA6B,KAAe,CAAC,OAAO,EAAE;YAC7D,IAAI,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;SACtB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,EAAM,EAAE,EAAE,CACnC,IAAI,CAAC;IACH,WAAW,EACT,wSAAwS;IAC1S,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;KACtE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;QAC3B,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC;oBAChC,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAE/C,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,KAAK;YAClB,mBAAmB,EAAE,EAAE,KAAK,EAAE;YAC9B,OAAO,EACL,MAAM,CAAC,KAAK,IAAI,SAAS,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,UAAU;SACtE,CAAC,CAAC;QAEH,OAAO;YACL,GAAG,MAAM;YACT,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;IACJ,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport interface BraveSearchResult {\n  title: string;\n  url: string;\n  description?: string;\n}\n\ninterface SearchResponse {\n  data?: {\n    results: BraveSearchResult[];\n  };\n  error?: string;\n}\n\ninterface BraveWebResult {\n  title?: string;\n  url?: string;\n  description?: string;\n  age?: string;\n  meta_url?: {\n    favicon?: string;\n  };\n}\n\ninterface BraveApiResponse {\n  web?: {\n    results?: BraveWebResult[];\n  };\n}\n\nasync function performBraveSearch(query: string): Promise<SearchResponse> {\n  try {\n    const encodedQuery = encodeURIComponent(query);\n    const response = await fetch(\n      `https://api.search.brave.com/res/v1/web/search?q=${encodedQuery}`,\n      {\n        method: \"GET\",\n        headers: {\n          Accept: \"application/json\",\n          \"Accept-Encoding\": \"gzip\",\n          \"X-Subscription-Token\": process.env.BRAVE_API_KEY!,\n        },\n      },\n    );\n\n    if (!response.ok) {\n      return {\n        error: `Brave API error: ${response.status} ${response.statusText}`,\n        data: { results: [] },\n      };\n    }\n\n    const data = (await response.json()) as BraveApiResponse;\n    const results: BraveSearchResult[] = [];\n\n    if (data?.web?.results && Array.isArray(data.web.results)) {\n      for (const item of data.web.results.slice(0, 5)) {\n        if (item.title && item.url) {\n          results.push({\n            title: item.title,\n            url: item.url,\n            description: item.description,\n          });\n        }\n      }\n    }\n\n    return { data: { results } };\n  } catch (error) {\n    console.error(\"Search error\", error);\n    return {\n      error: `Error performing search: ${(error as Error).message}`,\n      data: { results: [] },\n    };\n  }\n}\n\nexport const searchTool = (v3: V3) =>\n  tool({\n    description:\n      \"Perform a web search and returns results. Use this tool when you need information from the web or when you are unsure of the exact URL you want to navigate to. This can be used to find the ideal entry point, resulting in a task that is easier to complete due to starting further in the process.\",\n    inputSchema: z.object({\n      query: z.string().describe(\"The search query to look for on the web\"),\n    }),\n    execute: async ({ query }) => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: search`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: JSON.stringify({ query }),\n            type: \"object\",\n          },\n        },\n      });\n\n      const result = await performBraveSearch(query);\n\n      v3.recordAgentReplayStep({\n        type: \"search\",\n        instruction: query,\n        playwrightArguments: { query },\n        message:\n          result.error ?? `Found ${result.data?.results.length ?? 0} results`,\n      });\n\n      return {\n        ...result,\n        timestamp: Date.now(),\n      };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/think.d.ts

@@ -0,0 +1,6 @@
+export declare const thinkTool: () => import("ai").Tool<{
+    reasoning: string;
+}, {
+    acknowledged: boolean;
+    message: string;
+}>;

package/dist/esm/lib/v3/agent/tools/think.js

@@ -0,0 +1,24 @@
+import { tool } from "ai";
+import { z } from "zod";
+export const thinkTool = () => tool({
+    description: `Use this tool to think through complex problems or plan a sequence of steps. This is for internal reasoning only and doesn't perform any actions. Use this to:
+
+1. Plan a multi-step approach before taking action
+2. Break down complex tasks
+3. Reason through edge cases
+4. Evaluate options when you're unsure what to do next
+
+The output is only visible to you; use it to track your own reasoning process.`,
+    inputSchema: z.object({
+        reasoning: z
+            .string()
+            .describe("Your step-by-step reasoning or planning process. Be as detailed as needed."),
+    }),
+    execute: async ({ reasoning }) => {
+        return {
+            acknowledged: true,
+            message: reasoning,
+        };
+    },
+});
+//# sourceMappingURL=think.js.map

package/dist/esm/lib/v3/agent/tools/think.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"think.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/think.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,SAAS,GAAG,GAAG,EAAE,CAC5B,IAAI,CAAC;IACH,WAAW,EAAE;;;;;;;+EAO8D;IAC3E,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC;aACT,MAAM,EAAE;aACR,QAAQ,CACP,4EAA4E,CAC7E;KACJ,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE;QAC/B,OAAO;YACL,YAAY,EAAE,IAAI;YAClB,OAAO,EAAE,SAAS;SACnB,CAAC;IACJ,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\n\nexport const thinkTool = () =>\n  tool({\n    description: `Use this tool to think through complex problems or plan a sequence of steps. This is for internal reasoning only and doesn't perform any actions. Use this to:\n\n1. Plan a multi-step approach before taking action\n2. Break down complex tasks\n3. Reason through edge cases\n4. Evaluate options when you're unsure what to do next\n\nThe output is only visible to you; use it to track your own reasoning process.`,\n    inputSchema: z.object({\n      reasoning: z\n        .string()\n        .describe(\n          \"Your step-by-step reasoning or planning process. Be as detailed as needed.\",\n        ),\n    }),\n    execute: async ({ reasoning }) => {\n      return {\n        acknowledged: true,\n        message: reasoning,\n      };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/type.d.ts

@@ -0,0 +1,7 @@
+import type { V3 } from "../../v3.js";
+import type { TypeToolResult, Variables } from "../../types/public/agent.js";
+export declare const typeTool: (v3: V3, provider?: string, variables?: Variables) => import("ai").Tool<{
+    describe: string;
+    text: string;
+    coordinates: number[];
+}, TypeToolResult>;

package/dist/esm/lib/v3/agent/tools/type.js

@@ -0,0 +1,107 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { processCoordinates } from "../utils/coordinateNormalization.js";
+import { ensureXPath } from "../utils/xpath.js";
+import { waitAndCaptureScreenshot } from "../utils/screenshotHandler.js";
+import { substituteVariables } from "../utils/variables.js";
+export const typeTool = (v3, provider, variables) => {
+    const hasVariables = variables && Object.keys(variables).length > 0;
+    const textDescription = hasVariables
+        ? `The text to type into the element. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(", ")}`
+        : "The text to type into the element";
+    return tool({
+        description: "Type text into an element using its coordinates. This will click the element and then type the text into it (this is the most reliable way to type into an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)",
+        inputSchema: z.object({
+            describe: z
+                .string()
+                .describe("Describe the element to type into in a short, specific phrase that mentions the element type and a good visual description"),
+            text: z.string().describe(textDescription),
+            coordinates: z
+                .array(z.number())
+                .describe("The (x, y) coordinates to type into the element"),
+        }),
+        execute: async ({ describe, coordinates, text, }) => {
+            try {
+                const page = await v3.context.awaitActivePage();
+                const processed = processCoordinates(coordinates[0], coordinates[1], provider, v3);
+                // Substitute any %variableName% tokens in the text
+                const actualText = substituteVariables(text, variables);
+                v3.logger({
+                    category: "agent",
+                    message: `Agent calling tool: type`,
+                    level: 1,
+                    auxiliary: {
+                        arguments: {
+                            value: JSON.stringify({ describe, text }),
+                            type: "object",
+                        },
+                    },
+                });
+                // Only request XPath when caching is enabled to avoid unnecessary computation
+                const shouldCollectXpath = v3.isAgentReplayActive();
+                const xpath = await page.click(processed.x, processed.y, {
+                    returnXpath: shouldCollectXpath,
+                });
+                await page.type(actualText);
+                const screenshotBase64 = await waitAndCaptureScreenshot(page);
+                // Record as an "act" step with proper Action for deterministic replay (only when caching)
+                if (shouldCollectXpath) {
+                    const normalizedXpath = ensureXPath(xpath);
+                    if (normalizedXpath) {
+                        const action = {
+                            selector: normalizedXpath,
+                            description: describe,
+                            method: "type",
+                            arguments: [text],
+                        };
+                        v3.recordAgentReplayStep({
+                            type: "act",
+                            instruction: describe,
+                            actions: [action],
+                            actionDescription: describe,
+                        });
+                    }
+                }
+                return {
+                    success: true,
+                    describe,
+                    text, // Return original text (with %variableName% tokens) to avoid exposing sensitive values to LLM
+                    screenshotBase64,
+                };
+            }
+            catch (error) {
+                return {
+                    success: false,
+                    error: `Error typing: ${error.message}`,
+                };
+            }
+        },
+        toModelOutput: (result) => {
+            if (result.success === false || result.error !== undefined) {
+                return {
+                    type: "content",
+                    value: [{ type: "text", text: JSON.stringify(result) }],
+                };
+            }
+            const content = [
+                {
+                    type: "text",
+                    text: JSON.stringify({
+                        success: result.success,
+                        describe: result.describe,
+                        text: result.text,
+                    }),
+                },
+            ];
+            if (result.screenshotBase64) {
+                content.push({
+                    type: "media",
+                    mediaType: "image/png",
+                    data: result.screenshotBase64,
+                });
+            }
+            return { type: "content", value: content };
+        },
+    });
+};
+//# sourceMappingURL=type.js.map

package/dist/esm/lib/v3/agent/tools/type.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"type.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/type.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE5D,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,SAAqB,EAAE,EAAE;IAC3E,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,eAAe,GAAG,YAAY;QAClC,CAAC,CAAC,oGAAoG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACzI,CAAC,CAAC,mCAAmC,CAAC;IAExC,OAAO,IAAI,CAAC;QACV,WAAW,EACT,2QAA2Q;QAC7Q,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;YACpB,QAAQ,EAAE,CAAC;iBACR,MAAM,EAAE;iBACR,QAAQ,CACP,4HAA4H,CAC7H;YACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC;YAC1C,WAAW,EAAE,CAAC;iBACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;iBACjB,QAAQ,CAAC,iDAAiD,CAAC;SAC/D,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,WAAW,EACX,IAAI,GACL,EAA2B,EAAE;YAC5B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;gBAChD,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;gBAEF,mDAAmD;gBACnD,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBAExD,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,0BAA0B;oBACnC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;4BACzC,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBAEH,8EAA8E;gBAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;gBACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;oBACvD,WAAW,EAAE,kBAAkB;iBAChC,CAAC,CAAC;gBAEH,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAE5B,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;gBAE9D,0FAA0F;gBAC1F,IAAI,kBAAkB,EAAE,CAAC;oBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;oBAC3C,IAAI,eAAe,EAAE,CAAC;wBACpB,MAAM,MAAM,GAAW;4BACrB,QAAQ,EAAE,eAAe;4BACzB,WAAW,EAAE,QAAQ;4BACrB,MAAM,EAAE,MAAM;4BACd,SAAS,EAAE,CAAC,IAAI,CAAC;yBAClB,CAAC;wBACF,EAAE,CAAC,qBAAqB,CAAC;4BACvB,IAAI,EAAE,KAAK;4BACX,WAAW,EAAE,QAAQ;4BACrB,OAAO,EAAE,CAAC,MAAM,CAAC;4BACjB,iBAAiB,EAAE,QAAQ;yBAC5B,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,QAAQ;oBACR,IAAI,EAAE,8FAA8F;oBACpG,gBAAgB;iBACjB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,iBAAkB,KAAe,CAAC,OAAO,EAAE;iBACnD,CAAC;YACJ,CAAC;QACH,CAAC;QACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3D,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;iBACxD,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,IAAI,EAAE,MAAM,CAAC,IAAI;qBAClB,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  TypeToolResult,\n  ModelOutputContentItem,\n  Variables,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\nimport { substituteVariables } from \"../utils/variables.js\";\n\nexport const typeTool = (v3: V3, provider?: string, variables?: Variables) => {\n  const hasVariables = variables && Object.keys(variables).length > 0;\n  const textDescription = hasVariables\n    ? `The text to type into the element. Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}`\n    : \"The text to type into the element\";\n\n  return tool({\n    description:\n      \"Type text into an element using its coordinates. This will click the element and then type the text into it (this is the most reliable way to type into an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to type into in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      text: z.string().describe(textDescription),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to type into the element\"),\n    }),\n    execute: async ({\n      describe,\n      coordinates,\n      text,\n    }): Promise<TypeToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        // Substitute any %variableName% tokens in the text\n        const actualText = substituteVariables(text, variables);\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: type`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe, text }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const xpath = await page.click(processed.x, processed.y, {\n          returnXpath: shouldCollectXpath,\n        });\n\n        await page.type(actualText);\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"type\",\n              arguments: [text],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          text, // Return original text (with %variableName% tokens) to avoid exposing sensitive values to LLM\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error typing: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            describe: result.describe,\n            text: result.text,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n};\n"]}

package/dist/esm/lib/v3/agent/tools/wait.d.ts

@@ -0,0 +1,5 @@
+import type { V3 } from "../../v3.js";
+import type { AgentToolMode, WaitToolResult } from "../../types/public/agent.js";
+export declare const waitTool: (v3: V3, mode?: AgentToolMode) => import("ai").Tool<{
+    timeMs: number;
+}, WaitToolResult>;

package/dist/esm/lib/v3/agent/tools/wait.js

@@ -0,0 +1,59 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { waitAndCaptureScreenshot } from "../utils/screenshotHandler.js";
+export const waitTool = (v3, mode) => tool({
+    description: "Wait for a specified time",
+    inputSchema: z.object({
+        timeMs: z.number().describe("Time in milliseconds"),
+    }),
+    execute: async ({ timeMs }) => {
+        v3.logger({
+            category: "agent",
+            message: `Agent calling tool: wait`,
+            level: 1,
+            auxiliary: {
+                arguments: {
+                    value: `Waiting for ${timeMs} milliseconds`,
+                    type: "string",
+                },
+            },
+        });
+        await new Promise((resolve) => setTimeout(resolve, timeMs));
+        if (timeMs > 0) {
+            v3.recordAgentReplayStep({ type: "wait", timeMs });
+        }
+        // Take screenshot after wait in hybrid mode for visual feedback
+        if (mode === "hybrid") {
+            const page = await v3.context.awaitActivePage();
+            const screenshotBase64 = await waitAndCaptureScreenshot(page, 0);
+            return { success: true, waited: timeMs, screenshotBase64 };
+        }
+        return { success: true, waited: timeMs };
+    },
+    toModelOutput: (result) => {
+        if (result.success === false || result.error !== undefined) {
+            return {
+                type: "content",
+                value: [{ type: "text", text: JSON.stringify(result) }],
+            };
+        }
+        const content = [
+            {
+                type: "text",
+                text: JSON.stringify({
+                    success: result.success,
+                    waited: result.waited,
+                }),
+            },
+        ];
+        if (result.screenshotBase64) {
+            content.push({
+                type: "media",
+                mediaType: "image/png",
+                data: result.screenshotBase64,
+            });
+        }
+        return { type: "content", value: content };
+    },
+});
+//# sourceMappingURL=wait.js.map

package/dist/esm/lib/v3/agent/tools/wait.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"wait.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/wait.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,IAAoB,EAAE,EAAE,CACvD,IAAI,CAAC;IACH,WAAW,EAAE,2BAA2B;IACxC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sBAAsB,CAAC;KACpD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAA2B,EAAE;QACrD,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,0BAA0B;YACnC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,eAAe,MAAM,eAAe;oBAC3C,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QACH,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;QAC5D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YACf,EAAE,CAAC,qBAAqB,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,gEAAgE;QAChE,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtB,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACjE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;QAC7D,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IAC3C,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3D,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;aACxD,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,MAAM,EAAE,MAAM,CAAC,MAAM;iBACtB,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type {\n  AgentToolMode,\n  WaitToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const waitTool = (v3: V3, mode?: AgentToolMode) =>\n  tool({\n    description: \"Wait for a specified time\",\n    inputSchema: z.object({\n      timeMs: z.number().describe(\"Time in milliseconds\"),\n    }),\n    execute: async ({ timeMs }): Promise<WaitToolResult> => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: wait`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: `Waiting for ${timeMs} milliseconds`,\n            type: \"string\",\n          },\n        },\n      });\n      await new Promise((resolve) => setTimeout(resolve, timeMs));\n      if (timeMs > 0) {\n        v3.recordAgentReplayStep({ type: \"wait\", timeMs });\n      }\n\n      // Take screenshot after wait in hybrid mode for visual feedback\n      if (mode === \"hybrid\") {\n        const page = await v3.context.awaitActivePage();\n        const screenshotBase64 = await waitAndCaptureScreenshot(page, 0);\n        return { success: true, waited: timeMs, screenshotBase64 };\n      }\n\n      return { success: true, waited: timeMs };\n    },\n    toModelOutput: (result) => {\n      if (result.success === false || result.error !== undefined) {\n        return {\n          type: \"content\",\n          value: [{ type: \"text\", text: JSON.stringify(result) }],\n        };\n      }\n\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            waited: result.waited,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/utils/actionMapping.d.ts

@@ -0,0 +1,3 @@
+import { AgentAction } from "../../types/public/agent.js";
+import { ActionMappingOptions } from "../../types/private/agent.js";
+export declare function mapToolResultToActions({ toolCallName, toolResult, args, reasoning, }: ActionMappingOptions): AgentAction[];

package/dist/esm/lib/v3/agent/utils/actionMapping.js

@@ -0,0 +1,100 @@
+/**
+ * Keys to exclude from tool outputs when mapping to actions.
+ * These are large data fields that shouldn't be included in the actions array.
+ * Users can access this data through result.messages if needed.
+ */
+const EXCLUDED_OUTPUT_KEYS = ["screenshotBase64"];
+/**
+ * Strips excluded keys (like screenshotBase64) from a tool output object.
+ */
+function stripExcludedKeys(output) {
+    const result = {};
+    for (const [key, value] of Object.entries(output)) {
+        if (!EXCLUDED_OUTPUT_KEYS.includes(key)) {
+            result[key] = value;
+        }
+    }
+    return result;
+}
+export function mapToolResultToActions({ toolCallName, toolResult, args, reasoning, }) {
+    switch (toolCallName) {
+        case "act":
+            return mapActToolResult(toolResult, args, reasoning);
+        case "fillForm":
+            return mapFillFormToolResult(toolResult, args, reasoning);
+        default:
+            return [createStandardAction(toolCallName, toolResult, args, reasoning)];
+    }
+}
+function mapActToolResult(toolResult, args, reasoning) {
+    if (!toolResult || typeof toolResult !== "object") {
+        return [createStandardAction("act", toolResult, args, reasoning)];
+    }
+    const result = toolResult;
+    // AI SDK wraps the tool result in an output property
+    const output = result.output || result;
+    // Extract playwright arguments if they exist
+    const action = {
+        type: "act",
+        reasoning,
+        taskCompleted: false,
+        ...args,
+    };
+    if (output.playwrightArguments) {
+        action.playwrightArguments = output.playwrightArguments;
+    }
+    return [action];
+}
+function mapFillFormToolResult(toolResult, args, reasoning) {
+    if (!toolResult || typeof toolResult !== "object") {
+        return [createStandardAction("fillForm", toolResult, args, reasoning)];
+    }
+    const result = toolResult;
+    // AI SDK wraps the tool result in an output property
+    const output = result.output || result;
+    const observeResults = Array.isArray(output?.playwrightArguments)
+        ? output.playwrightArguments
+        : [];
+    const actions = [];
+    actions.push({
+        type: "fillForm",
+        reasoning,
+        taskCompleted: false,
+        ...args,
+    });
+    for (const observeResult of observeResults) {
+        actions.push({
+            type: "act",
+            reasoning: "acting from fillform tool",
+            taskCompleted: false,
+            playwrightArguments: observeResult,
+        });
+    }
+    return actions;
+}
+function createStandardAction(toolCallName, toolResult, args, reasoning) {
+    const action = {
+        type: toolCallName,
+        reasoning,
+        taskCompleted: toolCallName === "done" ? args?.taskComplete : false,
+        ...args,
+    };
+    // For screenshot tool, exclude base64 data and just indicate a screenshot was taken,
+    // if somebody really wants the base64 data, they can access it through messages
+    if (toolCallName === "screenshot") {
+        action.result = "screenshotTaken";
+        return action;
+    }
+    // Spread the output from the tool result if it exists
+    // Exclude ariaTree tool result as it is very large and unnecessary
+    if (toolCallName !== "ariaTree" && toolResult) {
+        const result = toolResult;
+        const output = result.output;
+        if (output && typeof output === "object" && !Array.isArray(output)) {
+            const cleanedOutput = stripExcludedKeys(output);
+            Object.assign(action, cleanedOutput);
+        }
+    }
+    return action;
+}
+//# sourceMappingURL=actionMapping.js.map