@browserbasehq/orca 3.0.9-alpha-4 → 3.1.0-patch.0

package/dist/esm/lib/v3/agent/tools/search.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAgCxB,KAAK,UAAU,kBAAkB,CAAC,KAAa;IAC7C,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAC1B,oDAAoD,YAAY,EAAE,EAClE;YACE,MAAM,EAAE,KAAK;YACb,OAAO,EAAE;gBACP,MAAM,EAAE,kBAAkB;gBAC1B,iBAAiB,EAAE,MAAM;gBACzB,sBAAsB,EAAE,OAAO,CAAC,GAAG,CAAC,aAAc;aACnD;SACF,CACF,CAAC;QAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,OAAO;gBACL,KAAK,EAAE,oBAAoB,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE;gBACnE,IAAI,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;aACtB,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAqB,CAAC;QACzD,MAAM,OAAO,GAAwB,EAAE,CAAC;QAExC,IAAI,IAAI,EAAE,GAAG,EAAE,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1D,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;gBAChD,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;oBAC3B,OAAO,CAAC,IAAI,CAAC;wBACX,KAAK,EAAE,IAAI,CAAC,KAAK;wBACjB,GAAG,EAAE,IAAI,CAAC,GAAG;wBACb,WAAW,EAAE,IAAI,CAAC,WAAW;qBAC9B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,EAAE,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;QACrC,OAAO;YACL,KAAK,EAAE,4BAA6B,KAAe,CAAC,OAAO,EAAE;YAC7D,IAAI,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;SACtB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,EAAM,EAAE,EAAE,CACnC,IAAI,CAAC;IACH,WAAW,EACT,wSAAwS;IAC1S,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;KACtE,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;QAC3B,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC;oBAChC,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAE/C,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,KAAK;YAClB,mBAAmB,EAAE,EAAE,KAAK,EAAE;YAC9B,OAAO,EACL,MAAM,CAAC,KAAK,IAAI,SAAS,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,UAAU;SACtE,CAAC,CAAC;QAEH,OAAO;YACL,GAAG,MAAM;YACT,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;IACJ,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport interface BraveSearchResult {\n  title: string;\n  url: string;\n  description?: string;\n}\n\ninterface SearchResponse {\n  data?: {\n    results: BraveSearchResult[];\n  };\n  error?: string;\n}\n\ninterface BraveWebResult {\n  title?: string;\n  url?: string;\n  description?: string;\n  age?: string;\n  meta_url?: {\n    favicon?: string;\n  };\n}\n\ninterface BraveApiResponse {\n  web?: {\n    results?: BraveWebResult[];\n  };\n}\n\nasync function performBraveSearch(query: string): Promise<SearchResponse> {\n  try {\n    const encodedQuery = encodeURIComponent(query);\n    const response = await fetch(\n      `https://api.search.brave.com/res/v1/web/search?q=${encodedQuery}`,\n      {\n        method: \"GET\",\n        headers: {\n          Accept: \"application/json\",\n          \"Accept-Encoding\": \"gzip\",\n          \"X-Subscription-Token\": process.env.BRAVE_API_KEY!,\n        },\n      },\n    );\n\n    if (!response.ok) {\n      return {\n        error: `Brave API error: ${response.status} ${response.statusText}`,\n        data: { results: [] },\n      };\n    }\n\n    const data = (await response.json()) as BraveApiResponse;\n    const results: BraveSearchResult[] = [];\n\n    if (data?.web?.results && Array.isArray(data.web.results)) {\n      for (const item of data.web.results.slice(0, 5)) {\n        if (item.title && item.url) {\n          results.push({\n            title: item.title,\n            url: item.url,\n            description: item.description,\n          });\n        }\n      }\n    }\n\n    return { data: { results } };\n  } catch (error) {\n    console.error(\"Search error\", error);\n    return {\n      error: `Error performing search: ${(error as Error).message}`,\n      data: { results: [] },\n    };\n  }\n}\n\nexport const searchTool = (v3: V3) =>\n  tool({\n    description:\n      \"Perform a web search and returns results. Use this tool when you need information from the web or when you are unsure of the exact URL you want to navigate to. This can be used to find the ideal entry point, resulting in a task that is easier to complete due to starting further in the process.\",\n    inputSchema: z.object({\n      query: z.string().describe(\"The search query to look for on the web\"),\n    }),\n    execute: async ({ query }) => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: search`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: JSON.stringify({ query }),\n            type: \"object\",\n          },\n        },\n      });\n\n      const result = await performBraveSearch(query);\n\n      v3.recordAgentReplayStep({\n        type: \"search\",\n        instruction: query,\n        playwrightArguments: { query },\n        message:\n          result.error ?? `Found ${result.data?.results.length ?? 0} results`,\n      });\n\n      return {\n        ...result,\n        timestamp: Date.now(),\n      };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/think.d.ts

@@ -0,0 +1,6 @@
+export declare const thinkTool: () => import("ai").Tool<{
+    reasoning: string;
+}, {
+    acknowledged: boolean;
+    message: string;
+}>;

package/dist/esm/lib/v3/agent/tools/think.js

@@ -0,0 +1,24 @@
+import { tool } from "ai";
+import { z } from "zod";
+export const thinkTool = () => tool({
+    description: `Use this tool to think through complex problems or plan a sequence of steps. This is for internal reasoning only and doesn't perform any actions. Use this to:
+
+1. Plan a multi-step approach before taking action
+2. Break down complex tasks
+3. Reason through edge cases
+4. Evaluate options when you're unsure what to do next
+
+The output is only visible to you; use it to track your own reasoning process.`,
+    inputSchema: z.object({
+        reasoning: z
+            .string()
+            .describe("Your step-by-step reasoning or planning process. Be as detailed as needed."),
+    }),
+    execute: async ({ reasoning }) => {
+        return {
+            acknowledged: true,
+            message: reasoning,
+        };
+    },
+});
+//# sourceMappingURL=think.js.map

package/dist/esm/lib/v3/agent/tools/think.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"think.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/think.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,SAAS,GAAG,GAAG,EAAE,CAC5B,IAAI,CAAC;IACH,WAAW,EAAE;;;;;;;+EAO8D;IAC3E,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC;aACT,MAAM,EAAE;aACR,QAAQ,CACP,4EAA4E,CAC7E;KACJ,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE;QAC/B,OAAO;YACL,YAAY,EAAE,IAAI;YAClB,OAAO,EAAE,SAAS;SACnB,CAAC;IACJ,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\n\nexport const thinkTool = () =>\n  tool({\n    description: `Use this tool to think through complex problems or plan a sequence of steps. This is for internal reasoning only and doesn't perform any actions. Use this to:\n\n1. Plan a multi-step approach before taking action\n2. Break down complex tasks\n3. Reason through edge cases\n4. Evaluate options when you're unsure what to do next\n\nThe output is only visible to you; use it to track your own reasoning process.`,\n    inputSchema: z.object({\n      reasoning: z\n        .string()\n        .describe(\n          \"Your step-by-step reasoning or planning process. Be as detailed as needed.\",\n        ),\n    }),\n    execute: async ({ reasoning }) => {\n      return {\n        acknowledged: true,\n        message: reasoning,\n      };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/type.d.ts

@@ -0,0 +1,7 @@
+import type { V3 } from "../../v3.js";
+import type { TypeToolResult } from "../../types/public/agent.js";
+export declare const typeTool: (v3: V3, provider?: string) => import("ai").Tool<{
+    describe: string;
+    text: string;
+    coordinates: number[];
+}, TypeToolResult>;

package/dist/esm/lib/v3/agent/tools/type.js

@@ -0,0 +1,106 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { processCoordinates } from "../utils/coordinateNormalization.js";
+import { ensureXPath } from "../utils/xpath.js";
+import { waitAndCaptureScreenshot } from "../utils/screenshotHandler.js";
+export const typeTool = (v3, provider) => tool({
+    description: "Type text into an element using its coordinates. This will click the element and then type the text into it (this is the most reliable way to type into an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)",
+    inputSchema: z.object({
+        describe: z
+            .string()
+            .describe("Describe the element to type into in a short, specific phrase that mentions the element type and a good visual description"),
+        text: z.string().describe("The text to type into the element"),
+        coordinates: z
+            .array(z.number())
+            .describe("The (x, y) coordinates to type into the element"),
+    }),
+    execute: async ({ describe, coordinates, text, }) => {
+        try {
+            const page = await v3.context.awaitActivePage();
+            const processed = processCoordinates(coordinates[0], coordinates[1], provider, v3);
+            v3.logger({
+                category: "agent",
+                message: `Agent calling tool: type`,
+                level: 1,
+                auxiliary: {
+                    arguments: {
+                        value: JSON.stringify({ describe, text }),
+                        type: "object",
+                    },
+                },
+            });
+            // Only request XPath when caching is enabled to avoid unnecessary computation
+            const shouldCollectXpath = v3.isAgentReplayActive();
+            const xpath = await page.click(processed.x, processed.y, {
+                returnXpath: shouldCollectXpath,
+            });
+            await page.type(text);
+            const screenshotBase64 = await waitAndCaptureScreenshot(page);
+            // Record as an "act" step with proper Action for deterministic replay (only when caching)
+            if (shouldCollectXpath) {
+                const normalizedXpath = ensureXPath(xpath);
+                if (normalizedXpath) {
+                    const action = {
+                        selector: normalizedXpath,
+                        description: describe,
+                        method: "type",
+                        arguments: [text],
+                    };
+                    v3.recordAgentReplayStep({
+                        type: "act",
+                        instruction: describe,
+                        actions: [action],
+                        actionDescription: describe,
+                    });
+                }
+            }
+            return {
+                success: true,
+                describe,
+                text,
+                screenshotBase64,
+            };
+        }
+        catch (error) {
+            return {
+                success: false,
+                error: `Error typing: ${error.message}`,
+            };
+        }
+    },
+    toModelOutput: (result) => {
+        if (result.success) {
+            const content = [
+                {
+                    type: "text",
+                    text: JSON.stringify({
+                        success: result.success,
+                        describe: result.describe,
+                        text: result.text,
+                    }),
+                },
+            ];
+            if (result.screenshotBase64) {
+                content.push({
+                    type: "media",
+                    mediaType: "image/png",
+                    data: result.screenshotBase64,
+                });
+            }
+            return { type: "content", value: content };
+        }
+        return {
+            type: "content",
+            value: [
+                {
+                    type: "text",
+                    text: JSON.stringify({
+                        success: result.success,
+                        error: result.error,
+                    }),
+                },
+            ],
+        };
+    },
+});
+//# sourceMappingURL=type.js.map

package/dist/esm/lib/v3/agent/tools/type.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"type.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/type.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CACpD,IAAI,CAAC;IACH,WAAW,EACT,2QAA2Q;IAC7Q,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,QAAQ,EAAE,CAAC;aACR,MAAM,EAAE;aACR,QAAQ,CACP,4HAA4H,CAC7H;QACH,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;QAC9D,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,CAAC,iDAAiD,CAAC;KAC/D,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,QAAQ,EACR,WAAW,EACX,IAAI,GACL,EAA2B,EAAE;QAC5B,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YAEF,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,0BAA0B;gBACnC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;wBACzC,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,8EAA8E;YAC9E,MAAM,kBAAkB,GAAG,EAAE,CAAC,mBAAmB,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,EAAE;gBACvD,WAAW,EAAE,kBAAkB;aAChC,CAAC,CAAC;YAEH,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAEtB,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;YAE9D,0FAA0F;YAC1F,IAAI,kBAAkB,EAAE,CAAC;gBACvB,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;gBAC3C,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,MAAM,GAAW;wBACrB,QAAQ,EAAE,eAAe;wBACzB,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,MAAM;wBACd,SAAS,EAAE,CAAC,IAAI,CAAC;qBAClB,CAAC;oBACF,EAAE,CAAC,qBAAqB,CAAC;wBACvB,IAAI,EAAE,KAAK;wBACX,WAAW,EAAE,QAAQ;wBACrB,OAAO,EAAE,CAAC,MAAM,CAAC;wBACjB,iBAAiB,EAAE,QAAQ;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,IAAI;gBACJ,gBAAgB;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,iBAAkB,KAAe,CAAC,OAAO,EAAE;aACnD,CAAC;QACJ,CAAC;IACH,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,OAAO,GAA6B;gBACxC;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;wBACzB,IAAI,EAAE,MAAM,CAAC,IAAI;qBAClB,CAAC;iBACH;aACF,CAAC;YACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,OAAO;oBACb,SAAS,EAAE,WAAW;oBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;iBAC9B,CAAC,CAAC;YACL,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAC7C,CAAC;QACD,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;wBACvB,KAAK,EAAE,MAAM,CAAC,KAAK;qBACpB,CAAC;iBACH;aACF;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type {\n  TypeToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { ensureXPath } from \"../utils/xpath.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const typeTool = (v3: V3, provider?: string) =>\n  tool({\n    description:\n      \"Type text into an element using its coordinates. This will click the element and then type the text into it (this is the most reliable way to type into an element, always use this over act, unless the element is not visible in the screenshot, but shown in ariaTree)\",\n    inputSchema: z.object({\n      describe: z\n        .string()\n        .describe(\n          \"Describe the element to type into in a short, specific phrase that mentions the element type and a good visual description\",\n        ),\n      text: z.string().describe(\"The text to type into the element\"),\n      coordinates: z\n        .array(z.number())\n        .describe(\"The (x, y) coordinates to type into the element\"),\n    }),\n    execute: async ({\n      describe,\n      coordinates,\n      text,\n    }): Promise<TypeToolResult> => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: type`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ describe, text }),\n              type: \"object\",\n            },\n          },\n        });\n\n        // Only request XPath when caching is enabled to avoid unnecessary computation\n        const shouldCollectXpath = v3.isAgentReplayActive();\n        const xpath = await page.click(processed.x, processed.y, {\n          returnXpath: shouldCollectXpath,\n        });\n\n        await page.type(text);\n\n        const screenshotBase64 = await waitAndCaptureScreenshot(page);\n\n        // Record as an \"act\" step with proper Action for deterministic replay (only when caching)\n        if (shouldCollectXpath) {\n          const normalizedXpath = ensureXPath(xpath);\n          if (normalizedXpath) {\n            const action: Action = {\n              selector: normalizedXpath,\n              description: describe,\n              method: \"type\",\n              arguments: [text],\n            };\n            v3.recordAgentReplayStep({\n              type: \"act\",\n              instruction: describe,\n              actions: [action],\n              actionDescription: describe,\n            });\n          }\n        }\n\n        return {\n          success: true,\n          describe,\n          text,\n          screenshotBase64,\n        };\n      } catch (error) {\n        return {\n          success: false,\n          error: `Error typing: ${(error as Error).message}`,\n        };\n      }\n    },\n    toModelOutput: (result) => {\n      if (result.success) {\n        const content: ModelOutputContentItem[] = [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              describe: result.describe,\n              text: result.text,\n            }),\n          },\n        ];\n        if (result.screenshotBase64) {\n          content.push({\n            type: \"media\",\n            mediaType: \"image/png\",\n            data: result.screenshotBase64,\n          });\n        }\n        return { type: \"content\", value: content };\n      }\n      return {\n        type: \"content\",\n        value: [\n          {\n            type: \"text\",\n            text: JSON.stringify({\n              success: result.success,\n              error: result.error,\n            }),\n          },\n        ],\n      };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/wait.d.ts

@@ -0,0 +1,5 @@
+import type { V3 } from "../../v3.js";
+import type { AgentToolMode, WaitToolResult } from "../../types/public/agent.js";
+export declare const waitTool: (v3: V3, mode?: AgentToolMode) => import("ai").Tool<{
+    timeMs: number;
+}, WaitToolResult>;

package/dist/esm/lib/v3/agent/tools/wait.js

@@ -0,0 +1,53 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { waitAndCaptureScreenshot } from "../utils/screenshotHandler.js";
+export const waitTool = (v3, mode) => tool({
+    description: "Wait for a specified time",
+    inputSchema: z.object({
+        timeMs: z.number().describe("Time in milliseconds"),
+    }),
+    execute: async ({ timeMs }) => {
+        v3.logger({
+            category: "agent",
+            message: `Agent calling tool: wait`,
+            level: 1,
+            auxiliary: {
+                arguments: {
+                    value: `Waiting for ${timeMs} milliseconds`,
+                    type: "string",
+                },
+            },
+        });
+        await new Promise((resolve) => setTimeout(resolve, timeMs));
+        if (timeMs > 0) {
+            v3.recordAgentReplayStep({ type: "wait", timeMs });
+        }
+        // Take screenshot after wait in hybrid mode for visual feedback
+        if (mode === "hybrid") {
+            const page = await v3.context.awaitActivePage();
+            const screenshotBase64 = await waitAndCaptureScreenshot(page, 0);
+            return { success: true, waited: timeMs, screenshotBase64 };
+        }
+        return { success: true, waited: timeMs };
+    },
+    toModelOutput: (result) => {
+        const content = [
+            {
+                type: "text",
+                text: JSON.stringify({
+                    success: result.success,
+                    waited: result.waited,
+                }),
+            },
+        ];
+        if (result.screenshotBase64) {
+            content.push({
+                type: "media",
+                mediaType: "image/png",
+                data: result.screenshotBase64,
+            });
+        }
+        return { type: "content", value: content };
+    },
+});
+//# sourceMappingURL=wait.js.map

package/dist/esm/lib/v3/agent/tools/wait.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"wait.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/wait.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,IAAoB,EAAE,EAAE,CACvD,IAAI,CAAC;IACH,WAAW,EAAE,2BAA2B;IACxC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sBAAsB,CAAC;KACpD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAA2B,EAAE;QACrD,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,0BAA0B;YACnC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,eAAe,MAAM,eAAe;oBAC3C,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QACH,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;QAC5D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YACf,EAAE,CAAC,qBAAqB,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,gEAAgE;QAChE,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtB,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACjE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;QAC7D,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IAC3C,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,MAAM,EAAE,MAAM,CAAC,MAAM;iBACtB,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type {\n  AgentToolMode,\n  WaitToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\nexport const waitTool = (v3: V3, mode?: AgentToolMode) =>\n  tool({\n    description: \"Wait for a specified time\",\n    inputSchema: z.object({\n      timeMs: z.number().describe(\"Time in milliseconds\"),\n    }),\n    execute: async ({ timeMs }): Promise<WaitToolResult> => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: wait`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: `Waiting for ${timeMs} milliseconds`,\n            type: \"string\",\n          },\n        },\n      });\n      await new Promise((resolve) => setTimeout(resolve, timeMs));\n      if (timeMs > 0) {\n        v3.recordAgentReplayStep({ type: \"wait\", timeMs });\n      }\n\n      // Take screenshot after wait in hybrid mode for visual feedback\n      if (mode === \"hybrid\") {\n        const page = await v3.context.awaitActivePage();\n        const screenshotBase64 = await waitAndCaptureScreenshot(page, 0);\n        return { success: true, waited: timeMs, screenshotBase64 };\n      }\n\n      return { success: true, waited: timeMs };\n    },\n    toModelOutput: (result) => {\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            waited: result.waited,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/utils/actionMapping.d.ts

@@ -0,0 +1,3 @@
+import { AgentAction } from "../../types/public/agent.js";
+import { ActionMappingOptions } from "../../types/private/agent.js";
+export declare function mapToolResultToActions({ toolCallName, toolResult, args, reasoning, }: ActionMappingOptions): AgentAction[];

package/dist/esm/lib/v3/agent/utils/actionMapping.js

@@ -0,0 +1,100 @@
+/**
+ * Keys to exclude from tool outputs when mapping to actions.
+ * These are large data fields that shouldn't be included in the actions array.
+ * Users can access this data through result.messages if needed.
+ */
+const EXCLUDED_OUTPUT_KEYS = ["screenshotBase64"];
+/**
+ * Strips excluded keys (like screenshotBase64) from a tool output object.
+ */
+function stripExcludedKeys(output) {
+    const result = {};
+    for (const [key, value] of Object.entries(output)) {
+        if (!EXCLUDED_OUTPUT_KEYS.includes(key)) {
+            result[key] = value;
+        }
+    }
+    return result;
+}
+export function mapToolResultToActions({ toolCallName, toolResult, args, reasoning, }) {
+    switch (toolCallName) {
+        case "act":
+            return mapActToolResult(toolResult, args, reasoning);
+        case "fillForm":
+            return mapFillFormToolResult(toolResult, args, reasoning);
+        default:
+            return [createStandardAction(toolCallName, toolResult, args, reasoning)];
+    }
+}
+function mapActToolResult(toolResult, args, reasoning) {
+    if (!toolResult || typeof toolResult !== "object") {
+        return [createStandardAction("act", toolResult, args, reasoning)];
+    }
+    const result = toolResult;
+    // AI SDK wraps the tool result in an output property
+    const output = result.output || result;
+    // Extract playwright arguments if they exist
+    const action = {
+        type: "act",
+        reasoning,
+        taskCompleted: false,
+        ...args,
+    };
+    if (output.playwrightArguments) {
+        action.playwrightArguments = output.playwrightArguments;
+    }
+    return [action];
+}
+function mapFillFormToolResult(toolResult, args, reasoning) {
+    if (!toolResult || typeof toolResult !== "object") {
+        return [createStandardAction("fillForm", toolResult, args, reasoning)];
+    }
+    const result = toolResult;
+    // AI SDK wraps the tool result in an output property
+    const output = result.output || result;
+    const observeResults = Array.isArray(output?.playwrightArguments)
+        ? output.playwrightArguments
+        : [];
+    const actions = [];
+    actions.push({
+        type: "fillForm",
+        reasoning,
+        taskCompleted: false,
+        ...args,
+    });
+    for (const observeResult of observeResults) {
+        actions.push({
+            type: "act",
+            reasoning: "acting from fillform tool",
+            taskCompleted: false,
+            playwrightArguments: observeResult,
+        });
+    }
+    return actions;
+}
+function createStandardAction(toolCallName, toolResult, args, reasoning) {
+    const action = {
+        type: toolCallName,
+        reasoning,
+        taskCompleted: toolCallName === "done" ? args?.taskComplete : false,
+        ...args,
+    };
+    // For screenshot tool, exclude base64 data and just indicate a screenshot was taken,
+    // if somebody really wants the base64 data, they can access it through messages
+    if (toolCallName === "screenshot") {
+        action.result = "screenshotTaken";
+        return action;
+    }
+    // Spread the output from the tool result if it exists
+    // Exclude ariaTree tool result as it is very large and unnecessary
+    if (toolCallName !== "ariaTree" && toolResult) {
+        const result = toolResult;
+        const output = result.output;
+        if (output && typeof output === "object" && !Array.isArray(output)) {
+            const cleanedOutput = stripExcludedKeys(output);
+            Object.assign(action, cleanedOutput);
+        }
+    }
+    return action;
+}
+//# sourceMappingURL=actionMapping.js.map

package/dist/esm/lib/v3/agent/utils/actionMapping.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"actionMapping.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/actionMapping.ts"],"names":[],"mappings":"AAGA;;;;GAIG;AACH,MAAM,oBAAoB,GAAG,CAAC,kBAAkB,CAAU,CAAC;AAE3D;;GAEG;AACH,SAAS,iBAAiB,CACxB,MAA+B;IAE/B,MAAM,MAAM,GAA4B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAClD,IACE,CAAC,oBAAoB,CAAC,QAAQ,CAC5B,GAA4C,CAC7C,EACD,CAAC;YACD,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACtB,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,EACrC,YAAY,EACZ,UAAU,EACV,IAAI,EACJ,SAAS,GACY;IACrB,QAAQ,YAAY,EAAE,CAAC;QACrB,KAAK,KAAK;YACR,OAAO,gBAAgB,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;QACvD,KAAK,UAAU;YACb,OAAO,qBAAqB,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;QAC5D;YACE,OAAO,CAAC,oBAAoB,CAAC,YAAY,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;IAC7E,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CACvB,UAAmB,EACnB,IAA6B,EAC7B,SAAkB;IAElB,IAAI,CAAC,UAAU,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;QAClD,OAAO,CAAC,oBAAoB,CAAC,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,MAAM,GAAG,UAAqC,CAAC;IAErD,qDAAqD;IACrD,MAAM,MAAM,GAAI,MAAM,CAAC,MAAkC,IAAI,MAAM,CAAC;IAEpE,6CAA6C;IAC7C,MAAM,MAAM,GAAgB;QAC1B,IAAI,EAAE,KAAK;QACX,SAAS;QACT,aAAa,EAAE,KAAK;QACpB,GAAG,IAAI;KACR,CAAC;IAEF,IAAI,MAAM,CAAC,mBAAmB,EAAE,CAAC;QAC/B,MAAM,CAAC,mBAAmB,GAAG,MAAM,CAAC,mBAAmB,CAAC;IAC1D,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,qBAAqB,CAC5B,UAAmB,EACnB,IAA6B,EAC7B,SAAkB;IAElB,IAAI,CAAC,UAAU,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;QAClD,OAAO,CAAC,oBAAoB,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,MAAM,GAAG,UAAqC,CAAC;IAErD,qDAAqD;IACrD,MAAM,MAAM,GAAI,MAAM,CAAC,MAAkC,IAAI,MAAM,CAAC;IAEpE,MAAM,cAAc,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,mBAAmB,CAAC;QAC/D,CAAC,CAAC,MAAM,CAAC,mBAAmB;QAC5B,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,OAAO,GAAkB,EAAE,CAAC;IAElC,OAAO,CAAC,IAAI,CAAC;QACX,IAAI,EAAE,UAAU;QAChB,SAAS;QACT,aAAa,EAAE,KAAK;QACpB,GAAG,IAAI;KACR,CAAC,CAAC;IAEH,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;QAC3C,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,KAAK;YACX,SAAS,EAAE,2BAA2B;YACtC,aAAa,EAAE,KAAK;YACpB,mBAAmB,EAAE,aAAa;SACnC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,oBAAoB,CAC3B,YAAoB,EACpB,UAAmB,EACnB,IAA6B,EAC7B,SAAkB;IAElB,MAAM,MAAM,GAAgB;QAC1B,IAAI,EAAE,YAAY;QAClB,SAAS;QACT,aAAa,EACX,YAAY,KAAK,MAAM,CAAC,CAAC,CAAE,IAAI,EAAE,YAAwB,CAAC,CAAC,CAAC,KAAK;QACnE,GAAG,IAAI;KACR,CAAC;IAEF,qFAAqF;IACrF,gFAAgF;IAChF,IAAI,YAAY,KAAK,YAAY,EAAE,CAAC;QAClC,MAAM,CAAC,MAAM,GAAG,iBAAiB,CAAC;QAClC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,sDAAsD;IACtD,mEAAmE;IACnE,IAAI,YAAY,KAAK,UAAU,IAAI,UAAU,EAAE,CAAC;QAC9C,MAAM,MAAM,GAAG,UAAkC,CAAC;QAClD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAE7B,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACnE,MAAM,aAAa,GAAG,iBAAiB,CACrC,MAAiC,CAClC,CAAC;YACF,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC","sourcesContent":["import { AgentAction } from \"../../types/public/agent.js\";\nimport { ActionMappingOptions } from \"../../types/private/agent.js\";\n\n/**\n * Keys to exclude from tool outputs when mapping to actions.\n * These are large data fields that shouldn't be included in the actions array.\n * Users can access this data through result.messages if needed.\n */\nconst EXCLUDED_OUTPUT_KEYS = [\"screenshotBase64\"] as const;\n\n/**\n * Strips excluded keys (like screenshotBase64) from a tool output object.\n */\nfunction stripExcludedKeys(\n  output: Record<string, unknown>,\n): Record<string, unknown> {\n  const result: Record<string, unknown> = {};\n  for (const [key, value] of Object.entries(output)) {\n    if (\n      !EXCLUDED_OUTPUT_KEYS.includes(\n        key as (typeof EXCLUDED_OUTPUT_KEYS)[number],\n      )\n    ) {\n      result[key] = value;\n    }\n  }\n  return result;\n}\n\nexport function mapToolResultToActions({\n  toolCallName,\n  toolResult,\n  args,\n  reasoning,\n}: ActionMappingOptions): AgentAction[] {\n  switch (toolCallName) {\n    case \"act\":\n      return mapActToolResult(toolResult, args, reasoning);\n    case \"fillForm\":\n      return mapFillFormToolResult(toolResult, args, reasoning);\n    default:\n      return [createStandardAction(toolCallName, toolResult, args, reasoning)];\n  }\n}\n\nfunction mapActToolResult(\n  toolResult: unknown,\n  args: Record<string, unknown>,\n  reasoning?: string,\n): AgentAction[] {\n  if (!toolResult || typeof toolResult !== \"object\") {\n    return [createStandardAction(\"act\", toolResult, args, reasoning)];\n  }\n\n  const result = toolResult as Record<string, unknown>;\n\n  // AI SDK wraps the tool result in an output property\n  const output = (result.output as Record<string, unknown>) || result;\n\n  // Extract playwright arguments if they exist\n  const action: AgentAction = {\n    type: \"act\",\n    reasoning,\n    taskCompleted: false,\n    ...args,\n  };\n\n  if (output.playwrightArguments) {\n    action.playwrightArguments = output.playwrightArguments;\n  }\n\n  return [action];\n}\n\nfunction mapFillFormToolResult(\n  toolResult: unknown,\n  args: Record<string, unknown>,\n  reasoning?: string,\n): AgentAction[] {\n  if (!toolResult || typeof toolResult !== \"object\") {\n    return [createStandardAction(\"fillForm\", toolResult, args, reasoning)];\n  }\n\n  const result = toolResult as Record<string, unknown>;\n\n  // AI SDK wraps the tool result in an output property\n  const output = (result.output as Record<string, unknown>) || result;\n\n  const observeResults = Array.isArray(output?.playwrightArguments)\n    ? output.playwrightArguments\n    : [];\n\n  const actions: AgentAction[] = [];\n\n  actions.push({\n    type: \"fillForm\",\n    reasoning,\n    taskCompleted: false,\n    ...args,\n  });\n\n  for (const observeResult of observeResults) {\n    actions.push({\n      type: \"act\",\n      reasoning: \"acting from fillform tool\",\n      taskCompleted: false,\n      playwrightArguments: observeResult,\n    });\n  }\n\n  return actions;\n}\n\nfunction createStandardAction(\n  toolCallName: string,\n  toolResult: unknown,\n  args: Record<string, unknown>,\n  reasoning?: string,\n): AgentAction {\n  const action: AgentAction = {\n    type: toolCallName,\n    reasoning,\n    taskCompleted:\n      toolCallName === \"done\" ? (args?.taskComplete as boolean) : false,\n    ...args,\n  };\n\n  // For screenshot tool, exclude base64 data and just indicate a screenshot was taken,\n  // if somebody really wants the base64 data, they can access it through messages\n  if (toolCallName === \"screenshot\") {\n    action.result = \"screenshotTaken\";\n    return action;\n  }\n\n  // Spread the output from the tool result if it exists\n  // Exclude ariaTree tool result as it is very large and unnecessary\n  if (toolCallName !== \"ariaTree\" && toolResult) {\n    const result = toolResult as { output?: unknown };\n    const output = result.output;\n\n    if (output && typeof output === \"object\" && !Array.isArray(output)) {\n      const cleanedOutput = stripExcludedKeys(\n        output as Record<string, unknown>,\n      );\n      Object.assign(action, cleanedOutput);\n    }\n  }\n\n  return action;\n}\n"]}

package/dist/esm/lib/v3/agent/utils/coordinateNormalization.d.ts

@@ -0,0 +1,13 @@
+import type { V3 } from "../../v3.js";
+export declare function isGoogleProvider(provider?: string): boolean;
+export declare function normalizeGoogleCoordinates(x: number, y: number, viewport: {
+    width: number;
+    height: number;
+}): {
+    x: number;
+    y: number;
+};
+export declare function processCoordinates(x: number, y: number, provider?: string, v3?: V3): {
+    x: number;
+    y: number;
+};

package/dist/esm/lib/v3/agent/utils/coordinateNormalization.js

@@ -0,0 +1,28 @@
+// Default viewport for advancedStealth mode
+const STEALTH_VIEWPORT = { width: 1288, height: 711 };
+export function isGoogleProvider(provider) {
+    if (!provider)
+        return false;
+    return provider.toLowerCase().includes("google");
+}
+// Google returns coordinates in a 0-1000 range, we need to normalize
+// them to the viewport dimensions
+export function normalizeGoogleCoordinates(x, y, viewport) {
+    const clampedX = Math.min(999, Math.max(0, x));
+    const clampedY = Math.min(999, Math.max(0, y));
+    return {
+        x: Math.floor((clampedX / 1000) * viewport.width),
+        y: Math.floor((clampedY / 1000) * viewport.height),
+    };
+}
+export function processCoordinates(x, y, provider, v3) {
+    if (isGoogleProvider(provider) && v3) {
+        // advancedStealth uses fixed viewport, otherwise use configured viewport
+        const viewport = v3.isAdvancedStealth
+            ? STEALTH_VIEWPORT
+            : v3.configuredViewport;
+        return normalizeGoogleCoordinates(x, y, viewport);
+    }
+    return { x, y };
+}
+//# sourceMappingURL=coordinateNormalization.js.map

package/dist/esm/lib/v3/agent/utils/coordinateNormalization.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"coordinateNormalization.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/coordinateNormalization.ts"],"names":[],"mappings":"AAEA,4CAA4C;AAC5C,MAAM,gBAAgB,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;AAEtD,MAAM,UAAU,gBAAgB,CAAC,QAAiB;IAChD,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5B,OAAO,QAAQ,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AACnD,CAAC;AAED,qEAAqE;AACrE,kCAAkC;AAClC,MAAM,UAAU,0BAA0B,CACxC,CAAS,EACT,CAAS,EACT,QAA2C;IAE3C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/C,OAAO;QACL,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC;QACjD,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;KACnD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,CAAS,EACT,CAAS,EACT,QAAiB,EACjB,EAAO;IAEP,IAAI,gBAAgB,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QACrC,yEAAyE;QACzE,MAAM,QAAQ,GAAG,EAAE,CAAC,iBAAiB;YACnC,CAAC,CAAC,gBAAgB;YAClB,CAAC,CAAC,EAAE,CAAC,kBAAkB,CAAC;QAC1B,OAAO,0BAA0B,CAAC,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;AAClB,CAAC","sourcesContent":["import type { V3 } from \"../../v3.js\";\n\n// Default viewport for advancedStealth mode\nconst STEALTH_VIEWPORT = { width: 1288, height: 711 };\n\nexport function isGoogleProvider(provider?: string): boolean {\n  if (!provider) return false;\n  return provider.toLowerCase().includes(\"google\");\n}\n\n// Google returns coordinates in a 0-1000 range, we need to normalize\n// them to the viewport dimensions\nexport function normalizeGoogleCoordinates(\n  x: number,\n  y: number,\n  viewport: { width: number; height: number },\n): { x: number; y: number } {\n  const clampedX = Math.min(999, Math.max(0, x));\n  const clampedY = Math.min(999, Math.max(0, y));\n  return {\n    x: Math.floor((clampedX / 1000) * viewport.width),\n    y: Math.floor((clampedY / 1000) * viewport.height),\n  };\n}\n\nexport function processCoordinates(\n  x: number,\n  y: number,\n  provider?: string,\n  v3?: V3,\n): { x: number; y: number } {\n  if (isGoogleProvider(provider) && v3) {\n    // advancedStealth uses fixed viewport, otherwise use configured viewport\n    const viewport = v3.isAdvancedStealth\n      ? STEALTH_VIEWPORT\n      : v3.configuredViewport;\n    return normalizeGoogleCoordinates(x, y, viewport);\n  }\n  return { x, y };\n}\n"]}

package/dist/esm/lib/v3/agent/utils/cuaKeyMapping.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Universal key mapping utility for converting various key representations
+ * to Playwright-compatible key names. Used by all CUA clients and handlers.
+ */
+/**
+ * Maps a key name from various formats to Playwright-compatible format
+ * @param key The key name in any supported format
+ * @returns The Playwright-compatible key name
+ */
+export declare function mapKeyToPlaywright(key: string): string;

package/dist/esm/lib/v3/agent/utils/cuaKeyMapping.js

@@ -0,0 +1,62 @@
+/**
+ * Universal key mapping utility for converting various key representations
+ * to Playwright-compatible key names. Used by all CUA clients and handlers.
+ */
+/**
+ * map of key variations to Playwright key names
+ * This handles keys from both Anthropic and OpenAI CUA APIs
+ */
+const KEY_MAP = {
+    ENTER: "Enter",
+    RETURN: "Enter",
+    ESCAPE: "Escape",
+    ESC: "Escape",
+    BACKSPACE: "Backspace",
+    TAB: "Tab",
+    SPACE: " ",
+    DELETE: "Delete",
+    DEL: "Delete",
+    ARROWUP: "ArrowUp",
+    ARROWDOWN: "ArrowDown",
+    ARROWLEFT: "ArrowLeft",
+    ARROWRIGHT: "ArrowRight",
+    ARROW_UP: "ArrowUp",
+    ARROW_DOWN: "ArrowDown",
+    ARROW_LEFT: "ArrowLeft",
+    ARROW_RIGHT: "ArrowRight",
+    UP: "ArrowUp",
+    DOWN: "ArrowDown",
+    LEFT: "ArrowLeft",
+    RIGHT: "ArrowRight",
+    SHIFT: "Shift",
+    CONTROL: "Control",
+    CTRL: "Control",
+    ALT: "Alt",
+    OPTION: "Alt", // macOS alternative name
+    META: "Meta",
+    COMMAND: "Meta", // macOS
+    CMD: "Meta", // macOS shorthand
+    SUPER: "Meta", // Linux
+    WINDOWS: "Meta", // Windows
+    WIN: "Meta", // Windows shorthand
+    HOME: "Home",
+    END: "End",
+    PAGEUP: "PageUp",
+    PAGEDOWN: "PageDown",
+    PAGE_UP: "PageUp",
+    PAGE_DOWN: "PageDown",
+    PGUP: "PageUp",
+    PGDN: "PageDown",
+};
+/**
+ * Maps a key name from various formats to Playwright-compatible format
+ * @param key The key name in any supported format
+ * @returns The Playwright-compatible key name
+ */
+export function mapKeyToPlaywright(key) {
+    if (!key)
+        return key;
+    const upperKey = key.toUpperCase();
+    return KEY_MAP[upperKey] || key;
+}
+//# sourceMappingURL=cuaKeyMapping.js.map

package/dist/esm/lib/v3/agent/utils/cuaKeyMapping.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cuaKeyMapping.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/cuaKeyMapping.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;GAGG;AACH,MAAM,OAAO,GAA2B;IACtC,KAAK,EAAE,OAAO;IACd,MAAM,EAAE,OAAO;IACf,MAAM,EAAE,QAAQ;IAChB,GAAG,EAAE,QAAQ;IACb,SAAS,EAAE,WAAW;IACtB,GAAG,EAAE,KAAK;IACV,KAAK,EAAE,GAAG;IACV,MAAM,EAAE,QAAQ;IAChB,GAAG,EAAE,QAAQ;IACb,OAAO,EAAE,SAAS;IAClB,SAAS,EAAE,WAAW;IACtB,SAAS,EAAE,WAAW;IACtB,UAAU,EAAE,YAAY;IACxB,QAAQ,EAAE,SAAS;IACnB,UAAU,EAAE,WAAW;IACvB,UAAU,EAAE,WAAW;IACvB,WAAW,EAAE,YAAY;IACzB,EAAE,EAAE,SAAS;IACb,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,WAAW;IACjB,KAAK,EAAE,YAAY;IACnB,KAAK,EAAE,OAAO;IACd,OAAO,EAAE,SAAS;IAClB,IAAI,EAAE,SAAS;IACf,GAAG,EAAE,KAAK;IACV,MAAM,EAAE,KAAK,EAAE,yBAAyB;IACxC,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,MAAM,EAAE,QAAQ;IACzB,GAAG,EAAE,MAAM,EAAE,kBAAkB;IAC/B,KAAK,EAAE,MAAM,EAAE,QAAQ;IACvB,OAAO,EAAE,MAAM,EAAE,UAAU;IAC3B,GAAG,EAAE,MAAM,EAAE,oBAAoB;IACjC,IAAI,EAAE,MAAM;IACZ,GAAG,EAAE,KAAK;IACV,MAAM,EAAE,QAAQ;IAChB,QAAQ,EAAE,UAAU;IACpB,OAAO,EAAE,QAAQ;IACjB,SAAS,EAAE,UAAU;IACrB,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,UAAU;CACjB,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC5C,IAAI,CAAC,GAAG;QAAE,OAAO,GAAG,CAAC;IACrB,MAAM,QAAQ,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;IACnC,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC;AAClC,CAAC","sourcesContent":["/**\n * Universal key mapping utility for converting various key representations\n * to Playwright-compatible key names. Used by all CUA clients and handlers.\n */\n\n/**\n * map of key variations to Playwright key names\n * This handles keys from both Anthropic and OpenAI CUA APIs\n */\nconst KEY_MAP: Record<string, string> = {\n  ENTER: \"Enter\",\n  RETURN: \"Enter\",\n  ESCAPE: \"Escape\",\n  ESC: \"Escape\",\n  BACKSPACE: \"Backspace\",\n  TAB: \"Tab\",\n  SPACE: \" \",\n  DELETE: \"Delete\",\n  DEL: \"Delete\",\n  ARROWUP: \"ArrowUp\",\n  ARROWDOWN: \"ArrowDown\",\n  ARROWLEFT: \"ArrowLeft\",\n  ARROWRIGHT: \"ArrowRight\",\n  ARROW_UP: \"ArrowUp\",\n  ARROW_DOWN: \"ArrowDown\",\n  ARROW_LEFT: \"ArrowLeft\",\n  ARROW_RIGHT: \"ArrowRight\",\n  UP: \"ArrowUp\",\n  DOWN: \"ArrowDown\",\n  LEFT: \"ArrowLeft\",\n  RIGHT: \"ArrowRight\",\n  SHIFT: \"Shift\",\n  CONTROL: \"Control\",\n  CTRL: \"Control\",\n  ALT: \"Alt\",\n  OPTION: \"Alt\", // macOS alternative name\n  META: \"Meta\",\n  COMMAND: \"Meta\", // macOS\n  CMD: \"Meta\", // macOS shorthand\n  SUPER: \"Meta\", // Linux\n  WINDOWS: \"Meta\", // Windows\n  WIN: \"Meta\", // Windows shorthand\n  HOME: \"Home\",\n  END: \"End\",\n  PAGEUP: \"PageUp\",\n  PAGEDOWN: \"PageDown\",\n  PAGE_UP: \"PageUp\",\n  PAGE_DOWN: \"PageDown\",\n  PGUP: \"PageUp\",\n  PGDN: \"PageDown\",\n};\n\n/**\n * Maps a key name from various formats to Playwright-compatible format\n * @param key The key name in any supported format\n * @returns The Playwright-compatible key name\n */\nexport function mapKeyToPlaywright(key: string): string {\n  if (!key) return key;\n  const upperKey = key.toUpperCase();\n  return KEY_MAP[upperKey] || key;\n}\n"]}

package/dist/esm/lib/v3/agent/utils/googleCustomToolHandler.d.ts

@@ -0,0 +1,25 @@
+import { Part, FunctionCall, FunctionDeclaration } from "@google/genai";
+import { ToolSet } from "ai";
+import { LogLine } from "../../types/public/logs.js";
+/**
+ * Result of executing a custom tool for Google CUA
+ */
+export interface CustomToolExecutionResult {
+    functionResponse: Part;
+    success: boolean;
+}
+/**
+ * Execute a custom tool and format the response for Google's API
+ * This handles tool execution, result formatting, and error handling
+ * specific to Google's function response format
+ */
+export declare function executeGoogleCustomTool(toolName: string, toolArgs: Record<string, unknown>, tools: ToolSet, functionCall: FunctionCall, logger: (message: LogLine) => void): Promise<CustomToolExecutionResult>;
+/**
+ * Check if a function call is a custom tool
+ */
+export declare function isCustomTool(functionCall: FunctionCall, tools?: ToolSet): boolean;
+/**
+ * Convert ToolSet to Google's FunctionDeclaration array
+ * Handles the conversion of Zod schemas to Google's parameter format
+ */
+export declare function convertToolSetToFunctionDeclarations(tools: ToolSet): FunctionDeclaration[];