@browserbasehq/orca 3.0.9-alpha-3 → 3.1.0-patch.0

package/dist/esm/lib/v3/agent/tools/goto.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"goto.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/goto.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,EAAE,CACjC,IAAI,CAAC;IACH,WAAW,EAAE,4BAA4B;IACzC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,wBAAwB,CAAC;KACnD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;QACzB,IAAI,CAAC;YACH,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,0BAA0B;gBACnC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,GAAG;wBACV,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;YAC5C,EAAE,CAAC,qBAAqB,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;YACnE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;QAChC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QACpE,CAAC;IACH,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport const gotoTool = (v3: V3) =>\n  tool({\n    description: \"Navigate to a specific URL\",\n    inputSchema: z.object({\n      url: z.string().describe(\"The URL to navigate to\"),\n    }),\n    execute: async ({ url }) => {\n      try {\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: goto`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: url,\n              type: \"string\",\n            },\n          },\n        });\n        const page = await v3.context.awaitActivePage();\n        await page.goto(url, { waitUntil: \"load\" });\n        v3.recordAgentReplayStep({ type: \"goto\", url, waitUntil: \"load\" });\n        return { success: true, url };\n      } catch (error) {\n        return { success: false, error: error?.message ?? String(error) };\n      }\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/index.d.ts

@@ -0,0 +1,92 @@
+import { gotoTool } from "./goto.js";
+import { actTool } from "./act.js";
+import { screenshotTool } from "./screenshot.js";
+import { waitTool } from "./wait.js";
+import { navBackTool } from "./navback.js";
+import { ariaTreeTool } from "./ariaTree.js";
+import { fillFormTool } from "./fillform.js";
+import { scrollTool, scrollVisionTool } from "./scroll.js";
+import { extractTool } from "./extract.js";
+import { clickTool } from "./click.js";
+import { typeTool } from "./type.js";
+import { dragAndDropTool } from "./dragAndDrop.js";
+import { clickAndHoldTool } from "./clickAndHold.js";
+import { keysTool } from "./keys.js";
+import { fillFormVisionTool } from "./fillFormVision.js";
+import { thinkTool } from "./think.js";
+import { searchTool } from "./search.js";
+import type { ToolSet, InferUITools } from "ai";
+import type { V3 } from "../../v3.js";
+import type { LogLine } from "../../types/public/logs.js";
+import type { AgentToolMode, AgentModelConfig } from "../../types/public/agent.js";
+export interface V3AgentToolOptions {
+    executionModel?: string | AgentModelConfig;
+    logger?: (message: LogLine) => void;
+    /**
+     * Tool mode determines which set of tools are available.
+     * - 'dom' (default): Uses DOM-based tools (act, fillForm) - removes coordinate-based tools
+     * - 'hybrid': Uses coordinate-based tools (click, type, dragAndDrop, etc.) - removes fillForm
+     */
+    mode?: AgentToolMode;
+    /**
+     * The model provider. Used for model-specific coordinate handling
+     */
+    provider?: string;
+    /**
+     * Tools to exclude from the available toolset.
+     * These tools will be filtered out after mode-based filtering.
+     */
+    excludeTools?: string[];
+}
+export declare function createAgentTools(v3: V3, options?: V3AgentToolOptions): ToolSet;
+export type AgentTools = ReturnType<typeof createAgentTools>;
+/**
+ * Type map of all agent tools for strong typing of tool calls and results.
+ * Note: `search` is optional as it's only available when BRAVE_API_KEY is configured.
+ */
+export type AgentToolTypesMap = {
+    act: ReturnType<typeof actTool>;
+    ariaTree: ReturnType<typeof ariaTreeTool>;
+    click: ReturnType<typeof clickTool>;
+    clickAndHold: ReturnType<typeof clickAndHoldTool>;
+    dragAndDrop: ReturnType<typeof dragAndDropTool>;
+    extract: ReturnType<typeof extractTool>;
+    fillForm: ReturnType<typeof fillFormTool>;
+    fillFormVision: ReturnType<typeof fillFormVisionTool>;
+    goto: ReturnType<typeof gotoTool>;
+    keys: ReturnType<typeof keysTool>;
+    navback: ReturnType<typeof navBackTool>;
+    screenshot: ReturnType<typeof screenshotTool>;
+    scroll: ReturnType<typeof scrollTool> | ReturnType<typeof scrollVisionTool>;
+    search?: ReturnType<typeof searchTool>;
+    think: ReturnType<typeof thinkTool>;
+    type: ReturnType<typeof typeTool>;
+    wait: ReturnType<typeof waitTool>;
+};
+/**
+ * Inferred UI tools type for type-safe tool inputs and outputs.
+ * Use with UIMessage for full type safety in UI contexts.
+ */
+export type AgentUITools = InferUITools<AgentToolTypesMap>;
+/**
+ * Union type for all possible agent tool calls.
+ * Provides type-safe access to tool call arguments.
+ */
+export type AgentToolCall = {
+    [K in keyof AgentToolTypesMap]: {
+        toolName: K;
+        toolCallId: string;
+        args: AgentUITools[K]["input"];
+    };
+}[keyof AgentToolTypesMap];
+/**
+ * Union type for all possible agent tool results.
+ * Provides type-safe access to tool result values.
+ */
+export type AgentToolResult = {
+    [K in keyof AgentToolTypesMap]: {
+        toolName: K;
+        toolCallId: string;
+        result: AgentUITools[K]["output"];
+    };
+}[keyof AgentToolTypesMap];

package/dist/esm/lib/v3/agent/tools/index.js

@@ -0,0 +1,74 @@
+import { gotoTool } from "./goto.js";
+import { actTool } from "./act.js";
+import { screenshotTool } from "./screenshot.js";
+import { waitTool } from "./wait.js";
+import { navBackTool } from "./navback.js";
+import { ariaTreeTool } from "./ariaTree.js";
+import { fillFormTool } from "./fillform.js";
+import { scrollTool, scrollVisionTool } from "./scroll.js";
+import { extractTool } from "./extract.js";
+import { clickTool } from "./click.js";
+import { typeTool } from "./type.js";
+import { dragAndDropTool } from "./dragAndDrop.js";
+import { clickAndHoldTool } from "./clickAndHold.js";
+import { keysTool } from "./keys.js";
+import { fillFormVisionTool } from "./fillFormVision.js";
+import { thinkTool } from "./think.js";
+import { searchTool } from "./search.js";
+/**
+ * Filters tools based on mode and explicit exclusions.
+ * - 'dom' mode: Removes coordinate-based tools (click, type, dragAndDrop, clickAndHold, fillFormVision)
+ * - 'hybrid' mode: Removes DOM-based form tool (fillForm) in favor of coordinate-based fillFormVision
+ * - excludeTools: Additional tools to remove from the toolset
+ */
+function filterTools(tools, mode, excludeTools) {
+    const filtered = { ...tools };
+    // Mode-based filtering
+    if (mode === "hybrid") {
+        delete filtered.fillForm;
+    }
+    else {
+        // DOM mode (default)
+        delete filtered.click;
+        delete filtered.type;
+        delete filtered.dragAndDrop;
+        delete filtered.clickAndHold;
+        delete filtered.fillFormVision;
+    }
+    if (excludeTools) {
+        for (const toolName of excludeTools) {
+            delete filtered[toolName];
+        }
+    }
+    return filtered;
+}
+export function createAgentTools(v3, options) {
+    const executionModel = options?.executionModel;
+    const mode = options?.mode ?? "dom";
+    const provider = options?.provider;
+    const excludeTools = options?.excludeTools;
+    const allTools = {
+        act: actTool(v3, executionModel),
+        ariaTree: ariaTreeTool(v3),
+        click: clickTool(v3, provider),
+        clickAndHold: clickAndHoldTool(v3, provider),
+        dragAndDrop: dragAndDropTool(v3, provider),
+        extract: extractTool(v3, executionModel),
+        fillForm: fillFormTool(v3, executionModel),
+        fillFormVision: fillFormVisionTool(v3, provider),
+        goto: gotoTool(v3),
+        keys: keysTool(v3),
+        navback: navBackTool(v3),
+        screenshot: screenshotTool(v3),
+        scroll: mode === "hybrid" ? scrollVisionTool(v3, provider) : scrollTool(v3),
+        think: thinkTool(),
+        type: typeTool(v3, provider),
+        wait: waitTool(v3, mode),
+    };
+    // Only include search tool if BRAVE_API_KEY is configured
+    if (process.env.BRAVE_API_KEY) {
+        allTools.search = searchTool(v3);
+    }
+    return filterTools(allTools, mode, excludeTools);
+}
+//# sourceMappingURL=index.js.map

package/dist/esm/lib/v3/agent/tools/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AACnC,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AA8BzC;;;;;GAKG;AACH,SAAS,WAAW,CAClB,KAAc,EACd,IAAmB,EACnB,YAAuB;IAEvB,MAAM,QAAQ,GAAY,EAAE,GAAG,KAAK,EAAE,CAAC;IAEvC,uBAAuB;IACvB,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtB,OAAO,QAAQ,CAAC,QAAQ,CAAC;IAC3B,CAAC;SAAM,CAAC;QACN,qBAAqB;QACrB,OAAO,QAAQ,CAAC,KAAK,CAAC;QACtB,OAAO,QAAQ,CAAC,IAAI,CAAC;QACrB,OAAO,QAAQ,CAAC,WAAW,CAAC;QAC5B,OAAO,QAAQ,CAAC,YAAY,CAAC;QAC7B,OAAO,QAAQ,CAAC,cAAc,CAAC;IACjC,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,KAAK,MAAM,QAAQ,IAAI,YAAY,EAAE,CAAC;YACpC,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,EAAM,EAAE,OAA4B;IACnE,MAAM,cAAc,GAAG,OAAO,EAAE,cAAc,CAAC;IAC/C,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,KAAK,CAAC;IACpC,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,CAAC;IACnC,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,CAAC;IAE3C,MAAM,QAAQ,GAAY;QACxB,GAAG,EAAE,OAAO,CAAC,EAAE,EAAE,cAAc,CAAC;QAChC,QAAQ,EAAE,YAAY,CAAC,EAAE,CAAC;QAC1B,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC9B,YAAY,EAAE,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC5C,WAAW,EAAE,eAAe,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC1C,OAAO,EAAE,WAAW,CAAC,EAAE,EAAE,cAAc,CAAC;QACxC,QAAQ,EAAE,YAAY,CAAC,EAAE,EAAE,cAAc,CAAC;QAC1C,cAAc,EAAE,kBAAkB,CAAC,EAAE,EAAE,QAAQ,CAAC;QAChD,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;QAClB,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;QAClB,OAAO,EAAE,WAAW,CAAC,EAAE,CAAC;QACxB,UAAU,EAAE,cAAc,CAAC,EAAE,CAAC;QAC9B,MAAM,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3E,KAAK,EAAE,SAAS,EAAE;QAClB,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC;QAC5B,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;KACzB,CAAC;IAEF,0DAA0D;IAC1D,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;QAC9B,QAAQ,CAAC,MAAM,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;IACnC,CAAC;IAED,OAAO,WAAW,CAAC,QAAQ,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;AACnD,CAAC","sourcesContent":["import { gotoTool } from \"./goto.js\";\nimport { actTool } from \"./act.js\";\nimport { screenshotTool } from \"./screenshot.js\";\nimport { waitTool } from \"./wait.js\";\nimport { navBackTool } from \"./navback.js\";\nimport { ariaTreeTool } from \"./ariaTree.js\";\nimport { fillFormTool } from \"./fillform.js\";\nimport { scrollTool, scrollVisionTool } from \"./scroll.js\";\nimport { extractTool } from \"./extract.js\";\nimport { clickTool } from \"./click.js\";\nimport { typeTool } from \"./type.js\";\nimport { dragAndDropTool } from \"./dragAndDrop.js\";\nimport { clickAndHoldTool } from \"./clickAndHold.js\";\nimport { keysTool } from \"./keys.js\";\nimport { fillFormVisionTool } from \"./fillFormVision.js\";\nimport { thinkTool } from \"./think.js\";\nimport { searchTool } from \"./search.js\";\n\nimport type { ToolSet, InferUITools } from \"ai\";\nimport type { V3 } from \"../../v3.js\";\nimport type { LogLine } from \"../../types/public/logs.js\";\nimport type {\n  AgentToolMode,\n  AgentModelConfig,\n} from \"../../types/public/agent.js\";\n\nexport interface V3AgentToolOptions {\n  executionModel?: string | AgentModelConfig;\n  logger?: (message: LogLine) => void;\n  /**\n   * Tool mode determines which set of tools are available.\n   * - 'dom' (default): Uses DOM-based tools (act, fillForm) - removes coordinate-based tools\n   * - 'hybrid': Uses coordinate-based tools (click, type, dragAndDrop, etc.) - removes fillForm\n   */\n  mode?: AgentToolMode;\n  /**\n   * The model provider. Used for model-specific coordinate handling\n   */\n  provider?: string;\n  /**\n   * Tools to exclude from the available toolset.\n   * These tools will be filtered out after mode-based filtering.\n   */\n  excludeTools?: string[];\n}\n\n/**\n * Filters tools based on mode and explicit exclusions.\n * - 'dom' mode: Removes coordinate-based tools (click, type, dragAndDrop, clickAndHold, fillFormVision)\n * - 'hybrid' mode: Removes DOM-based form tool (fillForm) in favor of coordinate-based fillFormVision\n * - excludeTools: Additional tools to remove from the toolset\n */\nfunction filterTools(\n  tools: ToolSet,\n  mode: AgentToolMode,\n  excludeTools?: string[],\n): ToolSet {\n  const filtered: ToolSet = { ...tools };\n\n  // Mode-based filtering\n  if (mode === \"hybrid\") {\n    delete filtered.fillForm;\n  } else {\n    // DOM mode (default)\n    delete filtered.click;\n    delete filtered.type;\n    delete filtered.dragAndDrop;\n    delete filtered.clickAndHold;\n    delete filtered.fillFormVision;\n  }\n\n  if (excludeTools) {\n    for (const toolName of excludeTools) {\n      delete filtered[toolName];\n    }\n  }\n\n  return filtered;\n}\n\nexport function createAgentTools(v3: V3, options?: V3AgentToolOptions) {\n  const executionModel = options?.executionModel;\n  const mode = options?.mode ?? \"dom\";\n  const provider = options?.provider;\n  const excludeTools = options?.excludeTools;\n\n  const allTools: ToolSet = {\n    act: actTool(v3, executionModel),\n    ariaTree: ariaTreeTool(v3),\n    click: clickTool(v3, provider),\n    clickAndHold: clickAndHoldTool(v3, provider),\n    dragAndDrop: dragAndDropTool(v3, provider),\n    extract: extractTool(v3, executionModel),\n    fillForm: fillFormTool(v3, executionModel),\n    fillFormVision: fillFormVisionTool(v3, provider),\n    goto: gotoTool(v3),\n    keys: keysTool(v3),\n    navback: navBackTool(v3),\n    screenshot: screenshotTool(v3),\n    scroll: mode === \"hybrid\" ? scrollVisionTool(v3, provider) : scrollTool(v3),\n    think: thinkTool(),\n    type: typeTool(v3, provider),\n    wait: waitTool(v3, mode),\n  };\n\n  // Only include search tool if BRAVE_API_KEY is configured\n  if (process.env.BRAVE_API_KEY) {\n    allTools.search = searchTool(v3);\n  }\n\n  return filterTools(allTools, mode, excludeTools);\n}\n\nexport type AgentTools = ReturnType<typeof createAgentTools>;\n\n/**\n * Type map of all agent tools for strong typing of tool calls and results.\n * Note: `search` is optional as it's only available when BRAVE_API_KEY is configured.\n */\nexport type AgentToolTypesMap = {\n  act: ReturnType<typeof actTool>;\n  ariaTree: ReturnType<typeof ariaTreeTool>;\n  click: ReturnType<typeof clickTool>;\n  clickAndHold: ReturnType<typeof clickAndHoldTool>;\n  dragAndDrop: ReturnType<typeof dragAndDropTool>;\n  extract: ReturnType<typeof extractTool>;\n  fillForm: ReturnType<typeof fillFormTool>;\n  fillFormVision: ReturnType<typeof fillFormVisionTool>;\n  goto: ReturnType<typeof gotoTool>;\n  keys: ReturnType<typeof keysTool>;\n  navback: ReturnType<typeof navBackTool>;\n  screenshot: ReturnType<typeof screenshotTool>;\n  scroll: ReturnType<typeof scrollTool> | ReturnType<typeof scrollVisionTool>;\n  search?: ReturnType<typeof searchTool>;\n  think: ReturnType<typeof thinkTool>;\n  type: ReturnType<typeof typeTool>;\n  wait: ReturnType<typeof waitTool>;\n};\n\n/**\n * Inferred UI tools type for type-safe tool inputs and outputs.\n * Use with UIMessage for full type safety in UI contexts.\n */\nexport type AgentUITools = InferUITools<AgentToolTypesMap>;\n\n/**\n * Union type for all possible agent tool calls.\n * Provides type-safe access to tool call arguments.\n */\nexport type AgentToolCall = {\n  [K in keyof AgentToolTypesMap]: {\n    toolName: K;\n    toolCallId: string;\n    args: AgentUITools[K][\"input\"];\n  };\n}[keyof AgentToolTypesMap];\n\n/**\n * Union type for all possible agent tool results.\n * Provides type-safe access to tool result values.\n */\nexport type AgentToolResult = {\n  [K in keyof AgentToolTypesMap]: {\n    toolName: K;\n    toolCallId: string;\n    result: AgentUITools[K][\"output\"];\n  };\n}[keyof AgentToolTypesMap];\n"]}

package/dist/esm/lib/v3/agent/tools/keys.d.ts

@@ -0,0 +1,24 @@
+import type { V3 } from "../../v3.js";
+export declare const keysTool: (v3: V3) => import("ai").Tool<{
+    method: "type" | "press";
+    value: string;
+    repeat?: number;
+}, {
+    success: boolean;
+    method: "type";
+    value: string;
+    times: number;
+    error?: undefined;
+} | {
+    success: boolean;
+    method: "press";
+    value: string;
+    times: number;
+    error?: undefined;
+} | {
+    success: boolean;
+    error: string;
+    method?: undefined;
+    value?: undefined;
+    times?: undefined;
+}>;

package/dist/esm/lib/v3/agent/tools/keys.js

@@ -0,0 +1,60 @@
+import { tool } from "ai";
+import { z } from "zod";
+export const keysTool = (v3) => tool({
+    description: `Send keyboard input to the page without targeting a specific element. Unlike the type tool which clicks then types into coordinates, this sends keystrokes directly to wherever focus currently is.
+
+Use method="type" to enter text into the currently focused element. Preferred when: input is already focused, text needs to flow across multiple fields (e.g., verification codes)
+
+Use method="press" for navigation keys (Enter, Tab, Escape, Backspace, arrows) and keyboard shortcuts (Cmd+A, Ctrl+C, Shift+Tab).`,
+    inputSchema: z.object({
+        method: z.enum(["press", "type"]),
+        value: z
+            .string()
+            .describe("The text to type, or the key/combo to press (Enter, Tab, Cmd+A)"),
+        repeat: z.number().optional(),
+    }),
+    execute: async ({ method, value, repeat }) => {
+        try {
+            const page = await v3.context.awaitActivePage();
+            v3.logger({
+                category: "agent",
+                message: `Agent calling tool: keys`,
+                level: 1,
+                auxiliary: {
+                    arguments: {
+                        value: JSON.stringify({ method, value, repeat }),
+                        type: "object",
+                    },
+                },
+            });
+            const times = Math.max(1, repeat ?? 1);
+            if (method === "type") {
+                for (let i = 0; i < times; i++) {
+                    await page.type(value, { delay: 100 });
+                }
+                v3.recordAgentReplayStep({
+                    type: "keys",
+                    instruction: `type "${value}"`,
+                    playwrightArguments: { method, text: value, times },
+                });
+                return { success: true, method, value, times };
+            }
+            if (method === "press") {
+                for (let i = 0; i < times; i++) {
+                    await page.keyPress(value, { delay: 100 });
+                }
+                v3.recordAgentReplayStep({
+                    type: "keys",
+                    instruction: `press ${value}`,
+                    playwrightArguments: { method, keys: value, times },
+                });
+                return { success: true, method, value, times };
+            }
+            return { success: false, error: `Unsupported method: ${method}` };
+        }
+        catch (error) {
+            return { success: false, error: error.message };
+        }
+    },
+});
+//# sourceMappingURL=keys.js.map

package/dist/esm/lib/v3/agent/tools/keys.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"keys.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/keys.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAM,EAAE,EAAE,CACjC,IAAI,CAAC;IACH,WAAW,EAAE;;;;kIAIiH;IAC9H,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACjC,KAAK,EAAE,CAAC;aACL,MAAM,EAAE;aACR,QAAQ,CACP,iEAAiE,CAClE;QACH,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAC9B,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE;QAC3C,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;YAChD,EAAE,CAAC,MAAM,CAAC;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO,EAAE,0BAA0B;gBACnC,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE;oBACT,SAAS,EAAE;wBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;wBAChD,IAAI,EAAE,QAAQ;qBACf;iBACF;aACF,CAAC,CAAC;YAEH,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;YAEvC,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;gBACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC/B,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBACzC,CAAC;gBACD,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,MAAM;oBACZ,WAAW,EAAE,SAAS,KAAK,GAAG;oBAC9B,mBAAmB,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE;iBACpD,CAAC,CAAC;gBACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;YACjD,CAAC;YAED,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;gBACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC7C,CAAC;gBACD,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,MAAM;oBACZ,WAAW,EAAE,SAAS,KAAK,EAAE;oBAC7B,mBAAmB,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE;iBACpD,CAAC,CAAC;gBACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;YACjD,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,MAAM,EAAE,EAAE,CAAC;QACpE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAG,KAAe,CAAC,OAAO,EAAE,CAAC;QAC7D,CAAC;IACH,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport const keysTool = (v3: V3) =>\n  tool({\n    description: `Send keyboard input to the page without targeting a specific element. Unlike the type tool which clicks then types into coordinates, this sends keystrokes directly to wherever focus currently is.\n\nUse method=\"type\" to enter text into the currently focused element. Preferred when: input is already focused, text needs to flow across multiple fields (e.g., verification codes)\n\nUse method=\"press\" for navigation keys (Enter, Tab, Escape, Backspace, arrows) and keyboard shortcuts (Cmd+A, Ctrl+C, Shift+Tab).`,\n    inputSchema: z.object({\n      method: z.enum([\"press\", \"type\"]),\n      value: z\n        .string()\n        .describe(\n          \"The text to type, or the key/combo to press (Enter, Tab, Cmd+A)\",\n        ),\n      repeat: z.number().optional(),\n    }),\n    execute: async ({ method, value, repeat }) => {\n      try {\n        const page = await v3.context.awaitActivePage();\n        v3.logger({\n          category: \"agent\",\n          message: `Agent calling tool: keys`,\n          level: 1,\n          auxiliary: {\n            arguments: {\n              value: JSON.stringify({ method, value, repeat }),\n              type: \"object\",\n            },\n          },\n        });\n\n        const times = Math.max(1, repeat ?? 1);\n\n        if (method === \"type\") {\n          for (let i = 0; i < times; i++) {\n            await page.type(value, { delay: 100 });\n          }\n          v3.recordAgentReplayStep({\n            type: \"keys\",\n            instruction: `type \"${value}\"`,\n            playwrightArguments: { method, text: value, times },\n          });\n          return { success: true, method, value, times };\n        }\n\n        if (method === \"press\") {\n          for (let i = 0; i < times; i++) {\n            await page.keyPress(value, { delay: 100 });\n          }\n          v3.recordAgentReplayStep({\n            type: \"keys\",\n            instruction: `press ${value}`,\n            playwrightArguments: { method, keys: value, times },\n          });\n          return { success: true, method, value, times };\n        }\n\n        return { success: false, error: `Unsupported method: ${method}` };\n      } catch (error) {\n        return { success: false, error: (error as Error).message };\n      }\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/navback.d.ts

@@ -0,0 +1,6 @@
+import type { V3 } from "../../v3.js";
+export declare const navBackTool: (v3: V3) => import("ai").Tool<{
+    reasoningText: string;
+}, {
+    success: boolean;
+}>;

package/dist/esm/lib/v3/agent/tools/navback.js

@@ -0,0 +1,23 @@
+import { tool } from "ai";
+import { z } from "zod";
+export const navBackTool = (v3) => tool({
+    description: "Navigate back to the previous page",
+    inputSchema: z.object({
+        reasoningText: z.string().describe("Why you're going back"),
+    }),
+    execute: async () => {
+        v3.logger({
+            category: "agent",
+            message: `Agent calling tool: navback`,
+            level: 1,
+        });
+        const page = await v3.context.awaitActivePage();
+        await page.goBack({ waitUntil: "domcontentloaded" });
+        v3.recordAgentReplayStep({
+            type: "navback",
+            waitUntil: "domcontentloaded",
+        });
+        return { success: true };
+    },
+});
+//# sourceMappingURL=navback.js.map

package/dist/esm/lib/v3/agent/tools/navback.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"navback.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/navback.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,EAAM,EAAE,EAAE,CACpC,IAAI,CAAC;IACH,WAAW,EAAE,oCAAoC;IACjD,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;KAC5D,CAAC;IACF,OAAO,EAAE,KAAK,IAAI,EAAE;QAClB,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,6BAA6B;YACtC,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAChD,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACrD,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,SAAS;YACf,SAAS,EAAE,kBAAkB;SAC9B,CAAC,CAAC;QACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport const navBackTool = (v3: V3) =>\n  tool({\n    description: \"Navigate back to the previous page\",\n    inputSchema: z.object({\n      reasoningText: z.string().describe(\"Why you're going back\"),\n    }),\n    execute: async () => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: navback`,\n        level: 1,\n      });\n      const page = await v3.context.awaitActivePage();\n      await page.goBack({ waitUntil: \"domcontentloaded\" });\n      v3.recordAgentReplayStep({\n        type: \"navback\",\n        waitUntil: \"domcontentloaded\",\n      });\n      return { success: true };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/screenshot.d.ts

@@ -0,0 +1,6 @@
+import type { V3 } from "../../v3.js";
+export declare const screenshotTool: (v3: V3) => import("ai").Tool<Record<string, never>, {
+    base64: string;
+    timestamp: number;
+    pageUrl: string;
+}>;

package/dist/esm/lib/v3/agent/tools/screenshot.js

@@ -0,0 +1,26 @@
+import { tool } from "ai";
+import { z } from "zod";
+export const screenshotTool = (v3) => tool({
+    description: "Takes a screenshot (PNG) of the current page. Use this to quickly verify page state.",
+    inputSchema: z.object({}),
+    execute: async () => {
+        v3.logger({
+            category: "agent",
+            message: `Agent calling tool: screenshot`,
+            level: 1,
+        });
+        const page = await v3.context.awaitActivePage();
+        const buffer = await page.screenshot({ fullPage: false });
+        const pageUrl = page.url();
+        return {
+            base64: buffer.toString("base64"),
+            timestamp: Date.now(),
+            pageUrl,
+        };
+    },
+    toModelOutput: (result) => ({
+        type: "content",
+        value: [{ type: "media", mediaType: "image/png", data: result.base64 }],
+    }),
+});
+//# sourceMappingURL=screenshot.js.map

package/dist/esm/lib/v3/agent/tools/screenshot.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"screenshot.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/screenshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,EAAM,EAAE,EAAE,CACvC,IAAI,CAAC;IACH,WAAW,EACT,sFAAsF;IACxF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;IACzB,OAAO,EAAE,KAAK,IAAI,EAAE;QAClB,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,gCAAgC;YACzC,KAAK,EAAE,CAAC;SACT,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAChD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACjC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,OAAO;SACR,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC1B,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;KACxE,CAAC;CACH,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\n\nexport const screenshotTool = (v3: V3) =>\n  tool({\n    description:\n      \"Takes a screenshot (PNG) of the current page. Use this to quickly verify page state.\",\n    inputSchema: z.object({}),\n    execute: async () => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: screenshot`,\n        level: 1,\n      });\n      const page = await v3.context.awaitActivePage();\n      const buffer = await page.screenshot({ fullPage: false });\n      const pageUrl = page.url();\n      return {\n        base64: buffer.toString(\"base64\"),\n        timestamp: Date.now(),\n        pageUrl,\n      };\n    },\n    toModelOutput: (result) => ({\n      type: \"content\",\n      value: [{ type: \"media\", mediaType: \"image/png\", data: result.base64 }],\n    }),\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/scroll.d.ts

@@ -0,0 +1,19 @@
+import type { V3 } from "../../v3.js";
+import type { ScrollToolResult, ScrollVisionToolResult } from "../../types/public/agent.js";
+/**
+ * Simple scroll tool for DOM mode (non-grounding models).
+ * No coordinates - scrolls from viewport center.
+ */
+export declare const scrollTool: (v3: V3) => import("ai").Tool<{
+    direction: "up" | "down";
+    percentage?: number;
+}, ScrollToolResult>;
+/**
+ * Scroll tool for hybrid mode (grounding models).
+ * Supports optional coordinates for scrolling within nested scrollable elements.
+ */
+export declare const scrollVisionTool: (v3: V3, provider?: string) => import("ai").Tool<{
+    direction: "up" | "down";
+    coordinates?: number[];
+    percentage?: number;
+}, ScrollVisionToolResult>;

package/dist/esm/lib/v3/agent/tools/scroll.js

@@ -0,0 +1,142 @@
+import { tool } from "ai";
+import { z } from "zod";
+import { processCoordinates } from "../utils/coordinateNormalization.js";
+import { waitAndCaptureScreenshot } from "../utils/screenshotHandler.js";
+/**
+ * Simple scroll tool for DOM mode (non-grounding models).
+ * No coordinates - scrolls from viewport center.
+ */
+export const scrollTool = (v3) => tool({
+    description: "Scroll the page up or down by a percentage of the viewport height. Default is 80%, and what should be typically used for general page scrolling",
+    inputSchema: z.object({
+        direction: z.enum(["up", "down"]),
+        percentage: z.number().min(1).max(200).optional(),
+    }),
+    execute: async ({ direction, percentage = 80, }) => {
+        v3.logger({
+            category: "agent",
+            message: `Agent calling tool: scroll`,
+            level: 1,
+            auxiliary: {
+                arguments: {
+                    value: JSON.stringify({ direction, percentage }),
+                    type: "object",
+                },
+            },
+        });
+        const page = await v3.context.awaitActivePage();
+        const { w, h } = await page.mainFrame().evaluate("({ w: window.innerWidth, h: window.innerHeight })");
+        const scrollDistance = Math.round((h * percentage) / 100);
+        const cx = Math.floor(w / 2);
+        const cy = Math.floor(h / 2);
+        const deltaY = direction === "up" ? -scrollDistance : scrollDistance;
+        await page.scroll(cx, cy, 0, deltaY);
+        v3.recordAgentReplayStep({
+            type: "scroll",
+            deltaX: 0,
+            deltaY,
+            anchor: { x: cx, y: cy },
+        });
+        return {
+            success: true,
+            message: `Scrolled ${percentage}% ${direction} (${scrollDistance}px)`,
+            scrolledPixels: scrollDistance,
+        };
+    },
+    toModelOutput: (result) => {
+        return {
+            type: "json",
+            value: {
+                success: result.success,
+                message: result.message,
+                scrolledPixels: result.scrolledPixels,
+            },
+        };
+    },
+});
+/**
+ * Scroll tool for hybrid mode (grounding models).
+ * Supports optional coordinates for scrolling within nested scrollable elements.
+ */
+export const scrollVisionTool = (v3, provider) => tool({
+    description: `Scroll the page up or down. For general page scrolling, no coordinates needed. Only provide coordinates when scrolling inside a nested scrollable element (e.g., a dropdown menu, modal with overflow, or scrollable sidebar). Default is 80%, and what should be typically used for general page scrolling`,
+    inputSchema: z.object({
+        direction: z.enum(["up", "down"]),
+        coordinates: z
+            .array(z.number())
+            .optional()
+            .describe("Only use coordinates for scrolling inside a nested scrollable element - provide (x, y) within that element"),
+        percentage: z.number().min(1).max(200).optional(),
+    }),
+    execute: async ({ direction, coordinates, percentage = 80, }) => {
+        const page = await v3.context.awaitActivePage();
+        const { w, h } = await page.mainFrame().evaluate("({ w: window.innerWidth, h: window.innerHeight })");
+        // Process coordinates if provided, otherwise use viewport center
+        let cx;
+        let cy;
+        if (coordinates) {
+            const processed = processCoordinates(coordinates[0], coordinates[1], provider, v3);
+            cx = processed.x;
+            cy = processed.y;
+        }
+        else {
+            cx = Math.floor(w / 2);
+            cy = Math.floor(h / 2);
+        }
+        v3.logger({
+            category: "agent",
+            message: `Agent calling tool: scroll`,
+            level: 1,
+            auxiliary: {
+                arguments: {
+                    value: JSON.stringify({
+                        direction,
+                        coordinates,
+                        percentage,
+                        processed: { cx, cy },
+                    }),
+                    type: "object",
+                },
+            },
+        });
+        const scrollDistance = Math.round((h * percentage) / 100);
+        const deltaY = direction === "up" ? -scrollDistance : scrollDistance;
+        await page.scroll(cx, cy, 0, deltaY);
+        const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);
+        v3.recordAgentReplayStep({
+            type: "scroll",
+            deltaX: 0,
+            deltaY,
+            anchor: { x: cx, y: cy },
+        });
+        return {
+            success: true,
+            message: coordinates
+                ? `Scrolled ${percentage}% ${direction} at (${cx}, ${cy})`
+                : `Scrolled ${percentage}% ${direction}`,
+            scrolledPixels: scrollDistance,
+            screenshotBase64,
+        };
+    },
+    toModelOutput: (result) => {
+        const content = [
+            {
+                type: "text",
+                text: JSON.stringify({
+                    success: result.success,
+                    message: result.message,
+                    scrolledPixels: result.scrolledPixels,
+                }),
+            },
+        ];
+        if (result.screenshotBase64) {
+            content.push({
+                type: "media",
+                mediaType: "image/png",
+                data: result.screenshotBase64,
+            });
+        }
+        return { type: "content", value: content };
+    },
+});
+//# sourceMappingURL=scroll.js.map

package/dist/esm/lib/v3/agent/tools/scroll.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scroll.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/scroll.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AACzE,OAAO,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAEzE;;;GAGG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,EAAM,EAAE,EAAE,CACnC,IAAI,CAAC;IACH,WAAW,EACT,iJAAiJ;IACnJ,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,UAAU,GAAG,EAAE,GAChB,EAA6B,EAAE;QAC9B,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;oBAChD,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,YAAY,UAAU,KAAK,SAAS,KAAK,cAAc,KAAK;YACrE,cAAc,EAAE,cAAc;SAC/B,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE;gBACL,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;aACtC;SACF,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEL;;;GAGG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,EAAM,EAAE,QAAiB,EAAE,EAAE,CAC5D,IAAI,CAAC;IACH,WAAW,EAAE,6SAA6S;IAC1T,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC;QACpB,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjC,WAAW,EAAE,CAAC;aACX,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;aACjB,QAAQ,EAAE;aACV,QAAQ,CACP,4GAA4G,CAC7G;QACH,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;KAClD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,EACd,SAAS,EACT,WAAW,EACX,UAAU,GAAG,EAAE,GAChB,EAAmC,EAAE;QACpC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAEhD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAG7C,mDAAmD,CAAC,CAAC;QAExD,iEAAiE;QACjE,IAAI,EAAU,CAAC;QACf,IAAI,EAAU,CAAC;QACf,IAAI,WAAW,EAAE,CAAC;YAChB,MAAM,SAAS,GAAG,kBAAkB,CAClC,WAAW,CAAC,CAAC,CAAC,EACd,WAAW,CAAC,CAAC,CAAC,EACd,QAAQ,EACR,EAAE,CACH,CAAC;YACF,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;YACjB,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACvB,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzB,CAAC;QAED,EAAE,CAAC,MAAM,CAAC;YACR,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,4BAA4B;YACrC,KAAK,EAAE,CAAC;YACR,SAAS,EAAE;gBACT,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC;wBACpB,SAAS;wBACT,WAAW;wBACX,UAAU;wBACV,SAAS,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE;qBACtB,CAAC;oBACF,IAAI,EAAE,QAAQ;iBACf;aACF;SACF,CAAC,CAAC;QAEH,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC;QAErE,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAErC,MAAM,gBAAgB,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAEnE,EAAE,CAAC,qBAAqB,CAAC;YACvB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,CAAC;YACT,MAAM;YACN,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;SACzB,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,WAAW;gBAClB,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,QAAQ,EAAE,KAAK,EAAE,GAAG;gBAC1D,CAAC,CAAC,YAAY,UAAU,KAAK,SAAS,EAAE;YAC1C,cAAc,EAAE,cAAc;YAC9B,gBAAgB;SACjB,CAAC;IACJ,CAAC;IACD,aAAa,EAAE,CAAC,MAAM,EAAE,EAAE;QACxB,MAAM,OAAO,GAA6B;YACxC;gBACE,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,cAAc,EAAE,MAAM,CAAC,cAAc;iBACtC,CAAC;aACH;SACF,CAAC;QACF,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,WAAW;gBACtB,IAAI,EAAE,MAAM,CAAC,gBAAgB;aAC9B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type {\n  ScrollToolResult,\n  ScrollVisionToolResult,\n  ModelOutputContentItem,\n} from \"../../types/public/agent.js\";\nimport { processCoordinates } from \"../utils/coordinateNormalization.js\";\nimport { waitAndCaptureScreenshot } from \"../utils/screenshotHandler.js\";\n\n/**\n * Simple scroll tool for DOM mode (non-grounding models).\n * No coordinates - scrolls from viewport center.\n */\nexport const scrollTool = (v3: V3) =>\n  tool({\n    description:\n      \"Scroll the page up or down by a percentage of the viewport height. Default is 80%, and what should be typically used for general page scrolling\",\n    inputSchema: z.object({\n      direction: z.enum([\"up\", \"down\"]),\n      percentage: z.number().min(1).max(200).optional(),\n    }),\n    execute: async ({\n      direction,\n      percentage = 80,\n    }): Promise<ScrollToolResult> => {\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: scroll`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: JSON.stringify({ direction, percentage }),\n            type: \"object\",\n          },\n        },\n      });\n\n      const page = await v3.context.awaitActivePage();\n\n      const { w, h } = await page.mainFrame().evaluate<{\n        w: number;\n        h: number;\n      }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n      const scrollDistance = Math.round((h * percentage) / 100);\n      const cx = Math.floor(w / 2);\n      const cy = Math.floor(h / 2);\n      const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n      await page.scroll(cx, cy, 0, deltaY);\n\n      v3.recordAgentReplayStep({\n        type: \"scroll\",\n        deltaX: 0,\n        deltaY,\n        anchor: { x: cx, y: cy },\n      });\n\n      return {\n        success: true,\n        message: `Scrolled ${percentage}% ${direction} (${scrollDistance}px)`,\n        scrolledPixels: scrollDistance,\n      };\n    },\n    toModelOutput: (result) => {\n      return {\n        type: \"json\",\n        value: {\n          success: result.success,\n          message: result.message,\n          scrolledPixels: result.scrolledPixels,\n        },\n      };\n    },\n  });\n\n/**\n * Scroll tool for hybrid mode (grounding models).\n * Supports optional coordinates for scrolling within nested scrollable elements.\n */\nexport const scrollVisionTool = (v3: V3, provider?: string) =>\n  tool({\n    description: `Scroll the page up or down. For general page scrolling, no coordinates needed. Only provide coordinates when scrolling inside a nested scrollable element (e.g., a dropdown menu, modal with overflow, or scrollable sidebar). Default is 80%, and what should be typically used for general page scrolling`,\n    inputSchema: z.object({\n      direction: z.enum([\"up\", \"down\"]),\n      coordinates: z\n        .array(z.number())\n        .optional()\n        .describe(\n          \"Only use coordinates for scrolling inside a nested scrollable element - provide (x, y) within that element\",\n        ),\n      percentage: z.number().min(1).max(200).optional(),\n    }),\n    execute: async ({\n      direction,\n      coordinates,\n      percentage = 80,\n    }): Promise<ScrollVisionToolResult> => {\n      const page = await v3.context.awaitActivePage();\n\n      const { w, h } = await page.mainFrame().evaluate<{\n        w: number;\n        h: number;\n      }>(\"({ w: window.innerWidth, h: window.innerHeight })\");\n\n      // Process coordinates if provided, otherwise use viewport center\n      let cx: number;\n      let cy: number;\n      if (coordinates) {\n        const processed = processCoordinates(\n          coordinates[0],\n          coordinates[1],\n          provider,\n          v3,\n        );\n        cx = processed.x;\n        cy = processed.y;\n      } else {\n        cx = Math.floor(w / 2);\n        cy = Math.floor(h / 2);\n      }\n\n      v3.logger({\n        category: \"agent\",\n        message: `Agent calling tool: scroll`,\n        level: 1,\n        auxiliary: {\n          arguments: {\n            value: JSON.stringify({\n              direction,\n              coordinates,\n              percentage,\n              processed: { cx, cy },\n            }),\n            type: \"object\",\n          },\n        },\n      });\n\n      const scrollDistance = Math.round((h * percentage) / 100);\n      const deltaY = direction === \"up\" ? -scrollDistance : scrollDistance;\n\n      await page.scroll(cx, cy, 0, deltaY);\n\n      const screenshotBase64 = await waitAndCaptureScreenshot(page, 100);\n\n      v3.recordAgentReplayStep({\n        type: \"scroll\",\n        deltaX: 0,\n        deltaY,\n        anchor: { x: cx, y: cy },\n      });\n\n      return {\n        success: true,\n        message: coordinates\n          ? `Scrolled ${percentage}% ${direction} at (${cx}, ${cy})`\n          : `Scrolled ${percentage}% ${direction}`,\n        scrolledPixels: scrollDistance,\n        screenshotBase64,\n      };\n    },\n    toModelOutput: (result) => {\n      const content: ModelOutputContentItem[] = [\n        {\n          type: \"text\",\n          text: JSON.stringify({\n            success: result.success,\n            message: result.message,\n            scrolledPixels: result.scrolledPixels,\n          }),\n        },\n      ];\n      if (result.screenshotBase64) {\n        content.push({\n          type: \"media\",\n          mediaType: \"image/png\",\n          data: result.screenshotBase64,\n        });\n      }\n      return { type: \"content\", value: content };\n    },\n  });\n"]}

package/dist/esm/lib/v3/agent/tools/search.d.ts

@@ -0,0 +1,15 @@
+import type { V3 } from "../../v3.js";
+export interface BraveSearchResult {
+    title: string;
+    url: string;
+    description?: string;
+}
+export declare const searchTool: (v3: V3) => import("ai").Tool<{
+    query: string;
+}, {
+    timestamp: number;
+    data?: {
+        results: BraveSearchResult[];
+    };
+    error?: string;
+}>;

package/dist/esm/lib/v3/agent/tools/search.js

@@ -0,0 +1,73 @@
+import { tool } from "ai";
+import { z } from "zod";
+async function performBraveSearch(query) {
+    try {
+        const encodedQuery = encodeURIComponent(query);
+        const response = await fetch(`https://api.search.brave.com/res/v1/web/search?q=${encodedQuery}`, {
+            method: "GET",
+            headers: {
+                Accept: "application/json",
+                "Accept-Encoding": "gzip",
+                "X-Subscription-Token": process.env.BRAVE_API_KEY,
+            },
+        });
+        if (!response.ok) {
+            return {
+                error: `Brave API error: ${response.status} ${response.statusText}`,
+                data: { results: [] },
+            };
+        }
+        const data = (await response.json());
+        const results = [];
+        if (data?.web?.results && Array.isArray(data.web.results)) {
+            for (const item of data.web.results.slice(0, 5)) {
+                if (item.title && item.url) {
+                    results.push({
+                        title: item.title,
+                        url: item.url,
+                        description: item.description,
+                    });
+                }
+            }
+        }
+        return { data: { results } };
+    }
+    catch (error) {
+        console.error("Search error", error);
+        return {
+            error: `Error performing search: ${error.message}`,
+            data: { results: [] },
+        };
+    }
+}
+export const searchTool = (v3) => tool({
+    description: "Perform a web search and returns results. Use this tool when you need information from the web or when you are unsure of the exact URL you want to navigate to. This can be used to find the ideal entry point, resulting in a task that is easier to complete due to starting further in the process.",
+    inputSchema: z.object({
+        query: z.string().describe("The search query to look for on the web"),
+    }),
+    execute: async ({ query }) => {
+        v3.logger({
+            category: "agent",
+            message: `Agent calling tool: search`,
+            level: 1,
+            auxiliary: {
+                arguments: {
+                    value: JSON.stringify({ query }),
+                    type: "object",
+                },
+            },
+        });
+        const result = await performBraveSearch(query);
+        v3.recordAgentReplayStep({
+            type: "search",
+            instruction: query,
+            playwrightArguments: { query },
+            message: result.error ?? `Found ${result.data?.results.length ?? 0} results`,
+        });
+        return {
+            ...result,
+            timestamp: Date.now(),
+        };
+    },
+});
+//# sourceMappingURL=search.js.map