@browserbasehq/orca 3.1.0-patch.3 → 3.2.0-middleware.1

package/dist/cjs/lib/v3/handlers/v3CuaAgentHandler.js

@@ -0,0 +1,652 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.V3CuaAgentHandler = void 0;
+const index_js_1 = require("../understudy/a11y/snapshot/index.js");
+const AgentProvider_js_1 = require("../agent/AgentProvider.js");
+const GoogleCUAClient_js_1 = require("../agent/GoogleCUAClient.js");
+const OpenAICUAClient_js_1 = require("../agent/OpenAICUAClient.js");
+const cuaKeyMapping_js_1 = require("../agent/utils/cuaKeyMapping.js");
+const xpath_js_1 = require("../agent/utils/xpath.js");
+const methods_js_1 = require("../types/public/methods.js");
+const FlowLogger_js_1 = require("../flowlogger/FlowLogger.js");
+const utils_js_1 = require("../../utils.js");
+const sdkErrors_js_1 = require("../types/public/sdkErrors.js");
+const captchaSolver_js_1 = require("../agent/utils/captchaSolver.js");
+class V3CuaAgentHandler {
+    v3;
+    agent;
+    provider;
+    logger;
+    agentClient;
+    options;
+    highlightCursor;
+    captchaSolver = null;
+    captchaClickGuardRemaining = 0;
+    currentInstruction = "";
+    constructor(v3, logger, options, tools) {
+        this.v3 = v3;
+        this.logger = logger;
+        this.options = options;
+        this.provider = new AgentProvider_js_1.AgentProvider(logger);
+        const client = this.provider.getClient(options.modelName, options.clientOptions || {}, options.userProvidedInstructions, tools);
+        this.agentClient = client;
+        this.setupAgentClient();
+        this.agent = client;
+    }
+    /**
+     * Ensures the V3 context is still available (not closed).
+     * Throws StagehandClosedError if stagehand.close() was called.
+     */
+    ensureNotClosed() {
+        if (!this.v3.context) {
+            throw new sdkErrors_js_1.StagehandClosedError();
+        }
+    }
+    setupAgentClient() {
+        // Provide screenshots to the agent client
+        this.agentClient.setScreenshotProvider(async () => {
+            this.ensureNotClosed();
+            const page = await this.v3.context.awaitActivePage();
+            const screenshotBuffer = await page.screenshot({ fullPage: false });
+            return screenshotBuffer.toString("base64"); // base64 png
+        });
+        // Provide action executor
+        this.agentClient.setActionHandler(async (action) => {
+            this.ensureNotClosed();
+            // Wait for captcha solver to finish before executing action
+            if (this.captchaSolver) {
+                if (this.captchaSolver.isSolving()) {
+                    this.logger({
+                        category: "agent",
+                        message: "Captcha detected — waiting for Browserbase to solve it before continuing",
+                        level: 1,
+                    });
+                }
+                await this.captchaSolver.waitIfSolving();
+                this.handleCaptchaSolveResult(this.captchaSolver.consumeSolveResult());
+            }
+            action.pageUrl = (await this.v3.context.awaitActivePage()).url();
+            if (await this.shouldSkipSolvedCaptchaInteraction(action)) {
+                this.captchaClickGuardRemaining = Math.max(0, this.captchaClickGuardRemaining - 1);
+                this.agentClient.addContextNote(`The captcha has already been solved automatically. Do not click the captcha checkbox, widget, or challenge again. Continue with the original task outside the captcha area. Original task: ${this.currentInstruction}`);
+                this.logger({
+                    category: "agent",
+                    message: "Skipped click on solved captcha widget — injected follow-up guidance",
+                    level: 1,
+                });
+                return;
+            }
+            const defaultDelay = 500;
+            const waitBetween = this.options.clientOptions?.waitBetweenActions ||
+                defaultDelay;
+            try {
+                // Try to inject cursor before each action if enabled
+                if (this.highlightCursor) {
+                    try {
+                        await this.injectCursor();
+                    }
+                    catch {
+                        // Ignore cursor injection failures
+                    }
+                }
+                await new Promise((r) => setTimeout(r, 300));
+                // Skip logging for screenshot actions - they're no-ops, the actual
+                // Page.screenshot in captureAndSendScreenshot() is logged separately
+                const shouldLog = action.type !== "screenshot";
+                if (shouldLog) {
+                    await FlowLogger_js_1.FlowLogger.runWithLogging({
+                        eventType: `V3Cua${(0, utils_js_1.toTitleCase)(action.type)}`, // e.g. "V3CuaClick"
+                        data: {
+                            target: this.computePointerTarget(action),
+                        },
+                    }, async (loggedAction) => await this.executeAction(loggedAction), [action]);
+                }
+                else {
+                    await this.executeAction(action);
+                }
+                action.timestamp = Date.now();
+                await new Promise((r) => setTimeout(r, waitBetween));
+                try {
+                    await this.captureAndSendScreenshot();
+                }
+                catch (e) {
+                    this.logger({
+                        category: "agent",
+                        message: `Warning: Failed to take screenshot after action: ${String(e?.message ?? e)}`,
+                        level: 1,
+                    });
+                }
+            }
+            catch (error) {
+                const msg = error?.message ?? String(error);
+                this.logger({
+                    category: "agent",
+                    message: `Error executing action ${action.type}: ${msg}`,
+                    level: 0,
+                });
+                throw error;
+            }
+        });
+        void this.updateClientViewport();
+        void this.updateClientUrl();
+    }
+    setSafetyConfirmationHandler(handler) {
+        if (this.agentClient instanceof GoogleCUAClient_js_1.GoogleCUAClient ||
+            this.agentClient instanceof OpenAICUAClient_js_1.OpenAICUAClient) {
+            this.agentClient.setSafetyConfirmationHandler(handler);
+        }
+    }
+    async execute(optionsOrInstruction) {
+        const options = typeof optionsOrInstruction === "string"
+            ? { instruction: optionsOrInstruction }
+            : optionsOrInstruction;
+        this.setSafetyConfirmationHandler(options.callbacks?.onSafetyConfirmation);
+        this.highlightCursor = options.highlightCursor !== false;
+        this.currentInstruction = options.instruction;
+        // Redirect if blank
+        const page = await this.v3.context.awaitActivePage();
+        const currentUrl = page.url();
+        if (!currentUrl || currentUrl === "about:blank") {
+            this.logger({
+                category: "agent",
+                message: `Page URL is empty. Navigating to https://www.google.com ...`,
+                level: 1,
+            });
+            await page.goto("https://www.google.com", { waitUntil: "load" });
+        }
+        // Set up captcha solver for Browserbase environments
+        if (this.v3.isCaptchaAutoSolveEnabled) {
+            this.captchaSolver = new captchaSolver_js_1.CaptchaSolver();
+            this.captchaSolver.init(() => this.v3.context.awaitActivePage());
+            // Block the CUA agent loop before each step while a captcha is being solved
+            this.agentClient.setPreStepHook(async () => {
+                if (this.captchaSolver?.isSolving()) {
+                    this.logger({
+                        category: "agent",
+                        message: "Captcha detected — waiting for Browserbase to solve it before continuing",
+                        level: 1,
+                    });
+                }
+                await this.captchaSolver?.waitIfSolving();
+                this.handleCaptchaSolveResult(this.captchaSolver?.consumeSolveResult());
+            });
+        }
+        if (this.highlightCursor) {
+            try {
+                await this.injectCursor();
+            }
+            catch (error) {
+                const errorMessage = error instanceof Error ? error.message : String(error);
+                this.logger({
+                    category: "agent",
+                    message: `Warning: Failed to inject cursor: ${errorMessage}. Continuing with execution.`,
+                    level: 1,
+                });
+                // Continue execution even if cursor injection fails
+            }
+        }
+        const start = Date.now();
+        let result;
+        try {
+            result = await this.agent.execute({ options, logger: this.logger });
+        }
+        finally {
+            this.captchaSolver?.dispose();
+            this.captchaSolver = null;
+        }
+        const inferenceTimeMs = Date.now() - start;
+        if (result.usage) {
+            this.v3.updateMetrics(methods_js_1.V3FunctionName.AGENT, result.usage.input_tokens, result.usage.output_tokens, result.usage.reasoning_tokens ?? 0, result.usage.cached_input_tokens ?? 0, inferenceTimeMs);
+        }
+        return result;
+    }
+    async executeAction(action) {
+        const page = await this.v3.context.awaitActivePage();
+        const recording = this.v3.isAgentReplayActive();
+        switch (action.type) {
+            case "click": {
+                const { x, y, button = "left", clickCount } = action;
+                if (recording) {
+                    const xpath = await page.click(x, y, {
+                        button: button ?? "left",
+                        clickCount: clickCount ?? 1,
+                        returnXpath: true,
+                    });
+                    const normalized = (0, xpath_js_1.ensureXPath)(xpath);
+                    if (normalized) {
+                        const stagehandAction = {
+                            selector: normalized,
+                            description: this.describePointerAction("click", x, y),
+                            method: "click",
+                            arguments: [],
+                        };
+                        this.recordCuaActStep(action, [stagehandAction], stagehandAction.description);
+                    }
+                }
+                else {
+                    await page.click(x, y, {
+                        button: button ?? "left",
+                        clickCount: clickCount ?? 1,
+                    });
+                }
+                return { success: true };
+            }
+            case "double_click":
+            case "doubleClick": {
+                const { x, y } = action;
+                if (recording) {
+                    const xpath = await page.click(x, y, {
+                        button: "left",
+                        clickCount: 2,
+                        returnXpath: true,
+                    });
+                    const normalized = (0, xpath_js_1.ensureXPath)(xpath);
+                    if (normalized) {
+                        const stagehandAction = {
+                            selector: normalized,
+                            description: this.describePointerAction("double click", x, y),
+                            method: "doubleClick",
+                            arguments: [],
+                        };
+                        this.recordCuaActStep(action, [stagehandAction], stagehandAction.description);
+                    }
+                }
+                else {
+                    await page.click(x, y, {
+                        button: "left",
+                        clickCount: 2,
+                    });
+                }
+                return { success: true };
+            }
+            case "tripleClick": {
+                const { x, y } = action;
+                if (recording) {
+                    const xpath = await page.click(x, y, {
+                        button: "left",
+                        clickCount: 3,
+                        returnXpath: true,
+                    });
+                    const normalized = (0, xpath_js_1.ensureXPath)(xpath);
+                    if (normalized) {
+                        const stagehandAction = {
+                            selector: normalized,
+                            description: this.describePointerAction("triple click", x, y),
+                            method: "tripleClick",
+                            arguments: [],
+                        };
+                        this.recordCuaActStep(action, [stagehandAction], stagehandAction.description);
+                    }
+                }
+                else {
+                    await page.click(x, y, {
+                        clickCount: 3,
+                    });
+                }
+                return { success: true };
+            }
+            case "type": {
+                const { text } = action;
+                await page.type(String(text ?? ""));
+                if (recording) {
+                    const xpath = await (0, index_js_1.computeActiveElementXpath)(page);
+                    const normalized = (0, xpath_js_1.ensureXPath)(xpath);
+                    if (normalized) {
+                        const stagehandAction = {
+                            selector: normalized,
+                            description: this.describeTypeAction(String(text ?? "")),
+                            method: "type",
+                            arguments: [String(text ?? "")],
+                        };
+                        this.recordCuaActStep(action, [stagehandAction], stagehandAction.description);
+                    }
+                }
+                return { success: true };
+            }
+            case "keypress": {
+                const { keys } = action;
+                const keyList = Array.isArray(keys) ? keys : [keys];
+                const stagehandActions = [];
+                for (const rawKey of keyList) {
+                    const mapped = (0, cuaKeyMapping_js_1.mapKeyToPlaywright)(String(rawKey ?? ""));
+                    await page.keyPress(mapped);
+                    if (recording) {
+                        stagehandActions.push({
+                            selector: "xpath=/html",
+                            description: `press ${mapped}`,
+                            method: "press",
+                            arguments: [mapped],
+                        });
+                    }
+                }
+                if (recording && stagehandActions.length > 0) {
+                    this.recordCuaActStep(action, stagehandActions, stagehandActions
+                        .map((a) => a.description)
+                        .filter(Boolean)
+                        .join(", ") || "keypress");
+                }
+                return { success: true };
+            }
+            case "scroll": {
+                const { x, y, scroll_x = 0, scroll_y = 0 } = action;
+                await page.scroll(x ?? 0, y ?? 0, scroll_x ?? 0, scroll_y ?? 0);
+                this.v3.recordAgentReplayStep({
+                    type: "scroll",
+                    deltaX: Number(scroll_x ?? 0),
+                    deltaY: Number(scroll_y ?? 0),
+                    anchor: typeof x === "number" && typeof y === "number"
+                        ? { x: Math.round(x), y: Math.round(y) }
+                        : undefined,
+                });
+                return { success: true };
+            }
+            case "drag": {
+                const { path } = action;
+                if (Array.isArray(path) && path.length >= 2) {
+                    const start = path[0];
+                    const end = path[path.length - 1];
+                    if (recording) {
+                        const xps = await page.dragAndDrop(start.x, start.y, end.x, end.y, {
+                            steps: Math.min(20, Math.max(5, path.length)),
+                            delay: 10,
+                            returnXpath: true,
+                        });
+                        const [fromXpath, toXpath] = xps || ["", ""];
+                        const from = (0, xpath_js_1.ensureXPath)(fromXpath);
+                        const to = (0, xpath_js_1.ensureXPath)(toXpath);
+                        if (from && to) {
+                            const stagehandAction = {
+                                selector: from,
+                                description: this.describeDragAction(),
+                                method: "dragAndDrop",
+                                arguments: [to],
+                            };
+                            this.recordCuaActStep(action, [stagehandAction], stagehandAction.description);
+                        }
+                    }
+                    else {
+                        await page.dragAndDrop(start.x, start.y, end.x, end.y, {
+                            steps: Math.min(20, Math.max(5, path.length)),
+                            delay: 10,
+                        });
+                    }
+                }
+                return { success: true };
+            }
+            case "move": {
+                const { x, y } = action;
+                if (typeof x === "number" && typeof y === "number") {
+                    if (recording) {
+                        const xpath = await page.hover(x, y, { returnXpath: true });
+                        const normalized = (0, xpath_js_1.ensureXPath)(xpath);
+                        if (normalized) {
+                            const stagehandAction = {
+                                selector: normalized,
+                                description: this.describePointerAction("hover", x, y),
+                                method: "hover",
+                                arguments: [],
+                            };
+                            this.recordCuaActStep(action, [stagehandAction], stagehandAction.description);
+                        }
+                    }
+                    else {
+                        await page.hover(x, y);
+                    }
+                }
+                return { success: true };
+            }
+            case "wait": {
+                const time = action?.timeMs ?? 1000;
+                await new Promise((r) => setTimeout(r, time));
+                if (time > 0 && recording) {
+                    this.v3.recordAgentReplayStep({ type: "wait", timeMs: Number(time) });
+                }
+                return { success: true };
+            }
+            case "screenshot": {
+                // No-op - screenshot is captured by captureAndSendScreenshot() after all actions
+                return { success: true };
+            }
+            case "goto": {
+                const { url } = action;
+                await page.goto(String(url ?? ""), { waitUntil: "load" });
+                if (recording) {
+                    this.v3.recordAgentReplayStep({
+                        type: "goto",
+                        url: String(url ?? ""),
+                    });
+                }
+                return { success: true };
+            }
+            case "back": {
+                await page.goBack();
+                if (recording) {
+                    this.v3.recordAgentReplayStep({
+                        type: "back",
+                    });
+                }
+                return { success: true };
+            }
+            case "forward": {
+                await page.goForward();
+                if (recording) {
+                    this.v3.recordAgentReplayStep({
+                        type: "forward",
+                    });
+                }
+                return { success: true };
+            }
+            case "open_web_browser": {
+                // Browser is already open, this is a no-op
+                return { success: true };
+            }
+            case "custom_tool": {
+                // Custom tools are handled by the agent client directly
+                return { success: true };
+            }
+            default:
+                this.logger({
+                    category: "agent",
+                    message: `Unknown action type: ${String(action.type)}`,
+                    level: 1,
+                });
+                return {
+                    success: false,
+                    error: `Unknown action ${String(action.type)}`,
+                };
+        }
+    }
+    // helper to make pointer target human-readable for logging
+    computePointerTarget(action) {
+        return typeof action.x === "number" && typeof action.y === "number"
+            ? `(${action.x}, ${action.y})`
+            : typeof action.selector === "string"
+                ? action.selector
+                : typeof action.input === "string"
+                    ? action.input
+                    : typeof action.description === "string"
+                        ? action.description
+                        : undefined;
+    }
+    describePointerAction(kind, x, y) {
+        const nx = Number(x);
+        const ny = Number(y);
+        if (Number.isFinite(nx) && Number.isFinite(ny)) {
+            return `${kind} at (${Math.round(nx)}, ${Math.round(ny)})`;
+        }
+        return kind;
+    }
+    describeTypeAction(text) {
+        const snippet = text.length > 30 ? `${text.slice(0, 27)}...` : text;
+        return `type "${snippet}"`;
+    }
+    describeDragAction() {
+        return "drag and drop";
+    }
+    buildInstructionFallback(agentAction, fallback) {
+        const raw = (typeof agentAction.action === "string" && agentAction.action.trim()) ||
+            (typeof agentAction.reasoning === "string" &&
+                agentAction.reasoning.trim());
+        return raw && raw.length > 0 ? raw : fallback;
+    }
+    recordCuaActStep(agentAction, stagehandActions, fallback) {
+        if (!stagehandActions.length)
+            return;
+        const instruction = this.buildInstructionFallback(agentAction, fallback);
+        const description = stagehandActions[0]?.description || instruction;
+        const actions = stagehandActions.map((act) => ({
+            ...act,
+            description: act.description || description,
+        }));
+        this.v3.recordAgentReplayStep({
+            type: "act",
+            instruction,
+            actions,
+            actionDescription: description,
+            message: typeof agentAction.reasoning === "string" &&
+                agentAction.reasoning.trim().length > 0
+                ? agentAction.reasoning.trim()
+                : undefined,
+        });
+    }
+    async updateClientViewport() {
+        try {
+            // For Google CUA, use configured viewport for coordinate normalization
+            // advancedStealth uses fixed 1288x711, otherwise use configured viewport
+            if (this.agentClient instanceof GoogleCUAClient_js_1.GoogleCUAClient) {
+                const dims = this.v3.isAdvancedStealth
+                    ? { width: 1288, height: 711 }
+                    : this.v3.configuredViewport;
+                this.agentClient.setViewport(dims.width, dims.height);
+            }
+            else {
+                // For other clients, use actual window dimensions
+                const page = await this.v3.context.awaitActivePage();
+                const { w, h } = await page.mainFrame().evaluate("({ w: window.innerWidth, h: window.innerHeight })");
+                if (w && h)
+                    this.agentClient.setViewport(w, h);
+            }
+        }
+        catch {
+            //
+        }
+    }
+    async updateClientUrl() {
+        try {
+            const page = await this.v3.context.awaitActivePage();
+            const url = page.url();
+            this.agentClient.setCurrentUrl(url);
+        }
+        catch {
+            //
+        }
+    }
+    async captureAndSendScreenshot() {
+        this.logger({
+            category: "agent",
+            message: "Capturing screenshot",
+            level: 1,
+        });
+        try {
+            const page = await this.v3.context.awaitActivePage();
+            const screenshotBuffer = await page.screenshot({ fullPage: false });
+            const currentUrl = page.url();
+            return await this.agentClient.captureScreenshot({
+                base64Image: screenshotBuffer.toString("base64"),
+                currentUrl,
+            });
+        }
+        catch (e) {
+            this.logger({
+                category: "agent",
+                message: `Error capturing screenshot: ${String(e?.message ?? e)}`,
+                level: 0,
+            });
+            return null;
+        }
+    }
+    handleCaptchaSolveResult(result) {
+        if (!result)
+            return;
+        if (result.solved) {
+            this.captchaClickGuardRemaining = 3;
+            this.agentClient.addContextNote(captchaSolver_js_1.CAPTCHA_SOLVED_MSG);
+            this.logger({
+                category: "agent",
+                message: "Captcha solved — continuing with task",
+                level: 1,
+            });
+        }
+        if (result.errored) {
+            this.captchaClickGuardRemaining = 0;
+            this.agentClient.addContextNote(captchaSolver_js_1.CAPTCHA_ERRORED_MSG);
+            this.logger({
+                category: "agent",
+                message: "Captcha solver failed or errored",
+                level: 1,
+            });
+        }
+    }
+    async shouldSkipSolvedCaptchaInteraction(action) {
+        if (this.captchaClickGuardRemaining <= 0) {
+            return false;
+        }
+        if (action.type !== "click") {
+            return false;
+        }
+        const x = action.x;
+        const y = action.y;
+        if (typeof x !== "number" || typeof y !== "number") {
+            return false;
+        }
+        try {
+            const page = await this.v3.context.awaitActivePage();
+            const boxes = await page.evaluate(() => {
+                const selectors = [
+                    'iframe[title*="reCAPTCHA"]',
+                    'iframe[src*="recaptcha"]',
+                    'iframe[src*="hcaptcha"]',
+                    'iframe[src*="turnstile"]',
+                    ".g-recaptcha",
+                    "[data-sitekey]",
+                    '[class*="captcha"]',
+                    '[id*="captcha"]',
+                ];
+                const seen = new Set();
+                const bounds = [];
+                for (const selector of selectors) {
+                    for (const element of document.querySelectorAll(selector)) {
+                        if (seen.has(element))
+                            continue;
+                        seen.add(element);
+                        const rect = element.getBoundingClientRect();
+                        if (rect.width <= 0 || rect.height <= 0)
+                            continue;
+                        bounds.push({
+                            left: rect.left,
+                            top: rect.top,
+                            right: rect.right,
+                            bottom: rect.bottom,
+                        });
+                    }
+                }
+                return bounds;
+            });
+            return boxes.some((box) => x >= box.left && x <= box.right && y >= box.top && y <= box.bottom);
+        }
+        catch {
+            return false;
+        }
+    }
+    async injectCursor() {
+        try {
+            const page = await this.v3.context.awaitActivePage();
+            await page.enableCursorOverlay();
+        }
+        catch {
+            // Best-effort only
+        }
+    }
+}
+exports.V3CuaAgentHandler = V3CuaAgentHandler;
+//# sourceMappingURL=v3CuaAgentHandler.js.map