@browserbasehq/orca 3.1.0-patch.3 → 3.2.0-middleware.1

package/dist/cjs/lib/v3/agent/OpenAICUAClient.js

@@ -0,0 +1,681 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpenAICUAClient = void 0;
+const openai_1 = __importDefault(require("openai"));
+const AgentClient_js_1 = require("./AgentClient.js");
+const sdkErrors_js_1 = require("../types/public/sdkErrors.js");
+const FlowLogger_js_1 = require("../flowlogger/FlowLogger.js");
+const uuid_1 = require("uuid");
+/**
+ * Client for OpenAI's Computer Use Assistant API
+ * This implementation uses the official OpenAI Responses API for Computer Use
+ */
+const CAPTCHA_PROCEED_TOOL = "captchaSolvedProceed";
+class OpenAICUAClient extends AgentClient_js_1.AgentClient {
+    pendingContextNotes = [];
+    captchaSolvedToolActive = false;
+    apiKey;
+    organization;
+    baseURL;
+    client;
+    lastResponseId;
+    currentViewport = { width: 1288, height: 711 };
+    currentUrl;
+    screenshotProvider;
+    actionHandler;
+    reasoningItems = new Map();
+    environment = "browser"; // "browser", "mac", "windows", or "ubuntu"
+    tools;
+    safetyConfirmationHandler;
+    constructor(type, modelName, userProvidedInstructions, clientOptions, tools) {
+        super(type, modelName, userProvidedInstructions);
+        // Process client options
+        this.apiKey =
+            clientOptions?.apiKey || process.env.OPENAI_API_KEY || "";
+        this.baseURL = clientOptions?.baseURL || undefined;
+        this.organization =
+            clientOptions?.organization || process.env.OPENAI_ORG;
+        // Get environment if specified
+        if (clientOptions?.environment &&
+            typeof clientOptions.environment === "string") {
+            this.environment = clientOptions.environment;
+        }
+        // Store client options for reference
+        this.clientOptions = {
+            apiKey: this.apiKey,
+        };
+        if (this.baseURL) {
+            this.clientOptions.baseURL = this.baseURL;
+        }
+        // Initialize the OpenAI client
+        this.client = new openai_1.default(this.clientOptions);
+        this.tools = tools;
+    }
+    setViewport(width, height) {
+        this.currentViewport = { width, height };
+    }
+    setCurrentUrl(url) {
+        this.currentUrl = url;
+    }
+    setScreenshotProvider(provider) {
+        this.screenshotProvider = provider;
+    }
+    setActionHandler(handler) {
+        this.actionHandler = handler;
+    }
+    setTools(tools) {
+        this.tools = tools;
+    }
+    setSafetyConfirmationHandler(handler) {
+        this.safetyConfirmationHandler = handler;
+    }
+    addContextNote(note) {
+        this.pendingContextNotes.push(note);
+        // When a captcha-related note arrives, expose a tool that the model can
+        // call instead of asking the user for confirmation.  This replaces
+        // fragile English-phrase parsing with a structured tool call.
+        if (note.toLowerCase().includes("captcha")) {
+            this.captchaSolvedToolActive = true;
+        }
+    }
+    /**
+     * Execute a task with the OpenAI CUA
+     * This is the main entry point for the agent
+     * @implements AgentClient.execute
+     */
+    async execute(executionOptions) {
+        const { options, logger } = executionOptions;
+        const { instruction } = options;
+        const maxSteps = options.maxSteps || 10;
+        let currentStep = 0;
+        let completed = false;
+        const actions = [];
+        const messageList = [];
+        let finalMessage = "";
+        this.reasoningItems.clear(); // Clear any previous reasoning items
+        // Start with the initial instruction
+        let inputItems = this.createInitialInputItems(instruction);
+        let previousResponseId = undefined;
+        let totalInputTokens = 0;
+        let totalOutputTokens = 0;
+        let totalInferenceTime = 0;
+        try {
+            // Execute steps until completion or max steps reached
+            while (!completed && currentStep < maxSteps) {
+                await this.preStepHook?.();
+                logger({
+                    category: "agent",
+                    message: `Executing step ${currentStep + 1}/${maxSteps}`,
+                    level: 1,
+                });
+                const result = await this.executeStep(inputItems, previousResponseId, logger);
+                totalInputTokens += result.usage.input_tokens;
+                totalOutputTokens += result.usage.output_tokens;
+                totalInferenceTime += result.usage.inference_time_ms;
+                // Add actions to the list
+                actions.push(...result.actions);
+                // Update completion status
+                completed = result.completed;
+                // Store the previous response ID for the next request
+                previousResponseId = result.responseId;
+                // Update the input items for the next step if we're continuing
+                if (!completed) {
+                    inputItems = result.nextInputItems;
+                    const contextNotes = this.drainContextNotes();
+                    if (contextNotes.length > 0) {
+                        inputItems = [
+                            ...inputItems,
+                            ...contextNotes.map((note) => ({
+                                role: "user",
+                                content: note,
+                            })),
+                        ];
+                    }
+                }
+                // Record any message for this step
+                if (result.message) {
+                    messageList.push(result.message);
+                    finalMessage = result.message;
+                }
+                // Increment step counter
+                currentStep++;
+            }
+            // Return the final result
+            return {
+                success: completed,
+                actions,
+                message: finalMessage,
+                completed,
+                usage: {
+                    input_tokens: totalInputTokens,
+                    output_tokens: totalOutputTokens,
+                    inference_time_ms: totalInferenceTime,
+                },
+            };
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            logger({
+                category: "agent",
+                message: `Error executing agent task: ${errorMessage}`,
+                level: 0,
+            });
+            return {
+                success: false,
+                actions,
+                message: `Failed to execute task: ${errorMessage}`,
+                completed: false,
+                usage: {
+                    input_tokens: totalInputTokens,
+                    output_tokens: totalOutputTokens,
+                    inference_time_ms: totalInferenceTime,
+                },
+            };
+        }
+    }
+    /**
+     * Execute a single step of the agent
+     * This coordinates the flow: Request → Get Action → Execute Action
+     */
+    async executeStep(inputItems, previousResponseId, logger) {
+        try {
+            // Get response from the model
+            const result = await this.getAction(inputItems, previousResponseId);
+            const output = result.output;
+            const responseId = result.responseId;
+            const usage = {
+                input_tokens: result.usage.input_tokens,
+                output_tokens: result.usage.output_tokens,
+                inference_time_ms: result.usage.inference_time_ms,
+            };
+            // Add any reasoning items to our map
+            for (const item of output) {
+                if (item.type === "reasoning") {
+                    this.reasoningItems.set(item.id, item);
+                    logger({
+                        category: "agent",
+                        message: `Reasoning: ${String(item.content || "")}`,
+                        level: 1,
+                    });
+                }
+            }
+            // Extract actions from the output
+            const stepActions = [];
+            for (const item of output) {
+                if (item.type === "computer_call" && this.isComputerCallItem(item)) {
+                    logger({
+                        category: "agent",
+                        message: `Found computer_call: ${item.action.type}, payload: ${JSON.stringify(item.action)}, call_id: ${item.call_id}`,
+                        level: 2,
+                    });
+                    const action = this.convertComputerCallToAction(item);
+                    if (action) {
+                        stepActions.push(action);
+                        logger({
+                            category: "agent",
+                            message: `Converted computer_call to action: ${action.type}`,
+                            level: 2,
+                        });
+                    }
+                }
+                else if (item.type === "function_call" &&
+                    this.isFunctionCallItem(item)) {
+                    logger({
+                        category: "agent",
+                        message: `Found function_call: ${item.name}, call_id: ${item.call_id}`,
+                        level: 2,
+                    });
+                    const action = this.convertFunctionCallToAction(item);
+                    if (action) {
+                        stepActions.push(action);
+                        logger({
+                            category: "agent",
+                            message: `Converted function_call to action: ${action.type}`,
+                            level: 2,
+                        });
+                    }
+                }
+            }
+            // Extract message text
+            let message = "";
+            for (const item of output) {
+                if (item.type === "message") {
+                    logger({
+                        category: "agent",
+                        message: `Found message block`,
+                        level: 2,
+                    });
+                    if (item.content && Array.isArray(item.content)) {
+                        for (const content of item.content) {
+                            if (content.type === "output_text" && content.text) {
+                                message += content.text + "\n";
+                                logger({
+                                    category: "agent",
+                                    message: `Message text: ${String(content.text || "")}`,
+                                    level: 1,
+                                });
+                            }
+                        }
+                    }
+                }
+            }
+            // Take actions and get results
+            const nextInputItems = await this.takeAction(output, logger);
+            // Check if completed
+            const completed = output.length === 0 ||
+                output.every((item) => item.type === "message" || item.type === "reasoning");
+            return {
+                actions: stepActions,
+                message: message.trim(),
+                completed,
+                nextInputItems,
+                responseId,
+                usage: usage,
+            };
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            logger({
+                category: "agent",
+                message: `Error executing step: ${errorMessage}`,
+                level: 0,
+            });
+            throw error;
+        }
+    }
+    isComputerCallItem(item) {
+        return (item.type === "computer_call" &&
+            "call_id" in item &&
+            "action" in item &&
+            typeof item.action === "object");
+    }
+    async handleSafetyConfirmation(pendingSafetyChecks, logger) {
+        if (this.safetyConfirmationHandler) {
+            logger({
+                category: "agent",
+                message: `Requesting safety confirmation for ${pendingSafetyChecks.length} check(s): ${pendingSafetyChecks.map((c) => c.code).join(", ")}`,
+                level: 1,
+            });
+            const response = await this.safetyConfirmationHandler(pendingSafetyChecks);
+            if (response.acknowledged) {
+                logger({
+                    category: "agent",
+                    message: `Safety checks acknowledged by user`,
+                    level: 1,
+                });
+                return pendingSafetyChecks;
+            }
+            else {
+                logger({
+                    category: "agent",
+                    message: `Safety checks rejected by user`,
+                    level: 1,
+                });
+                return undefined;
+            }
+        }
+        logger({
+            category: "agent",
+            message: `Auto-acknowledging ${pendingSafetyChecks.length} safety check(s)`,
+            level: 2,
+        });
+        return pendingSafetyChecks;
+    }
+    isFunctionCallItem(item) {
+        return (item.type === "function_call" &&
+            "call_id" in item &&
+            "name" in item &&
+            "arguments" in item);
+    }
+    createInitialInputItems(instruction) {
+        // For the initial request, we use a simple array with the user's instruction
+        return [
+            {
+                role: "system",
+                content: this.userProvidedInstructions,
+            },
+            {
+                role: "user",
+                content: instruction,
+            },
+        ];
+    }
+    async getAction(inputItems, previousResponseId) {
+        try {
+            // Create the request parameters
+            const requestParams = {
+                model: this.modelName,
+                tools: [
+                    {
+                        type: "computer_use_preview",
+                        display_width: this.currentViewport.width,
+                        display_height: this.currentViewport.height,
+                        environment: this.environment,
+                    },
+                ],
+                input: inputItems,
+                truncation: "auto",
+            };
+            // Add custom tools if available
+            if (this.tools && Object.keys(this.tools).length > 0) {
+                const customTools = Object.entries(this.tools).map(([name, tool]) => ({
+                    type: "function",
+                    name,
+                    function: {
+                        name,
+                        description: tool.description,
+                        parameters: tool.inputSchema,
+                    },
+                }));
+                requestParams.tools = [
+                    ...requestParams.tools,
+                    ...customTools,
+                ];
+            }
+            // When a captcha was just solved, expose a tool the model can call
+            // to confirm it should proceed.  This avoids fragile English-phrase
+            // parsing and works regardless of the model's output language.
+            if (this.captchaSolvedToolActive) {
+                requestParams.tools = [
+                    ...requestParams.tools,
+                    {
+                        type: "function",
+                        name: CAPTCHA_PROCEED_TOOL,
+                        function: {
+                            name: CAPTCHA_PROCEED_TOOL,
+                            description: "The captcha on this page was solved automatically. " +
+                                "Call this tool to confirm and continue with your task " +
+                                "instead of asking the user for permission.",
+                            parameters: { type: "object", properties: {}, required: [] },
+                        },
+                    },
+                ];
+            }
+            // Add previous_response_id if available
+            if (previousResponseId) {
+                requestParams.previous_response_id = previousResponseId;
+            }
+            // Log LLM request
+            const llmRequestId = (0, uuid_1.v7)();
+            FlowLogger_js_1.FlowLogger.logLlmRequest({
+                requestId: llmRequestId,
+                model: this.modelName,
+                prompt: (0, FlowLogger_js_1.extractLlmCuaPromptSummary)(inputItems),
+            });
+            const startTime = Date.now();
+            // Create the response using the OpenAI Responses API
+            // @ts-expect-error - Force type to match what the OpenAI SDK expects
+            const response = await this.client.responses.create(requestParams);
+            const endTime = Date.now();
+            const elapsedMs = endTime - startTime;
+            // Extract only the input_tokens and output_tokens
+            const usage = {
+                input_tokens: response.usage.input_tokens,
+                output_tokens: response.usage.output_tokens,
+                inference_time_ms: elapsedMs,
+            };
+            // Log LLM response
+            FlowLogger_js_1.FlowLogger.logLlmResponse({
+                requestId: llmRequestId,
+                model: this.modelName,
+                output: (0, FlowLogger_js_1.extractLlmCuaResponseSummary)(response.output),
+                inputTokens: response.usage.input_tokens,
+                outputTokens: response.usage.output_tokens,
+            });
+            // Store the response ID for future use
+            this.lastResponseId = response.id;
+            // Return the output and response ID
+            return {
+                output: response.output,
+                responseId: response.id,
+                usage,
+            };
+        }
+        catch (error) {
+            console.error("Error getting action from OpenAI:", error);
+            throw error;
+        }
+    }
+    async takeAction(output, logger) {
+        const nextInputItems = [];
+        // Process each output item
+        for (const item of output) {
+            if (item.type === "computer_call" && this.isComputerCallItem(item)) {
+                // Handle computer calls
+                try {
+                    const action = this.convertComputerCallToAction(item);
+                    if (action && this.actionHandler) {
+                        logger({
+                            category: "agent",
+                            message: `Executing computer action: ${action.type}`,
+                            level: 1,
+                        });
+                        await this.actionHandler(action);
+                    }
+                    // Capture a screenshot
+                    const screenshot = await this.captureScreenshot();
+                    // Create a computer_call_output for the next request
+                    const outputItem = {
+                        type: "computer_call_output",
+                        call_id: item.call_id,
+                        output: {
+                            type: "input_image",
+                            image_url: screenshot,
+                        },
+                    };
+                    logger({
+                        category: "agent",
+                        message: `Added computer_call_output for call_id: ${item.call_id}`,
+                        level: 2,
+                    });
+                    // Add current URL if available
+                    if (this.currentUrl) {
+                        const computerCallOutput = outputItem;
+                        computerCallOutput.output.current_url = this.currentUrl;
+                    }
+                    if (item.pending_safety_checks &&
+                        item.pending_safety_checks.length > 0) {
+                        const acknowledgedChecks = await this.handleSafetyConfirmation(item.pending_safety_checks, logger);
+                        if (acknowledgedChecks) {
+                            const computerCallOutput = outputItem;
+                            computerCallOutput.acknowledged_safety_checks =
+                                acknowledgedChecks;
+                        }
+                    }
+                    nextInputItems.push(outputItem);
+                }
+                catch (error) {
+                    if (error instanceof sdkErrors_js_1.StagehandClosedError) {
+                        throw error;
+                    }
+                    const errorMessage = error instanceof Error ? error.message : String(error);
+                    logger({
+                        category: "agent",
+                        message: `Error executing computer call: ${errorMessage}`,
+                        level: 0,
+                    });
+                    try {
+                        // Capture a screenshot even on error
+                        const screenshot = await this.captureScreenshot();
+                        const errorOutputItem = {
+                            type: "computer_call_output",
+                            call_id: item.call_id,
+                            output: {
+                                type: "input_image",
+                                image_url: screenshot,
+                                error: errorMessage,
+                            },
+                        };
+                        // Add current URL if available
+                        if (this.currentUrl) {
+                            const computerCallOutput = errorOutputItem;
+                            computerCallOutput.output.current_url = this.currentUrl;
+                        }
+                        if (item.pending_safety_checks &&
+                            item.pending_safety_checks.length > 0) {
+                            const acknowledgedChecks = await this.handleSafetyConfirmation(item.pending_safety_checks, logger);
+                            if (acknowledgedChecks) {
+                                const computerCallOutput = errorOutputItem;
+                                computerCallOutput.acknowledged_safety_checks =
+                                    acknowledgedChecks;
+                            }
+                        }
+                        nextInputItems.push(errorOutputItem);
+                    }
+                    catch (screenshotError) {
+                        if (screenshotError instanceof sdkErrors_js_1.StagehandClosedError) {
+                            throw screenshotError;
+                        }
+                        // If we can't capture a screenshot, just send the error
+                        logger({
+                            category: "agent",
+                            message: `Error capturing screenshot: ${String(screenshotError)}`,
+                            level: 0,
+                        });
+                        // For error cases without a screenshot, we need to use a string output
+                        nextInputItems.push({
+                            type: "computer_call_output",
+                            call_id: item.call_id,
+                            output: `Error: ${errorMessage}`,
+                        });
+                    }
+                }
+            }
+            else if (item.type === "function_call" &&
+                this.isFunctionCallItem(item)) {
+                // Handle the captcha-proceed tool — just return a confirmation and
+                // deactivate the tool so it doesn't appear on subsequent steps.
+                if (item.name === CAPTCHA_PROCEED_TOOL) {
+                    this.captchaSolvedToolActive = false;
+                    nextInputItems.push({
+                        type: "function_call_output",
+                        call_id: item.call_id,
+                        output: "Confirmed. The captcha is solved. Continue completing the original task autonomously without asking for further confirmation.",
+                    });
+                    continue;
+                }
+                // Handle function calls (tool calls)
+                try {
+                    const action = this.convertFunctionCallToAction(item);
+                    if (action && this.actionHandler) {
+                        await this.actionHandler(action);
+                    }
+                    // Execute the tool if available
+                    let toolResult = "Tool executed successfully";
+                    if (this.tools && item.name in this.tools) {
+                        try {
+                            const tool = this.tools[item.name];
+                            const args = JSON.parse(item.arguments);
+                            logger({
+                                category: "agent",
+                                message: `Executing tool call: ${item.name} with args: ${item.arguments}`,
+                                level: 1,
+                            });
+                            const result = await tool.execute(args, {
+                                toolCallId: item.call_id,
+                                messages: [],
+                            });
+                            toolResult = JSON.stringify(result);
+                            logger({
+                                category: "agent",
+                                message: `Tool ${item.name} completed successfully. Result: ${toolResult}`,
+                                level: 1,
+                            });
+                        }
+                        catch (toolError) {
+                            const errorMessage = toolError instanceof Error
+                                ? toolError.message
+                                : String(toolError);
+                            toolResult = `Error executing tool: ${errorMessage}`;
+                            logger({
+                                category: "agent",
+                                message: `Error executing tool ${item.name}: ${errorMessage}`,
+                                level: 0,
+                            });
+                        }
+                    }
+                    // Create a function_call_output for the next request
+                    const outputItem = {
+                        type: "function_call_output",
+                        call_id: item.call_id,
+                        output: toolResult,
+                    };
+                    nextInputItems.push(outputItem);
+                }
+                catch (error) {
+                    if (error instanceof sdkErrors_js_1.StagehandClosedError) {
+                        throw error;
+                    }
+                    const errorMessage = error instanceof Error ? error.message : String(error);
+                    logger({
+                        category: "agent",
+                        message: `Error executing function call: ${errorMessage}`,
+                        level: 0,
+                    });
+                    // Send error result back
+                    const errorOutputItem = {
+                        type: "function_call_output",
+                        call_id: item.call_id,
+                        output: `Error: ${errorMessage}`,
+                    };
+                    nextInputItems.push(errorOutputItem);
+                }
+            }
+        }
+        return nextInputItems;
+    }
+    convertComputerCallToAction(call) {
+        const { action } = call;
+        // Instead of wrapping the action in a params object, spread the action properties directly
+        // This ensures properties like x, y, button, etc. are directly accessible on the AgentAction
+        return {
+            type: action.type,
+            ...action, // Spread all properties from the action
+        };
+    }
+    drainContextNotes() {
+        if (this.pendingContextNotes.length === 0) {
+            return [];
+        }
+        const notes = [...this.pendingContextNotes];
+        this.pendingContextNotes = [];
+        return notes;
+    }
+    convertFunctionCallToAction(call) {
+        try {
+            const args = JSON.parse(call.arguments);
+            return {
+                type: call.name,
+                params: args,
+            };
+        }
+        catch (error) {
+            console.error("Error parsing function call arguments:", error);
+            return null;
+        }
+    }
+    async captureScreenshot(options) {
+        // Use provided options if available
+        if (options?.base64Image) {
+            return `data:image/png;base64,${options.base64Image}`;
+        }
+        // Use the screenshot provider if available
+        if (this.screenshotProvider) {
+            try {
+                const base64Image = await this.screenshotProvider();
+                return `data:image/png;base64,${base64Image}`;
+            }
+            catch (error) {
+                console.error("Error capturing screenshot:", error);
+                throw error;
+            }
+        }
+        throw new sdkErrors_js_1.AgentScreenshotProviderError("`screenshotProvider` has not been set. " +
+            "Please call `setScreenshotProvider()` with a valid function that returns a base64-encoded image");
+    }
+}
+exports.OpenAICUAClient = OpenAICUAClient;
+//# sourceMappingURL=OpenAICUAClient.js.map