@browserbasehq/orca 3.2.0-preview.4 → 3.2.1-preview.0

package/dist/esm/lib/v3/agent/OpenAICUAClient.d.ts

@@ -1,13 +1,13 @@
+import type { EasyInputMessage } from "openai/resources/responses/responses";
 import { LogLine } from "../types/public/logs.js";
 import { AgentAction, AgentResult, AgentType, AgentExecutionOptions, ResponseInputItem, ResponseItem, SafetyConfirmationHandler } from "../types/public/agent.js";
 import { ClientOptions } from "../types/public/model.js";
 import { AgentClient } from "./AgentClient.js";
 import { ToolSet } from "ai";
-/**
- * Client for OpenAI's Computer Use Assistant API
- * This implementation uses the official OpenAI Responses API for Computer Use
- */
+type OpenAIRequestInputItem = ResponseInputItem | EasyInputMessage;
 export declare class OpenAICUAClient extends AgentClient {
+    private pendingContextNotes;
+    private captchaSolvedToolActive;
     private apiKey;
     private organization?;
     private baseURL;
@@ -21,6 +21,7 @@ export declare class OpenAICUAClient extends AgentClient {
     private environment;
     private tools?;
     private safetyConfirmationHandler?;
+    private get usesNewComputerTool();
     constructor(type: AgentType, modelName: string, userProvidedInstructions?: string, clientOptions?: ClientOptions, tools?: ToolSet);
     setViewport(width: number, height: number): void;
     setCurrentUrl(url: string): void;
@@ -28,6 +29,7 @@ export declare class OpenAICUAClient extends AgentClient {
     setActionHandler(handler: (action: AgentAction) => Promise<void>): void;
     setTools(tools: ToolSet): void;
     setSafetyConfirmationHandler(handler?: SafetyConfirmationHandler): void;
+    addContextNote(note: string): void;
     /**
      * Execute a task with the OpenAI CUA
      * This is the main entry point for the agent
@@ -38,7 +40,7 @@ export declare class OpenAICUAClient extends AgentClient {
      * Execute a single step of the agent
      * This coordinates the flow: Request → Get Action → Execute Action
      */
-    executeStep(inputItems: ResponseInputItem[], previousResponseId: string | undefined, logger: (message: LogLine) => void): Promise<{
+    executeStep(inputItems: OpenAIRequestInputItem[], previousResponseId: string | undefined, logger: (message: LogLine) => void): Promise<{
         actions: AgentAction[];
         message: string;
         completed: boolean;
@@ -54,16 +56,20 @@ export declare class OpenAICUAClient extends AgentClient {
     private handleSafetyConfirmation;
     private isFunctionCallItem;
     private createInitialInputItems;
-    getAction(inputItems: ResponseInputItem[], previousResponseId?: string): Promise<{
+    getAction(inputItems: OpenAIRequestInputItem[], previousResponseId?: string): Promise<{
         output: ResponseItem[];
         responseId: string;
         usage: Record<string, number>;
     }>;
     takeAction(output: ResponseItem[], logger: (message: LogLine) => void): Promise<ResponseInputItem[]>;
     private convertComputerCallToAction;
+    private drainContextNotes;
+    private captureInitialScreenshot;
+    private convertComputerCallToActions;
     private convertFunctionCallToAction;
     captureScreenshot(options?: {
         base64Image?: string;
         currentUrl?: string;
     }): Promise<string>;
 }
+export {};

package/dist/esm/lib/v3/agent/OpenAICUAClient.js

@@ -7,7 +7,10 @@ import { v7 as uuidv7 } from "uuid";
  * Client for OpenAI's Computer Use Assistant API
  * This implementation uses the official OpenAI Responses API for Computer Use
  */
+const CAPTCHA_PROCEED_TOOL = "captchaSolvedProceed";
 export class OpenAICUAClient extends AgentClient {
+    pendingContextNotes = [];
+    captchaSolvedToolActive = false;
     apiKey;
     organization;
     baseURL;
@@ -21,6 +24,9 @@ export class OpenAICUAClient extends AgentClient {
     environment = "browser"; // "browser", "mac", "windows", or "ubuntu"
     tools;
     safetyConfirmationHandler;
+    get usesNewComputerTool() {
+        return this.modelName.startsWith("gpt-5");
+    }
     constructor(type, modelName, userProvidedInstructions, clientOptions, tools) {
         super(type, modelName, userProvidedInstructions);
         // Process client options
@@ -63,6 +69,15 @@ export class OpenAICUAClient extends AgentClient {
     setSafetyConfirmationHandler(handler) {
         this.safetyConfirmationHandler = handler;
     }
+    addContextNote(note) {
+        this.pendingContextNotes.push(note);
+        // When a captcha-related note arrives, expose a tool that the model can
+        // call instead of asking the user for confirmation.  This replaces
+        // fragile English-phrase parsing with a structured tool call.
+        if (note.toLowerCase().includes("captcha")) {
+            this.captchaSolvedToolActive = true;
+        }
+    }
     /**
      * Execute a task with the OpenAI CUA
      * This is the main entry point for the agent
@@ -79,7 +94,7 @@ export class OpenAICUAClient extends AgentClient {
         let finalMessage = "";
         this.reasoningItems.clear(); // Clear any previous reasoning items
         // Start with the initial instruction
-        let inputItems = this.createInitialInputItems(instruction);
+        let inputItems = await this.createInitialInputItems(instruction);
         let previousResponseId = undefined;
         let totalInputTokens = 0;
         let totalOutputTokens = 0;
@@ -87,6 +102,7 @@ export class OpenAICUAClient extends AgentClient {
         try {
             // Execute steps until completion or max steps reached
             while (!completed && currentStep < maxSteps) {
+                await this.preStepHook?.();
                 logger({
                     category: "agent",
                     message: `Executing step ${currentStep + 1}/${maxSteps}`,
@@ -105,6 +121,16 @@ export class OpenAICUAClient extends AgentClient {
                 // Update the input items for the next step if we're continuing
                 if (!completed) {
                     inputItems = result.nextInputItems;
+                    const contextNotes = this.drainContextNotes();
+                    if (contextNotes.length > 0) {
+                        inputItems = [
+                            ...inputItems,
+                            ...contextNotes.map((note) => ({
+                                role: "user",
+                                content: note,
+                            })),
+                        ];
+                    }
                 }
                 // Record any message for this step
                 if (result.message) {
@@ -179,15 +205,15 @@ export class OpenAICUAClient extends AgentClient {
                 if (item.type === "computer_call" && this.isComputerCallItem(item)) {
                     logger({
                         category: "agent",
-                        message: `Found computer_call: ${item.action.type}, call_id: ${item.call_id}`,
+                        message: `Found computer_call with call_id: ${item.call_id}`,
                         level: 2,
                     });
-                    const action = this.convertComputerCallToAction(item);
-                    if (action) {
+                    const actions = this.convertComputerCallToActions(item);
+                    for (const action of actions) {
                         stepActions.push(action);
                         logger({
                             category: "agent",
-                            message: `Converted computer_call to action: ${action.type}`,
+                            message: `Found computer_call action: ${action.type}, payload: ${JSON.stringify(action)}, call_id: ${item.call_id}`,
                             level: 2,
                         });
                     }
@@ -260,8 +286,8 @@ export class OpenAICUAClient extends AgentClient {
     isComputerCallItem(item) {
         return (item.type === "computer_call" &&
             "call_id" in item &&
-            "action" in item &&
-            typeof item.action === "object");
+            (("action" in item && typeof item.action === "object") ||
+                ("actions" in item && Array.isArray(item.actions))));
     }
     async handleSafetyConfirmation(pendingSafetyChecks, logger) {
         if (this.safetyConfirmationHandler) {
@@ -301,34 +327,54 @@ export class OpenAICUAClient extends AgentClient {
             "name" in item &&
             "arguments" in item);
     }
-    createInitialInputItems(instruction) {
-        // For the initial request, we use a simple array with the user's instruction
-        return [
-            {
+    async createInitialInputItems(instruction) {
+        const inputItems = [];
+        if (this.userProvidedInstructions) {
+            const systemMessage = {
                 role: "system",
                 content: this.userProvidedInstructions,
-            },
-            {
-                role: "user",
-                content: instruction,
-            },
+            };
+            inputItems.push(systemMessage);
+        }
+        const textInput = {
+            type: "input_text",
+            text: instruction,
+        };
+        const userContent = [
+            textInput,
         ];
+        const initialScreenshot = await this.captureInitialScreenshot();
+        if (initialScreenshot) {
+            const screenshotInput = {
+                type: "input_image",
+                image_url: initialScreenshot,
+                detail: "high",
+            };
+            userContent.push(screenshotInput);
+        }
+        const userMessage = {
+            role: "user",
+            content: userContent,
+        };
+        inputItems.push(userMessage);
+        return inputItems;
     }
     async getAction(inputItems, previousResponseId) {
         try {
-            // Create the request parameters
+            // Create the request parameters, branching on tool format
+            const computerTool = this.usesNewComputerTool
+                ? { type: "computer" }
+                : {
+                    type: "computer_use_preview",
+                    display_width: this.currentViewport.width,
+                    display_height: this.currentViewport.height,
+                    environment: this.environment,
+                };
             const requestParams = {
                 model: this.modelName,
-                tools: [
-                    {
-                        type: "computer_use_preview",
-                        display_width: this.currentViewport.width,
-                        display_height: this.currentViewport.height,
-                        environment: this.environment,
-                    },
-                ],
+                tools: [computerTool],
                 input: inputItems,
-                truncation: "auto",
+                ...(this.usesNewComputerTool ? {} : { truncation: "auto" }),
             };
             // Add custom tools if available
             if (this.tools && Object.keys(this.tools).length > 0) {
@@ -346,6 +392,25 @@ export class OpenAICUAClient extends AgentClient {
                     ...customTools,
                 ];
             }
+            // When a captcha was just solved, expose a tool the model can call
+            // to confirm it should proceed.  This avoids fragile English-phrase
+            // parsing and works regardless of the model's output language.
+            if (this.captchaSolvedToolActive) {
+                requestParams.tools = [
+                    ...requestParams.tools,
+                    {
+                        type: "function",
+                        name: CAPTCHA_PROCEED_TOOL,
+                        function: {
+                            name: CAPTCHA_PROCEED_TOOL,
+                            description: "The captcha on this page was solved automatically. " +
+                                "Call this tool to confirm and continue with your task " +
+                                "instead of asking the user for permission.",
+                            parameters: { type: "object", properties: {}, required: [] },
+                        },
+                    },
+                ];
+            }
             // Add previous_response_id if available
             if (previousResponseId) {
                 requestParams.previous_response_id = previousResponseId;
@@ -396,26 +461,34 @@ export class OpenAICUAClient extends AgentClient {
         // Process each output item
         for (const item of output) {
             if (item.type === "computer_call" && this.isComputerCallItem(item)) {
-                // Handle computer calls
+                // Handle computer calls (both single-action and batched-actions formats)
                 try {
-                    const action = this.convertComputerCallToAction(item);
-                    if (action && this.actionHandler) {
-                        logger({
-                            category: "agent",
-                            message: `Executing computer action: ${action.type}`,
-                            level: 1,
-                        });
-                        await this.actionHandler(action);
+                    const actions = this.convertComputerCallToActions(item);
+                    if (this.actionHandler) {
+                        for (const action of actions) {
+                            logger({
+                                category: "agent",
+                                message: `Executing computer action: ${action.type}`,
+                                level: 1,
+                            });
+                            await this.actionHandler(action);
+                        }
                     }
-                    // Capture a screenshot
+                    // Capture a screenshot after all actions in the batch
                     const screenshot = await this.captureScreenshot();
-                    // Create a computer_call_output for the next request
+                    // Build the output — use "computer_screenshot" for new format, "input_image" for legacy
+                    const outputType = this.usesNewComputerTool
+                        ? "computer_screenshot"
+                        : "input_image";
                     const outputItem = {
                         type: "computer_call_output",
                         call_id: item.call_id,
                         output: {
-                            type: "input_image",
+                            type: outputType,
                             image_url: screenshot,
+                            ...(this.usesNewComputerTool
+                                ? { detail: "original" }
+                                : {}),
                         },
                     };
                     logger({
@@ -423,8 +496,8 @@ export class OpenAICUAClient extends AgentClient {
                         message: `Added computer_call_output for call_id: ${item.call_id}`,
                         level: 2,
                     });
-                    // Add current URL if available
-                    if (this.currentUrl) {
+                    // Legacy format supports current_url on the output; new format does not
+                    if (!this.usesNewComputerTool && this.currentUrl) {
                         const computerCallOutput = outputItem;
                         computerCallOutput.output.current_url = this.currentUrl;
                     }
@@ -450,19 +523,23 @@ export class OpenAICUAClient extends AgentClient {
                         level: 0,
                     });
                     try {
-                        // Capture a screenshot even on error
                         const screenshot = await this.captureScreenshot();
+                        const outputType = this.usesNewComputerTool
+                            ? "computer_screenshot"
+                            : "input_image";
                         const errorOutputItem = {
                             type: "computer_call_output",
                             call_id: item.call_id,
                             output: {
-                                type: "input_image",
+                                type: outputType,
                                 image_url: screenshot,
                                 error: errorMessage,
+                                ...(this.usesNewComputerTool
+                                    ? { detail: "original" }
+                                    : {}),
                             },
                         };
-                        // Add current URL if available
-                        if (this.currentUrl) {
+                        if (!this.usesNewComputerTool && this.currentUrl) {
                             const computerCallOutput = errorOutputItem;
                             computerCallOutput.output.current_url = this.currentUrl;
                         }
@@ -481,13 +558,11 @@ export class OpenAICUAClient extends AgentClient {
                         if (screenshotError instanceof StagehandClosedError) {
                             throw screenshotError;
                         }
-                        // If we can't capture a screenshot, just send the error
                         logger({
                             category: "agent",
                             message: `Error capturing screenshot: ${String(screenshotError)}`,
                             level: 0,
                         });
-                        // For error cases without a screenshot, we need to use a string output
                         nextInputItems.push({
                             type: "computer_call_output",
                             call_id: item.call_id,
@@ -498,6 +573,17 @@ export class OpenAICUAClient extends AgentClient {
             }
             else if (item.type === "function_call" &&
                 this.isFunctionCallItem(item)) {
+                // Handle the captcha-proceed tool — just return a confirmation and
+                // deactivate the tool so it doesn't appear on subsequent steps.
+                if (item.name === CAPTCHA_PROCEED_TOOL) {
+                    this.captchaSolvedToolActive = false;
+                    nextInputItems.push({
+                        type: "function_call_output",
+                        call_id: item.call_id,
+                        output: "Confirmed. The captcha is solved. Continue completing the original task autonomously without asking for further confirmation.",
+                    });
+                    continue;
+                }
                 // Handle function calls (tool calls)
                 try {
                     const action = this.convertFunctionCallToAction(item);
@@ -570,13 +656,42 @@ export class OpenAICUAClient extends AgentClient {
     }
     convertComputerCallToAction(call) {
         const { action } = call;
-        // Instead of wrapping the action in a params object, spread the action properties directly
-        // This ensures properties like x, y, button, etc. are directly accessible on the AgentAction
+        if (!action)
+            return null;
         return {
             type: action.type,
-            ...action, // Spread all properties from the action
+            ...action,
         };
     }
+    drainContextNotes() {
+        if (this.pendingContextNotes.length === 0) {
+            return [];
+        }
+        const notes = [...this.pendingContextNotes];
+        this.pendingContextNotes = [];
+        return notes;
+    }
+    async captureInitialScreenshot() {
+        if (!this.screenshotProvider) {
+            return undefined;
+        }
+        try {
+            return await this.captureScreenshot();
+        }
+        catch {
+            return undefined;
+        }
+    }
+    convertComputerCallToActions(call) {
+        if (call.actions && Array.isArray(call.actions)) {
+            return call.actions.map((action) => ({
+                type: action.type,
+                ...action,
+            }));
+        }
+        const single = this.convertComputerCallToAction(call);
+        return single ? [single] : [];
+    }
     convertFunctionCallToAction(call) {
         try {
             const args = JSON.parse(call.arguments);