@browserbasehq/orca 3.7.0-preview.1 → 3.7.2-gemini-cua

package/dist/esm/lib/v3/agent/GoogleCUAClient.js

@@ -7,6 +7,15 @@ import { mapKeyToPlaywright } from "./utils/cuaKeyMapping.js";
 import { executeGoogleCustomTool, isCustomTool, convertToolSetToFunctionDeclarations, } from "./utils/googleCustomToolHandler.js";
 import { FlowLogger, extractLlmCuaPromptSummary, extractLlmCuaResponseSummary, } from "../flowlogger/FlowLogger.js";
 import { v7 as uuidv7 } from "uuid";
+/** Errors worth retrying: rate limits, server errors, network failures. */
+function isTransientError(error) {
+    const status = error?.status;
+    if (typeof status === "number") {
+        return status === 429 || status >= 500;
+    }
+    const message = error instanceof Error ? error.message : String(error);
+    return /network|timeout|timed out|fetch failed|econn|socket hang up/i.test(message);
+}
 /**
  * Client for Google's Computer Use Assistant API
  * This implementation uses the Google Generative AI SDK for Computer Use
@@ -24,6 +33,13 @@ export class GoogleCUAClient extends AgentClient {
     tools;
     baseURL;
     safetyConfirmationHandler;
+    /**
+     * Request API. "interactions" uses the stateful Interactions API (server-side
+     * history via previous_interaction_id) instead of generateContent.
+     */
+    api = "generateContent";
+    previousInteractionId;
+    pendingInteractionInput;
     constructor(type, modelName, userProvidedInstructions, clientOptions, tools) {
         super(type, modelName, userProvidedInstructions);
         this.tools = tools;
@@ -41,19 +57,23 @@ export class GoogleCUAClient extends AgentClient {
             ...(this.baseURL ? { httpOptions: { baseUrl: this.baseURL } } : {}),
         };
         this.client = new GoogleGenAI(genAIOptions);
+        if (clientOptions?.googleApi === "interactions") {
+            this.api = "interactions";
+        }
         // Get environment if specified
         if (clientOptions?.environment &&
             typeof clientOptions.environment === "string") {
             this.environment = clientOptions.environment;
         }
+        // No maxOutputTokens: thinking tokens count against it on gemini-3.x, so a
+        // fixed cap truncates turns (finishReason MAX_TOKENS). Use the model max.
         this.generateContentConfig = {
             temperature: 1,
             topP: 0.95,
             topK: 40,
-            maxOutputTokens: 8192,
-            // systemInstruction: this.userProvidedInstructions
-            //   ? { parts: [{ text: this.userProvidedInstructions }] }
-            //   : { parts: [{ text: buildGoogleCUASystemPrompt() }] },
+            systemInstruction: this.userProvidedInstructions
+                ? this.userProvidedInstructions
+                : buildGoogleCUASystemPrompt().content,
             tools: [
                 {
                     computerUse: {
@@ -163,6 +183,7 @@ export class GoogleCUAClient extends AgentClient {
         const maxSteps = options.maxSteps || 10;
         let currentStep = 0;
         let completed = false;
+        let declined = false;
         const actions = [];
         const messageList = [];
         let finalMessage = "";
@@ -193,6 +214,9 @@ export class GoogleCUAClient extends AgentClient {
                 actions.push(...result.actions);
                 // Update completion status
                 completed = result.completed;
+                if (result.stopped) {
+                    declined = true;
+                }
                 // Record any message for this step
                 if (result.message) {
                     messageList.push(result.message);
@@ -203,7 +227,7 @@ export class GoogleCUAClient extends AgentClient {
             }
             // Return the final result
             return {
-                success: completed,
+                success: completed && !declined,
                 actions,
                 message: finalMessage,
                 completed,
@@ -243,20 +267,12 @@ export class GoogleCUAClient extends AgentClient {
      */
     async initializeHistory(instruction) {
         const parts = [{ text: instruction }];
+        this.previousInteractionId = undefined;
+        this.pendingInteractionInput = instruction;
         // Note: The Python implementation doesn't include the initial screenshot
-        // Following the same pattern here
-        const systemPromptContent = this.userProvidedInstructions
-            ? this.userProvidedInstructions
-            : buildGoogleCUASystemPrompt().content;
+        // Following the same pattern here. The system prompt travels in
+        // generateContentConfig.systemInstruction.
         this.history = [
-            {
-                role: "user",
-                parts: [
-                    {
-                        text: "System prompt: " + systemPromptContent,
-                    },
-                ],
-            },
             {
                 role: "user",
                 parts,
@@ -272,11 +288,14 @@ export class GoogleCUAClient extends AgentClient {
             // Compress images in conversation history before sending to the model
             const compressedResult = compressGoogleConversationImages(this.history, 2);
             const compressedHistory = compressedResult.items;
-            // Use the SDK's generateContent method with retry logic (matching Python's get_model_response)
+            // Use the SDK's generateContent method, retrying only transient failures
+            // (rate limits, 5xx, network, empty candidates). Anything else — a 400,
+            // or a candidate that stopped for a non-STOP reason — fails fast.
             const maxRetries = 5;
             const baseDelayS = 1;
             let lastError = null;
             let response = null;
+            let emptyResponse = false;
             // Log LLM request
             const llmRequestId = uuidv7();
             FlowLogger.logLlmRequest({
@@ -297,21 +316,20 @@ export class GoogleCUAClient extends AgentClient {
                         await new Promise((resolve) => setTimeout(resolve, delay));
                     }
                     // Use the SDK's generateContent method - following Python SDK pattern
-                    response = await this.client.models.generateContent({
-                        model: this.modelName,
-                        contents: compressedHistory,
-                        config: this.generateContentConfig,
-                    });
-                    // Check if we have valid response content
+                    response = await this.requestModel(compressedHistory);
+                    // An empty response is treated as transient and retried; an early
+                    // stop (non-STOP finish reason) is not — it is rejected below.
                     if (!response.candidates || response.candidates.length === 0) {
+                        emptyResponse = true;
                         throw new LLMResponseError("agent", "Response has no candidates!");
                     }
-                    const candidate = response.candidates[0];
-                    if (!candidate.content || !candidate.content.parts) {
-                        const reason = candidate.finishReason || "unknown";
-                        throw new LLMResponseError("agent", `Response has no content (finish reason: ${reason})`);
+                    const first = response.candidates[0];
+                    const stoppedEarly = first.finishReason && first.finishReason !== "STOP";
+                    if (!stoppedEarly && !first.content?.parts?.length) {
+                        emptyResponse = true;
+                        throw new LLMResponseError("agent", `Response has no content (finish reason: ${first.finishReason || "unknown"})`);
                     }
-                    // Success - we have a valid response
+                    // Success - validated below, outside the retry loop
                     break;
                 }
                 catch (error) {
@@ -321,6 +339,11 @@ export class GoogleCUAClient extends AgentClient {
                         message: `API call error: ${lastError.message}`,
                         level: 2,
                     });
+                    const retryable = emptyResponse || isTransientError(error);
+                    emptyResponse = false;
+                    if (!retryable) {
+                        throw lastError;
+                    }
                     // If this was the last attempt, throw the error
                     if (attempt === maxRetries - 1) {
                         logger({
@@ -335,6 +358,13 @@ export class GoogleCUAClient extends AgentClient {
             if (!response) {
                 throw (lastError || new Error("Failed to get response after all retries"));
             }
+            // A candidate that stopped for any reason other than STOP (MAX_TOKENS,
+            // SAFETY, MALFORMED_FUNCTION_CALL, ...) is incomplete: fail the run
+            // rather than executing its calls or reporting its text as the answer.
+            const candidate = response.candidates[0];
+            if (candidate.finishReason && candidate.finishReason !== "STOP") {
+                throw new LLMResponseError("agent", `Model response stopped early (finish reason: ${candidate.finishReason})`);
+            }
             const endTime = Date.now();
             const elapsedMs = endTime - startTime;
             const { usageMetadata } = response;
@@ -368,63 +398,58 @@ export class GoogleCUAClient extends AgentClient {
                 }
                 this.history.push(sanitizedContent);
             }
-            // Execute actions and collect function responses
+            // Execute each function call and answer it with its own function
+            // response: the call's outcome (output or error text), the URL, and a
+            // fresh screenshot taken after that call. The model expects exactly one
+            // response per call, including calls with nothing to execute (e.g. a
+            // standalone take_screenshot) and calls that could not be converted.
             const functionResponses = [];
-            // Always process the model's response, even when it produced no
-            // executable action (e.g. a standalone take_screenshot). The model
-            // expects exactly one function response — a fresh screenshot — per
-            // computer-use call; skipping it when there are 0 actions leaves the
-            // model blind and it gives up after one empty turn.
-            {
-                let hasError = false;
-                // Execute all actions (a no-op when there are none).
-                for (let i = 0; i < result.actions.length; i++) {
-                    const action = result.actions[i];
-                    logger({
-                        category: "agent",
-                        message: `Executing action ${i + 1}/${result.actions.length}: ${action.type}`,
-                        level: 2,
-                    });
-                    // Special handling for open_web_browser - don't execute it
-                    if (action.type === "open_web_browser") {
-                        // Set pageUrl for open_web_browser since it doesn't go through action handler
-                        action.pageUrl = this.currentUrl;
-                        logger({
-                            category: "agent",
-                            message: "Skipping open_web_browser action",
-                            level: 2,
-                        });
+            let stopped;
+            for (const call of result.calls) {
+                const { functionCall } = call;
+                if (call.actions[0]?.type === "custom_tool") {
+                    const action = call.actions[0];
+                    const toolName = action.name;
+                    if (this.tools && toolName in this.tools) {
+                        const executionResult = await executeGoogleCustomTool(toolName, action.arguments, this.tools, functionCall, logger);
+                        functionResponses.push(executionResult.functionResponse);
+                    }
+                    continue;
+                }
+                // Ask for confirmation BEFORE acting. If the user declines, the
+                // action is not executed and the run stops.
+                let safetyAcknowledgement;
+                if (functionCall.args?.safety_decision) {
+                    safetyAcknowledgement = await this.handleSafetyConfirmation(functionCall.args.safety_decision, logger);
+                    if (safetyAcknowledgement === undefined) {
+                        stopped = `Stopped: the user declined to confirm "${functionCall.name}".`;
+                        break;
                     }
-                    else if (action.type === "screenshot") {
-                        // No interaction to perform — the screenshot captured below is
-                        // returned to the model as this call's function response.
+                }
+                let error = call.error;
+                if (!error) {
+                    for (let i = 0; i < call.actions.length; i++) {
+                        const action = call.actions[i];
                         logger({
                             category: "agent",
-                            message: "take_screenshot: capturing current page",
+                            message: `Executing action ${i + 1}/${call.actions.length} for ${functionCall.name}: ${action.type}`,
                             level: 2,
                         });
-                    }
-                    else if (action.type === "custom_tool") {
-                        const toolName = action.name;
-                        const toolArgs = action.arguments;
-                        if (this.tools && toolName in this.tools) {
-                            const correspondingFunctionCall = result.functionCalls.find((fc) => fc.name === toolName);
-                            if (correspondingFunctionCall) {
-                                const executionResult = await executeGoogleCustomTool(toolName, toolArgs, this.tools, correspondingFunctionCall, logger);
-                                functionResponses.push(executionResult.functionResponse);
-                                if (!executionResult.success) {
-                                    hasError = true;
-                                }
-                            }
+                        // open_web_browser and take_screenshot have nothing to execute;
+                        // the screenshot captured below is the call's result.
+                        if (action.type === "open_web_browser") {
+                            action.pageUrl = this.currentUrl;
+                            continue;
+                        }
+                        if (action.type === "screenshot" || !this.actionHandler) {
+                            continue;
                         }
-                    }
-                    else if (this.actionHandler) {
                         try {
                             await this.actionHandler(action);
                             // Add a delay between actions to ensure they complete properly
                             // Longer delay for typing actions to ensure fields are ready
-                            if (i < result.actions.length - 1) {
-                                const nextAction = result.actions[i + 1];
+                            if (i < call.actions.length - 1) {
+                                const nextAction = call.actions[i + 1];
                                 const isTypingAction = action.type === "type" || nextAction.type === "type";
                                 const delay = isTypingAction ? 500 : 200;
                                 await new Promise((resolve) => setTimeout(resolve, delay));
@@ -434,87 +459,88 @@ export class GoogleCUAClient extends AgentClient {
                             if (actionError instanceof StagehandClosedError) {
                                 throw actionError;
                             }
+                            error =
+                                actionError instanceof Error
+                                    ? actionError.message
+                                    : String(actionError);
                             logger({
                                 category: "agent",
-                                message: `Error executing action ${action.type}: ${actionError}`,
+                                message: `Error executing action ${action.type}: ${error}`,
                                 level: 0,
                             });
-                            hasError = true;
-                            // Continue processing other actions even if one fails
+                            // Skip the rest of this call; report the failure to the model.
+                            break;
                         }
                     }
                 }
-                // Create function responses for computer use actions (non-custom tools)
-                // We need exactly one response per function call, regardless of how many actions were generated
-                if (result.functionCalls.length > 0 || hasError) {
-                    // Filter out custom tool function calls as they've already been handled
-                    const computerUseFunctionCalls = result.functionCalls.filter((fc) => !isCustomTool(fc, this.tools));
-                    if (computerUseFunctionCalls.length > 0) {
-                        try {
-                            logger({
-                                category: "agent",
-                                message: `Taking screenshot after executing ${result.actions.length} actions${hasError ? " (with errors)" : ""}`,
-                                level: 2,
-                            });
-                            const screenshot = await this.captureScreenshot();
-                            const base64Data = screenshot.base64;
-                            // Create one function response for each computer use function call
-                            // Following Python SDK pattern: FunctionResponse with parts containing inline_data
-                            for (const functionCall of computerUseFunctionCalls) {
-                                let safetyAcknowledgement;
-                                if (functionCall.args?.safety_decision) {
-                                    safetyAcknowledgement = await this.handleSafetyConfirmation(functionCall.args.safety_decision, logger);
-                                }
-                                const functionResponsePart = {
-                                    functionResponse: {
-                                        name: functionCall.name,
-                                        response: {
-                                            url: this.currentUrl || "",
-                                            ...(safetyAcknowledgement !== undefined
-                                                ? {
-                                                    safety_acknowledgement: safetyAcknowledgement,
-                                                }
-                                                : {}),
-                                        },
-                                        parts: [
-                                            {
-                                                inlineData: {
-                                                    mimeType: screenshot.mediaType,
-                                                    data: base64Data,
-                                                },
-                                            },
-                                        ],
-                                    },
-                                };
-                                functionResponses.push(functionResponsePart);
-                            }
-                        }
-                        catch (error) {
-                            logger({
-                                category: "agent",
-                                message: `Error capturing screenshot: ${error}`,
-                                level: 0,
-                            });
-                        }
-                    }
+                let screenshot;
+                let screenshotError;
+                try {
+                    screenshot = await this.captureScreenshot();
                 }
-                // Add all function responses to history in a single user message
-                if (functionResponses.length > 0) {
+                catch (screenshotFailure) {
+                    if (screenshotFailure instanceof StagehandClosedError) {
+                        throw screenshotFailure;
+                    }
+                    screenshotError =
+                        screenshotFailure instanceof Error
+                            ? screenshotFailure.message
+                            : String(screenshotFailure);
                     logger({
                         category: "agent",
-                        message: `Adding ${functionResponses.length} function responses to history`,
-                        level: 2,
-                    });
-                    this.history.push({
-                        role: "user",
-                        parts: functionResponses,
+                        message: `Error capturing screenshot: ${screenshotError}`,
+                        level: 0,
                     });
                 }
+                functionResponses.push({
+                    functionResponse: {
+                        ...(functionCall.id ? { id: functionCall.id } : {}),
+                        name: functionCall.name,
+                        response: {
+                            url: this.currentUrl || "",
+                            ...(error ? { error } : {}),
+                            ...(screenshotError ? { screenshot_error: screenshotError } : {}),
+                            ...(safetyAcknowledgement !== undefined
+                                ? { safety_acknowledgement: safetyAcknowledgement }
+                                : {}),
+                        },
+                        ...(screenshot
+                            ? {
+                                parts: [
+                                    {
+                                        inlineData: {
+                                            mimeType: screenshot.mediaType,
+                                            data: screenshot.base64,
+                                        },
+                                    },
+                                ],
+                            }
+                            : {}),
+                    },
+                });
+            }
+            // The Interactions API keeps history server-side: the next request sends
+            // only these results, chained by previous_interaction_id.
+            if (this.api === "interactions" && !stopped) {
+                this.pendingInteractionInput = functionResponses.map(GoogleCUAClient.toFunctionResult);
+            }
+            // Add all function responses to history in a single user message
+            if (!stopped && functionResponses.length > 0) {
+                logger({
+                    category: "agent",
+                    message: `Adding ${functionResponses.length} function responses to history`,
+                    level: 2,
+                });
+                this.history.push({
+                    role: "user",
+                    parts: functionResponses,
+                });
             }
             return {
                 actions: result.actions,
-                message: result.message,
-                completed: result.completed,
+                message: stopped ?? result.message,
+                completed: stopped ? true : result.completed,
+                stopped: stopped !== undefined,
                 usage: {
                     // promptTokenCount is the TOTAL input and already includes the
                     // cached portion; cachedContentTokenCount is the cache-hit subset
@@ -538,19 +564,128 @@ export class GoogleCUAClient extends AgentClient {
             throw error;
         }
     }
+    /** Send one model request on the configured API. */
+    async requestModel(contents) {
+        if (this.api === "interactions") {
+            return this.createInteraction();
+        }
+        return this.client.models.generateContent({
+            model: this.modelName,
+            contents,
+            config: this.generateContentConfig,
+        });
+    }
+    /**
+     * Create an interaction (Interactions API, REST) continuing the previous one,
+     * and map it onto the generateContent response shape the step loop expects.
+     */
+    async createInteraction() {
+        const baseUrl = (this.baseURL ?? "https://generativelanguage.googleapis.com").replace(/\/+$/, "");
+        const body = {
+            model: this.modelName.replace(/^models\//, ""),
+            input: this.pendingInteractionInput,
+            system_instruction: this.generateContentConfig.systemInstruction,
+            tools: [
+                {
+                    type: "computer_use",
+                    environment: this.environment === "ENVIRONMENT_DESKTOP" ? "desktop" : "browser",
+                },
+            ],
+            ...(this.previousInteractionId
+                ? { previous_interaction_id: this.previousInteractionId }
+                : {}),
+        };
+        const res = await fetch(`${baseUrl}/v1beta/interactions`, {
+            method: "POST",
+            headers: {
+                "content-type": "application/json",
+                "x-goog-api-key": this.apiKey,
+            },
+            body: JSON.stringify(body),
+        });
+        const json = (await res.json().catch(() => ({})));
+        if (!res.ok || !json.id) {
+            const { message = res.statusText, code } = json.error ?? {};
+            // code "safety": Google's input classifier rejected the turn. Not
+            // transient (status 400), so it fails the run instead of retrying.
+            throw Object.assign(new Error(code === "safety"
+                ? `Gemini blocked the request (safety): ${message}`
+                : `Interactions API error ${res.status}: ${message}`), { status: res.status });
+        }
+        const parts = [];
+        for (const step of json.steps ?? []) {
+            if (step.type === "function_call" && step.name) {
+                parts.push({
+                    functionCall: { id: step.id, name: step.name, args: step.arguments },
+                });
+            }
+            else if (step.type === "model_output") {
+                for (const content of step.content ?? []) {
+                    if (content.type === "text" && content.text) {
+                        parts.push({ text: content.text });
+                    }
+                }
+            }
+        }
+        if (parts.length === 0 && json.output_text) {
+            parts.push({ text: json.output_text });
+        }
+        this.previousInteractionId = json.id;
+        // requires_action (calls to answer) and completed are normal stops; any
+        // other status (incomplete, failed, budget_exceeded, ...) is an early stop.
+        const finishReason = json.status === "requires_action" || json.status === "completed"
+            ? "STOP"
+            : String(json.status).toUpperCase();
+        return {
+            candidates: [{ content: { role: "model", parts }, finishReason }],
+            usageMetadata: {
+                promptTokenCount: json.usage?.total_input_tokens,
+                candidatesTokenCount: json.usage?.total_output_tokens,
+                thoughtsTokenCount: json.usage?.total_thought_tokens,
+                cachedContentTokenCount: json.usage?.total_cached_tokens,
+            },
+        };
+    }
+    /** Convert a generateContent function response into an Interactions result. */
+    static toFunctionResult(part) {
+        const fr = part.functionResponse;
+        const response = { ...(fr.response ?? {}) };
+        // The Interactions API takes a boolean acknowledgement.
+        if (response.safety_acknowledgement !== undefined) {
+            response.safety_acknowledgement = true;
+        }
+        const image = fr.parts?.[0]?.inlineData;
+        return {
+            type: "function_result",
+            call_id: fr.id ?? "",
+            name: fr.name,
+            ...(response.error ? { is_error: true } : {}),
+            result: [
+                { type: "text", text: JSON.stringify(response) },
+                ...(image?.data
+                    ? [
+                        {
+                            type: "image",
+                            data: image.data,
+                            mime_type: image.mimeType,
+                        },
+                    ]
+                    : []),
+            ],
+        };
+    }
     /**
      * Process the response from Google's API
      */
     async processResponse(response, logger) {
-        const actions = [];
+        const calls = [];
         let message = "";
-        const functionCalls = [];
         if (!response.candidates || response.candidates.length === 0) {
             return {
                 actions: [],
                 message: "No candidates in response",
                 completed: true,
-                functionCalls: [],
+                calls: [],
             };
         }
         const candidate = response.candidates[0];
@@ -571,83 +706,71 @@ export class GoogleCUAClient extends AgentClient {
                 });
             }
             if (part.functionCall) {
-                functionCalls.push(part.functionCall);
+                const functionCall = part.functionCall;
                 logger({
                     category: "agent",
-                    message: `Found function call: ${part.functionCall.name} with args: ${JSON.stringify(part.functionCall.args)}`,
+                    message: `Found function call: ${functionCall.name} with args: ${JSON.stringify(functionCall.args)}`,
                     level: 2,
                 });
-                // Convert function call to action(s)
-                const action = this.convertFunctionCallToAction(part.functionCall, logger);
-                if (action) {
-                    // Special handling for type actions. gemini-2.5 type_text_at carries
-                    // coordinates (click the field first); gemini-3.x `type` has none and
-                    // types into the element the model already focused.
-                    if (action.type === "type") {
-                        logger({
-                            category: "agent",
-                            message: `Adding action: ${JSON.stringify(action)}`,
-                            level: 2,
-                        });
-                        // Click the target first only when coordinates are provided.
-                        if (typeof action.x === "number" && typeof action.y === "number") {
-                            actions.push({
-                                type: "click",
-                                x: action.x,
-                                y: action.y,
-                                button: "left",
-                            });
-                        }
-                        // If clear_before_typing is true (default), add a select all
-                        if (action.clearBeforeTyping) {
-                            // Select all text in the field
-                            actions.push({
-                                type: "keypress",
-                                keys: ["ControlOrMeta+A"],
-                            });
-                            actions.push({
-                                type: "keypress",
-                                keys: ["Backspace"],
-                            });
-                        }
-                        // Then add the type action
-                        actions.push(action);
-                        if (action.pressEnter) {
-                            actions.push({
-                                type: "keypress",
-                                keys: ["Enter"],
-                            });
-                        }
-                    }
-                    else {
-                        actions.push(action);
-                    }
-                }
-                else {
+                const action = this.convertFunctionCallToAction(functionCall, logger);
+                if (!action) {
                     logger({
                         category: "agent",
-                        message: `Warning: Could not convert function call ${part.functionCall.name} to action`,
+                        message: `Warning: Could not convert function call ${functionCall.name} to action`,
                         level: 1,
                     });
+                    calls.push({
+                        functionCall,
+                        actions: [],
+                        error: `Unsupported or malformed computer-use action "${functionCall.name}" with args ${JSON.stringify(functionCall.args ?? {})}`,
+                    });
+                    continue;
                 }
+                calls.push({ functionCall, actions: this.expandAction(action) });
             }
         }
+        const actions = calls.flatMap((call) => call.actions);
         // Log summary of what we found
         logger({
             category: "agent",
-            message: `Found ${functionCalls.length} function calls, converted to ${actions.length} actions`,
+            message: `Found ${calls.length} function calls, converted to ${actions.length} actions`,
             level: 2,
         });
-        // Check if task is completed
-        const completed = functionCalls.length === 0 ||
-            (candidate.finishReason && candidate.finishReason !== "STOP");
+        if (calls.length === 0 && !message.trim()) {
+            throw new LLMResponseError("agent", "Response has neither function calls nor text");
+        }
+        // Early stops (non-STOP finish reasons) are rejected in executeStep, so a
+        // turn is complete exactly when the model made no function calls.
         return {
             actions,
             message: message.trim(),
-            completed,
-            functionCalls,
+            completed: calls.length === 0,
+            calls,
         };
     }
+    /**
+     * Expand a converted action into the executor steps it needs. Typing may
+     * click the field first, clear it, and press Enter afterwards.
+     */
+    expandAction(action) {
+        if (action.type !== "type") {
+            return [action];
+        }
+        const steps = [];
+        // Click the target first only when coordinates are provided.
+        if (typeof action.x === "number" && typeof action.y === "number") {
+            steps.push({ type: "click", x: action.x, y: action.y, button: "left" });
+        }
+        if (action.clearBeforeTyping) {
+            steps.push({ type: "keypress", keys: ["ControlOrMeta+A"] });
+            steps.push({ type: "keypress", keys: ["Backspace"] });
+        }
+        steps.push(action);
+        if (action.pressEnter) {
+            steps.push({ type: "keypress", keys: ["Enter"] });
+        }
+        return steps;
+    }
     /**
      * Convert Google function call to Stagehand action
      */
@@ -689,7 +812,6 @@ export class GoogleCUAClient extends AgentClient {
             press_keys: "key_combination",
             hotkey: "key_combination",
             screenshot: "take_screenshot",
-            wait: "wait_5_seconds",
         };
         const name = NAME_ALIASES[rawName] ?? rawName;
         // Predefined computer-use tools take precedence over custom tools. If a
@@ -761,15 +883,37 @@ export class GoogleCUAClient extends AgentClient {
                     button: name === "right_click" ? "right" : "middle",
                 };
             }
+            // gemini-3.x press/release primitives (e.g. a manual drag or a
+            // modifier held across clicks).
+            case "mouse_down":
+            case "mouse_up": {
+                if (!GoogleCUAClient.isFiniteCoord(args.x) ||
+                    !GoogleCUAClient.isFiniteCoord(args.y)) {
+                    return null;
+                }
+                const { x, y } = this.normalizeCoordinates(args.x, args.y);
+                return { type: name, x, y };
+            }
+            case "key_down":
+            case "key_up": {
+                if (typeof args.key !== "string" || !args.key.trim()) {
+                    return null;
+                }
+                return { type: name, key: args.key.trim() };
+            }
             case "type_text_at": {
                 // text is required; reject a malformed call rather than typing
                 // "undefined". An empty string is valid (e.g. clear the field).
                 if (typeof args.text !== "string") {
                     return null;
                 }
-                // press_enter and clear_before_typing are shared across generations.
+                // gemini-2.5 type_text_at clears the field by default
+                // (clear_before_typing). gemini-3.x `type` has no such arg and types
+                // into the focused element, so only clear when asked or when the call
+                // targets a field by coordinates.
+                const hasCoords = typeof args.x === "number" && typeof args.y === "number";
                 const pressEnter = args.press_enter ?? false;
-                const clearBeforeTyping = args.clear_before_typing ?? true;
+                const clearBeforeTyping = args.clear_before_typing ?? hasCoords;
                 const base = {
                     type: "type",
                     text: args.text,
@@ -780,7 +924,7 @@ export class GoogleCUAClient extends AgentClient {
                 // gemini-3.x `type` has no coordinates and types into the element the
                 // model already focused with a preceding click. Only attach coords when
                 // present so the executor doesn't click at NaN.
-                if (typeof args.x === "number" && typeof args.y === "number") {
+                if (hasCoords) {
                     const { x, y } = this.normalizeCoordinates(args.x, args.y);
                     return { ...base, x, y };
                 }
@@ -826,12 +970,15 @@ export class GoogleCUAClient extends AgentClient {
                 }
                 const { x, y } = this.normalizeCoordinates(args.x, args.y);
                 const direction = (args.direction || "down").toLowerCase();
-                // 2.5 uses `magnitude`; gemini-3.x `scroll` uses `magnitude_in_pixels`.
+                // 2.5 uses `magnitude` (default 800); gemini-3.x `scroll` uses
+                // `magnitude_in_pixels` (default 300).
                 const magnitude = GoogleCUAClient.isFiniteCoord(args.magnitude)
                     ? args.magnitude
                     : GoogleCUAClient.isFiniteCoord(args.magnitude_in_pixels)
                         ? args.magnitude_in_pixels
-                        : 800;
+                        : rawName === "scroll"
+                            ? 300
+                            : 800;
                 let scroll_x = 0;
                 let scroll_y = 0;
                 if (direction === "up") {
@@ -881,6 +1028,16 @@ export class GoogleCUAClient extends AgentClient {
                     type: "wait",
                     timeMs: 5000, // Google CUA waits for 5 seconds
                 };
+            case "wait": {
+                // gemini-3.x `wait` takes optional `seconds` (default 1).
+                const seconds = GoogleCUAClient.isFiniteCoord(args.seconds) && args.seconds >= 0
+                    ? args.seconds
+                    : 1;
+                return {
+                    type: "wait",
+                    timeMs: Math.round(seconds * 1000),
+                };
+            }
             case "hover_at": {
                 const { x, y } = this.normalizeCoordinates(args.x, args.y);
                 return {