askui 0.25.0 → 0.26.0

package/dist/cjs/execution/ui-control-client.d.ts

@@ -5,7 +5,7 @@ import { AnnotationRequest } from '../core/model/annotation-result/annotation-in
 import { DetectedElement } from '../core/model/annotation-result/detected-element';
 import { ClientArgs } from './ui-controller-client-interface';
 import { ModelCompositionBranch } from './model-composition-branch';
-import { AskUIAgent, AgentHistory } from '../core/models/anthropic';
+import { AskUIAgent, AgentHistory, ActOptions } from '../core/models/anthropic';
 export type RelationsForConvenienceMethods = 'nearestTo' | 'leftOf' | 'above' | 'rightOf' | 'below' | 'contains';
 export type TextMatchingOption = 'similar' | 'exact' | 'regex';
 export type ElementExistsQueryType = 'otherElement' | 'switch' | 'element' | 'container' | 'checkbox' | 'element' | 'button' | 'table' | 'text' | 'icon' | 'image' | 'textfield';
@@ -476,51 +476,101 @@ export declare class UiControlClient extends ApiCommands {
      */
     expectAllExist(query: ElementExistsQuery[]): Promise<ExpectAllExistResult>;
     /**
-     * Instructs the agent to achieve a specified goal through autonomous actions.
+     * Instructs the agent to autonomously achieve a specified goal through UI interactions.
      *
-     * The agent will analyze the screen, determine necessary steps, and perform actions
-     * to accomplish the goal. This may include clicking, typing, scrolling, and other
-     * interface interactions.
+     * This method enables AI-powered automation by allowing the agent to:
+     * - Analyze the current screen state and/or provided images
+     * - Plan and execute a sequence of UI interactions
+     * - Handle complex tasks through natural language instructions
+     * - Maintain context across multiple actions
      *
-     * The `options` parameter allows the caller to maintain contextual continuity across
-     * multiple `act` calls, either from the same or different agent interfaces.
+     * The agent can perform various UI interactions including:
+     * - Clicking buttons, links, and other interactive elements
+     * - Typing text into input fields
+     * - Scrolling and navigating through interfaces
      *
-     * **Examples:**
+     * ### Method Signatures
+     * ```typescript
+     * act(goal: string, options?: ActOptions): Promise<AgentHistory>
+     * act(goal: string, imagePathOrBase64: string, options?: ActOptions): Promise<AgentHistory>
+     * ```
      *
-     * ```ts
-     * // Use chatId to maintain context across consecutive steps
-     * await aui.act("Search online for the current gold price", {
-     *   chatId: "session-gold-price"
+     * ### Parameters
+     * @param goal - A natural language instruction describing the task to accomplish.
+     *               Be specific and clear about the desired outcome.
+     * @param imagePathOrBase64 - (Optional) Path to an image file or base64-encoded image string.
+     *                           Used to provide additional visual context for the task.
+     * @param options - (Optional) Configuration options for the agent's behavior.
+     * @param options.chatId - A unique identifier to maintain context between related actions.
+     *                        Useful for multi-step tasks that require state preservation.
+     * @param options.agentHistory
+     *                          - (Optional) Previous interaction history to share between
+     *                           different agent instances. Enables cross-platform task coordination.
+     *
+     * ### Returns
+     * @returns Promise<AgentHistory> - A promise that resolves to the updated interaction history,
+     *                                 containing details about the actions taken and their outcomes.
+     *
+     * ### Throws
+     * - If the agent is not properly connected
+     * - If the provided goal cannot be understood or executed
+     * - If required UI elements are not found or accessible
+     * - If the image path is invalid or the base64 string is malformed
+     *
+     * ### Examples
+     *
+     * #### Basic Usage
+     * ```typescript
+     * // Simple task execution
+     * await aui.act("Open Chrome and navigate to google.com");
+     * ```
+     *
+     * #### Maintaining Context
+     * ```typescript
+     * // Multi-step task with context preservation
+     * await aui.act("Search for current gold prices", {
+     *   chatId: "gold-price-task"
      * });
-     * await aui.act("Create a new text file and type the gold price result into it", {
-     *   chatId: "session-gold-price"
+     *
+     * await aui.act("Create a new text file and save the price", {
+     *   chatId: "gold-price-task"
      * });
+     * ```
      *
-     * // Share history explicitly between separate agents (e.g., desktop and Android)
-     * // By default, the agent operates as a computer agent.
-     * // To control an Android device, you must configure it explicitly:
+     * #### Cross-Platform Coordination
+     * ```typescript
+     * // Share context between desktop and mobile agents
      * await auiAndroid.agent.configureAsAndroidAgent();
+     *
      * const history = await auiDesktop.act("Copy username from desktop app");
-     * await auiAndroid.act("Paste username into the mobile login screen", {
+     * await auiAndroid.act("Paste username into mobile login", {
      *   agentHistory: history
      * });
      * ```
      *
-     * @param {string} goal - A description of what the agent should achieve.
-     * @param {Object} [options] - Optional parameters to maintain or share context.
-     * @param {string} [options.chatId] - A session identifier used to persist memory between
-     *                                    consecutive `act` calls. When multiple actions share the
-     *                                    same `chatId`, the agent retains knowledge of prior steps,
-     *                                    such as extracted data or navigation history.
-     * @param {AgentHistory} [options.agentHistory] - A shared interaction history object that can be
-     *                                           passed between different agent clients (e.g., between
-     *                                           `auiDesktop` and `auiAndroid`) to ensure continuity
-     *                                           of understanding and task flow.
-     * @returns {Promise<AgentHistory>} - Updated action history after executing the goal.
-     * @throws {Error} If the agent is not connected when the method is called.
+     * #### Using Images for Context
+     * ```typescript
+     * // Using image file
+     * await aui.act(
+     *   "Click the 'Submit' button in the provided image",
+     *   'path/to/screenshot.png'
+     * );
+     *
+     * // Using base64 image
+     * const base64Image = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...";
+     * await aui.act(
+     *   "Click the 'Submit' button in the provided image",
+     *   base64Image
+     * );
+     * ```
+     *
+     * ### Best Practices
+     * 1. Be specific in your goal descriptions
+     * 2. Use chatId for related tasks to maintain context
+     * 3. Provide clear visual context when needed
+     * 4. Handle errors appropriately in your implementation
+     * 5. Consider using agentHistory for complex cross-platform workflows
      */
-    act(goal: string, options?: {
-        chatId?: string;
-        agentHistory?: AgentHistory;
-    }): Promise<AgentHistory>;
+    act(goal: string, options?: ActOptions): Promise<AgentHistory>;
+    act(goal: string, imagePathOrBase64String: string, options?: ActOptions): Promise<AgentHistory>;
 }

package/dist/cjs/execution/ui-control-client.js

@@ -738,56 +738,12 @@ class UiControlClient extends dsl_1.ApiCommands {
             };
         });
     }
-    /**
-     * Instructs the agent to achieve a specified goal through autonomous actions.
-     *
-     * The agent will analyze the screen, determine necessary steps, and perform actions
-     * to accomplish the goal. This may include clicking, typing, scrolling, and other
-     * interface interactions.
-     *
-     * The `options` parameter allows the caller to maintain contextual continuity across
-     * multiple `act` calls, either from the same or different agent interfaces.
-     *
-     * **Examples:**
-     *
-     * ```ts
-     * // Use chatId to maintain context across consecutive steps
-     * await aui.act("Search online for the current gold price", {
-     *   chatId: "session-gold-price"
-     * });
-     * await aui.act("Create a new text file and type the gold price result into it", {
-     *   chatId: "session-gold-price"
-     * });
-     *
-     * // Share history explicitly between separate agents (e.g., desktop and Android)
-     * // By default, the agent operates as a computer agent.
-     * // To control an Android device, you must configure it explicitly:
-     * await auiAndroid.agent.configureAsAndroidAgent();
-     * const history = await auiDesktop.act("Copy username from desktop app");
-     * await auiAndroid.act("Paste username into the mobile login screen", {
-     *   agentHistory: history
-     * });
-     * ```
-     *
-     * @param {string} goal - A description of what the agent should achieve.
-     * @param {Object} [options] - Optional parameters to maintain or share context.
-     * @param {string} [options.chatId] - A session identifier used to persist memory between
-     *                                    consecutive `act` calls. When multiple actions share the
-     *                                    same `chatId`, the agent retains knowledge of prior steps,
-     *                                    such as extracted data or navigation history.
-     * @param {AgentHistory} [options.agentHistory] - A shared interaction history object that can be
-     *                                           passed between different agent clients (e.g., between
-     *                                           `auiDesktop` and `auiAndroid`) to ensure continuity
-     *                                           of understanding and task flow.
-     * @returns {Promise<AgentHistory>} - Updated action history after executing the goal.
-     * @throws {Error} If the agent is not connected when the method is called.
-     */
-    act(goal, options) {
+    act(goal, imageOrOptions, options) {
         return __awaiter(this, void 0, void 0, function* () {
-            if (!this.agent.isConnected()) {
-                throw new Error('Agent is not connected, Please call connect() first');
+            if (typeof imageOrOptions === 'string') {
+                return this.agent.act(goal, imageOrOptions, options);
             }
-            return this.agent.act(goal, options);
+            return this.agent.act(goal, undefined, imageOrOptions);
         });
     }
 }

package/dist/cjs/lib/interactive_cli/create-example-project.js

@@ -176,10 +176,6 @@ class CreateExampleProject {
                                 title: 'Add eslint run command',
                                 task: () => __awaiter(this, void 0, void 0, function* () { return this.addESLintRunCommand(); }),
                             },
-                            {
-                                title: 'Add vscode settings',
-                                task: () => __awaiter(this, void 0, void 0, function* () { return this.addVSCodeSettings(); }),
-                            },
                         ]);
                     }),
                 }];
@@ -274,6 +270,7 @@ class CreateExampleProject {
             const tasks = new listr_1.default();
             tasks.add([
                 ...(yield this.copyTemplateProject()),
+                ...(yield this.addVSCodeSettings()),
                 ...(yield this.setupTestFrameWork()),
                 ...(yield this.copyESLintConfigFiles()),
                 ...(yield this.copyGitignore()),

package/dist/esm/core/models/anthropic/askui-agent.js

@@ -7,7 +7,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-import { DesktopKeyPressSequenceTool, DesktopSingleKeyPressTool, MouseClickTool, MouseMoveTool, MouseScrollTool, OsAgentHandler, ScreenShotTool, TypeTool, AgentErrorTool, AndroidSequenceKeyPressTool, AndroidSingleKeyPressTool, ExecuteShellCommandTool, } from './tools/os-agent-tools';
+import { DesktopPressAndReleaseKeysTool, MouseClickTool, MouseMoveTool, MouseScrollTool, OsAgentHandler, ScreenShotTool, TypeTool, AgentErrorTool, AndroidSequenceKeyPressTool, AndroidSingleKeyPressTool, ExecuteShellCommandTool, DesktopKeyHoldDownTool, DesktopKeyReleaseTool, MouseReleaseLeftButtonTool, MouseHoldLeftButtonDownTool, MouseDragAndDropTool, WaitTool, } from './tools/os-agent-tools';
 import { ClaudeAgent } from './claude-agent';
 export class AskUIAgent extends ClaudeAgent {
     constructor(executionRuntime) {
@@ -34,9 +34,14 @@ export class AskUIAgent extends ClaudeAgent {
                 new MouseMoveTool(this.osAgentHandler),
                 new MouseClickTool(this.osAgentHandler),
                 new MouseScrollTool(this.osAgentHandler),
-                new DesktopKeyPressSequenceTool(this.osAgentHandler),
-                new DesktopSingleKeyPressTool(this.osAgentHandler),
                 new TypeTool(this.osAgentHandler),
+                new DesktopPressAndReleaseKeysTool(this.osAgentHandler),
+                new DesktopKeyHoldDownTool(this.osAgentHandler),
+                new DesktopKeyReleaseTool(this.osAgentHandler),
+                new MouseHoldLeftButtonDownTool(this.osAgentHandler),
+                new MouseReleaseLeftButtonTool(this.osAgentHandler),
+                new MouseDragAndDropTool(this.osAgentHandler),
+                new WaitTool(),
             ];
             this.setTools(tools);
             this.setSystemPrompt(AskUIAgent.DesktopSystemPrompt);
@@ -57,6 +62,7 @@ export class AskUIAgent extends ClaudeAgent {
                 new AndroidSequenceKeyPressTool(this.osAgentHandler),
                 new TypeTool(this.osAgentHandler),
                 new ExecuteShellCommandTool(this.osAgentHandler),
+                new WaitTool(),
             ];
             this.setTools(tools);
             this.setSystemPrompt(AskUIAgent.AndroidSystemPrompt);
@@ -66,22 +72,13 @@ export class AskUIAgent extends ClaudeAgent {
 AskUIAgent.DesktopSystemPrompt = `
 <SYSTEM_CAPABILITY>
 You are an autonomous AI assistant operating on a ${process.platform} machine with ${process.arch} architecture. You have full access to the system and internet connectivity.
-Your main goal is to mimic a human user interacting with a desktop computer. So you should try to use the tools in a way that a human would use a mouse and keyboard to interact with a computer.
-
-Key Capabilities:
-* Full system control through mouse and keyboard interactions
-* Screen capture and analysis
-* Web browser automation and navigation
-* File system access and manipulation
-* PDF document handling and text extraction
-* Error handling and recovery mechanisms
-
-Available Tools:
-* Mouse control (move, click, scroll)
-* Keyboard input (single keys, key combinations, typing)
-* Screen capture and analysis
-* Error reporting and recovery
-
+Your main goal is to mimic a human user interacting with a desktop computer.
+Use a mouse and keyboard to interact with a computer, and take screenshots.
+* This is an interface to a desktop GUI. You do not have access to a terminal or applications menu. You must click on desktop icons to start applications.
+* Some applications may take time to start or process actions, so you may need to wait and take successive screenshots to see the results of your actions. E.g. if you click on Firefox and a window doesn't open, try taking another screenshot.
+* Whenever you intend to move the cursor to click on an element like an icon, you should consult a screenshot to determine the coordinates of the element before moving the cursor.
+* If you tried clicking on a program or link but it failed to load, even after waiting, try adjusting your cursor position so that the tip of the cursor visually falls on the element that you want to click.
+* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges unless asked.
 Current Date: ${new Date().toUTCString()} UTC
 </SYSTEM_CAPABILITY>
 
@@ -130,20 +127,11 @@ AskUIAgent.AndroidSystemPrompt = `
 <SYSTEM_CAPABILITY>
 You are an autonomous AI assistant operating on an Android device via ADB. The host machine is ${process.platform} with ${process.arch} architecture and internet connectivity.
 Your main goal is to mimic a human user interacting with an Android device. So you should try to use the tools in a way that a human would use a touch screen to interact with an Android device.
-
-Key Capabilities:
-* Full Android device control through ADB
-* Screen capture and analysis
-* Touch input simulation
-* Android-specific key events
-* Error handling and recovery mechanisms
-
-Available Tools:
-* Touch control (click, swipe, scroll)
-* Android key events (single and sequence)
-* Screen capture and analysis
-* Error reporting and recovery
-
+Use a gestures and adb commands to interact with the android device, and take screenshots.
+* Some applications may take time to start or process actions, so you may need to wait and take successive screenshots to see the results of your actions. E.g. if you click on Firefox and a window doesn't open, try taking another screenshot.
+* Whenever you intend to move the cursor to click on an element like an icon, you should consult a screenshot to determine the coordinates of the element before moving the cursor.
+* If you tried clicking on a program or link but it failed to load, even after waiting, try adjusting your cursor position so that the tip of the cursor visually falls on the element that you want to click.
+* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges unless asked.
 Current Date: ${new Date().toUTCString()} UTC
 </SYSTEM_CAPABILITY>
 

package/dist/esm/core/models/anthropic/claude-agent.d.ts

@@ -1,5 +1,6 @@
 import { Beta } from '@anthropic-ai/sdk/resources';
 import { BaseAgentTool } from './tools/base';
+import { BetaMessageParam } from '@anthropic-ai/sdk/resources/beta/messages';
 type PredictActResponseFunction = (params: {
     max_tokens: number;
     messages: Beta.BetaMessageParam[];
@@ -7,7 +8,16 @@ type PredictActResponseFunction = (params: {
     system?: string;
     tools?: any[];
     betas?: string[];
+    tool_choice?: {
+        type: 'tool' | 'any' | 'auto';
+        name?: string;
+    };
 }) => Promise<Beta.BetaMessage>;
+export type AgentHistory = BetaMessageParam[];
+export interface ActOptions {
+    chatId?: string;
+    agentHistory?: Beta.BetaMessageParam[];
+}
 export declare class ClaudeAgent {
     private predictActResponseFunction;
     private maxTokens;
@@ -19,7 +29,12 @@ export declare class ClaudeAgent {
     private _toolCollection;
     private tools;
     private history;
+    private toolChoice;
     constructor(predictActResponseFunction: PredictActResponseFunction);
+    setToolChoice(toolChoice: {
+        type: 'tool' | 'any' | 'auto';
+        name?: string;
+    }): void;
     setTools(tools: BaseAgentTool[]): void;
     addTool(tool: BaseAgentTool): void;
     listToolNames(): string[];
@@ -29,10 +44,7 @@ export declare class ClaudeAgent {
     private get toolCollection();
     private setHistory;
     private getHistory;
-    act(goal: string, options?: {
-        chatId?: string;
-        agentHistory?: Beta.BetaMessageParam[];
-    }): Promise<Beta.BetaMessageParam[]>;
+    act(goal: string, imagePathOrBase64String?: string, options?: ActOptions): Promise<Beta.BetaMessageParam[]>;
     private makeApiToolResult;
     private maybePrependSystemToolResult;
     private static filterNMostRecentImages;

package/dist/esm/core/models/anthropic/claude-agent.js

@@ -9,6 +9,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 import { ToolCollection } from './tools/base';
 import { logger } from '../../../lib/logger';
+import { Base64Image } from '../../../utils/base_64_image/base-64-image';
 export class ClaudeAgent {
     constructor(predictActResponseFunction) {
         this.predictActResponseFunction = predictActResponseFunction;
@@ -16,11 +17,17 @@ export class ClaudeAgent {
         this.onlyNMostRecentImages = 3;
         this.imageTruncationThreshold = 10;
         this.systemPrompt = '';
-        this.model = 'claude-3-5-sonnet-20241022';
-        this.betas = ['computer-use-2024-10-22'];
+        this.model = 'claude-sonnet-4-20250514';
+        this.betas = ['computer-use-2025-01-24'];
         this._toolCollection = undefined;
         this.tools = [];
         this.history = {};
+        this.toolChoice = {
+            type: 'any',
+        };
+    }
+    setToolChoice(toolChoice) {
+        this.toolChoice = toolChoice;
     }
     setTools(tools) {
         this._toolCollection = undefined;
@@ -37,7 +44,14 @@ export class ClaudeAgent {
         this.tools = this.tools.filter((tool) => tool.ToolName !== toolName);
     }
     setSystemPrompt(systemPrompt) {
-        this.systemPrompt = systemPrompt;
+        const enhancedPrompt = `${systemPrompt}
+        If you cannot complete a request due to safety concerns, please:
+        1. Explain what specific aspect is problematic
+        2. Suggest alternative approaches that would be acceptable
+        3. Provide partial assistance where possible within guidelines.
+        Raise an exception After you have provided the above information. include the error message in the exception.
+        `;
+        this.systemPrompt = enhancedPrompt;
     }
     IsConfigured() {
         return this.tools.length > 0 && this.systemPrompt !== '';
@@ -54,7 +68,7 @@ export class ClaudeAgent {
     getHistory(key) {
         return this.history[key] || [];
     }
-    act(goal, options) {
+    act(goal, imagePathOrBase64String, options) {
         return __awaiter(this, void 0, void 0, function* () {
             if (!goal.trim()) {
                 throw new Error('Goal cannot be empty');
@@ -70,8 +84,24 @@ export class ClaudeAgent {
                 messages.push(...this.getHistory(options.chatId));
             }
             // Add the new goal as a user message
+            const userContent = [{
+                    type: 'text',
+                    text: goal,
+                }];
+            if (imagePathOrBase64String !== undefined) {
+                const image = yield Base64Image.fromPathOrString(imagePathOrBase64String);
+                const imageString = image.toString(false);
+                userContent.push({
+                    type: 'image',
+                    source: {
+                        type: 'base64',
+                        media_type: 'image/png',
+                        data: imageString,
+                    },
+                });
+            }
             messages.push({
-                content: goal,
+                content: userContent,
                 role: 'user',
             });
             if (this.onlyNMostRecentImages) {
@@ -85,7 +115,15 @@ export class ClaudeAgent {
                     system: this.systemPrompt,
                     tools: (new ToolCollection(this.tools).toParams()),
                     betas: this.betas,
+                    tool_choice: this.toolChoice,
                 });
+                if (response.stop_reason === 'refusal') {
+                    const refusalMessage = response.content
+                        .filter(block => block.type === 'text')
+                        .map(block => block.text)
+                        .join(' ');
+                    throw new Error(`Agent refused to answer: ${refusalMessage || 'The request violates agent\'s usage policies'}`);
+                }
                 messages.push({
                     content: response.content,
                     role: 'assistant',

package/dist/esm/core/models/anthropic/index.d.ts

@@ -1,4 +1,3 @@
-import { BetaMessageParam } from '@anthropic-ai/sdk/resources/beta/messages';
-export type AgentHistory = BetaMessageParam[];
 export { AskUIAgent } from './askui-agent';
 export { ToolFailure, ToolError, BaseAgentTool } from './tools/base';
+export { AgentHistory, ActOptions } from './claude-agent';

package/dist/esm/core/models/anthropic/tools/os-agent-tools.d.ts

@@ -5,21 +5,36 @@ import { ExecutionRuntime } from '../../../../execution/execution-runtime';
 import { ControlCommand } from '../../../ui-control-commands';
 export declare class OsAgentHandler {
     private AgentOsClient;
-    private TargetResolution;
+    private targetResolution;
     private screenDimensions;
+    private paddingInfo;
     constructor(AgentOsClient: ExecutionRuntime, screenDimensions: {
         width: number;
         height: number;
     });
+    private updatePaddingInfo;
     static createInstance(AgentOsClient: ExecutionRuntime): Promise<OsAgentHandler>;
     getTargetResolution(): {
         width: number;
         height: number;
     };
+    getScreenDimensions(): {
+        width: number;
+        height: number;
+    };
     setTargetResolution(width: number, height: number): void;
     takeScreenshot(): Promise<string>;
     private scaleCoordinates;
     requestControl(controlCommand: ControlCommand): Promise<void>;
+    mouseMove(x: number, y: number): Promise<void>;
+    mouseClick(button: "left" | "right" | "middle", doubleClick: boolean): Promise<void>;
+    mouseScroll(dx: number, dy: number): Promise<void>;
+    mouseHoldLeftButtonDown(): Promise<void>;
+    mouseReleaseLeftButton(): Promise<void>;
+    desktopKeyPressAndRelease(key: PC_AND_MODIFIER_KEY, modifiers?: MODIFIER_KEY[]): Promise<void>;
+    desktopKeyHoldDown(key: PC_AND_MODIFIER_KEY, modifiers?: MODIFIER_KEY[]): Promise<void>;
+    desktopKeyRelease(key: PC_AND_MODIFIER_KEY, modifiers?: MODIFIER_KEY[]): Promise<void>;
+    typeText(text: string): Promise<void>;
 }
 export declare class ScreenShotTool extends BaseAgentTool {
     private osAgentHandler;
@@ -54,21 +69,53 @@ export declare class MouseScrollTool extends BaseAgentTool {
     }): Promise<ToolResult>;
     toParams(): BetaTool;
 }
-export declare class DesktopKeyPressSequenceTool extends BaseAgentTool {
+export declare class MouseDragAndDropTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(command: {
+        startX: number;
+        startY: number;
+        endX: number;
+        endY: number;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class MouseHoldLeftButtonDownTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class MouseReleaseLeftButtonTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class DesktopPressAndReleaseKeysTool extends BaseAgentTool {
     private osAgentHandler;
     constructor(osAgentHandler: OsAgentHandler);
     execute(command: {
         key: PC_KEY;
-        firstModifier?: MODIFIER_KEY;
-        secondModifier?: MODIFIER_KEY;
+        modifiers?: MODIFIER_KEY[];
     }): Promise<ToolResult>;
     toParams(): BetaTool;
 }
-export declare class DesktopSingleKeyPressTool extends BaseAgentTool {
+export declare class DesktopKeyHoldDownTool extends BaseAgentTool {
     private osAgentHandler;
     constructor(osAgentHandler: OsAgentHandler);
     execute(command: {
         key: PC_AND_MODIFIER_KEY;
+        modifiers?: MODIFIER_KEY[];
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class DesktopKeyReleaseTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(command: {
+        key: PC_AND_MODIFIER_KEY;
+        modifiers?: MODIFIER_KEY[];
     }): Promise<ToolResult>;
     toParams(): BetaTool;
 }
@@ -111,3 +158,10 @@ export declare class ExecuteShellCommandTool extends BaseAgentTool {
     }): Promise<ToolResult>;
     toParams(): BetaTool;
 }
+export declare class WaitTool extends BaseAgentTool {
+    constructor();
+    execute(command: {
+        milliseconds: number;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}