askui 0.25.0 → 0.26.0

package/dist/cjs/core/models/anthropic/tools/os-agent-tools.js

@@ -9,7 +9,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ExecuteShellCommandTool = exports.AgentErrorTool = exports.AndroidSequenceKeyPressTool = exports.AndroidSingleKeyPressTool = exports.TypeTool = exports.DesktopSingleKeyPressTool = exports.DesktopKeyPressSequenceTool = exports.MouseScrollTool = exports.MouseClickTool = exports.MouseMoveTool = exports.ScreenShotTool = exports.OsAgentHandler = void 0;
+exports.WaitTool = exports.ExecuteShellCommandTool = exports.AgentErrorTool = exports.AndroidSequenceKeyPressTool = exports.AndroidSingleKeyPressTool = exports.TypeTool = exports.DesktopKeyReleaseTool = exports.DesktopKeyHoldDownTool = exports.DesktopPressAndReleaseKeysTool = exports.MouseReleaseLeftButtonTool = exports.MouseHoldLeftButtonDownTool = exports.MouseDragAndDropTool = exports.MouseScrollTool = exports.MouseClickTool = exports.MouseMoveTool = exports.ScreenShotTool = exports.OsAgentHandler = void 0;
 const dsl_1 = require("../../../../execution/dsl");
 const base_1 = require("./base");
 const ui_control_commands_1 = require("../../../ui-control-commands");
@@ -18,9 +18,40 @@ const agent_errors_1 = require("./agent-errors");
 class OsAgentHandler {
     constructor(AgentOsClient, screenDimensions) {
         this.AgentOsClient = AgentOsClient;
-        this.TargetResolution = { width: 1280, height: 800 };
+        this.targetResolution = { width: 1280, height: 800 };
+        this.paddingInfo = null;
         this.screenDimensions = screenDimensions;
+        this.updatePaddingInfo();
+    }
+    updatePaddingInfo() {
+        const targetAspectRatio = this.targetResolution.width / this.targetResolution.height;
+        const screenAspectRatio = this.screenDimensions.width / this.screenDimensions.height;
+        let scaledWidth;
+        let scaledHeight;
+        let scaleFactor;
+        let padLeft = 0;
+        let padTop = 0;
+        if (targetAspectRatio > screenAspectRatio) {
+            scaleFactor = this.targetResolution.height / this.screenDimensions.height;
+            scaledWidth = Math.floor(this.screenDimensions.width * scaleFactor);
+            scaledHeight = this.targetResolution.height;
+            padLeft = Math.floor((this.targetResolution.width - scaledWidth) / 2);
+        }
+        else {
+            scaleFactor = this.targetResolution.width / this.screenDimensions.width;
+            scaledWidth = this.targetResolution.width;
+            scaledHeight = Math.floor(this.screenDimensions.height * scaleFactor);
+            padTop = Math.floor((this.targetResolution.height - scaledHeight) / 2);
+        }
+        this.paddingInfo = {
+            scaleFactor,
+            scaledWidth,
+            scaledHeight,
+            padLeft,
+            padTop
+        };
     }
+    // Add image support to act, an check for function overload in typescript.
     static createInstance(AgentOsClient) {
         return __awaiter(this, void 0, void 0, function* () {
             const base64ImageString = yield AgentOsClient.getScreenshot();
@@ -32,10 +63,14 @@ class OsAgentHandler {
         });
     }
     getTargetResolution() {
-        return this.TargetResolution;
+        return this.targetResolution;
+    }
+    getScreenDimensions() {
+        return this.screenDimensions;
     }
     setTargetResolution(width, height) {
-        this.TargetResolution = { width, height };
+        this.targetResolution = { width, height };
+        this.updatePaddingInfo();
     }
     takeScreenshot() {
         return __awaiter(this, void 0, void 0, function* () {
@@ -46,36 +81,130 @@ class OsAgentHandler {
                 width: image_info.width,
                 height: image_info.height,
             };
-            const resized_image = yield base64Image.resizeWithSameAspectRatio(this.TargetResolution.width, this.TargetResolution.height);
+            this.updatePaddingInfo();
+            const resized_image = yield base64Image.resizeWithSameAspectRatio(this.targetResolution.width, this.targetResolution.height);
             return resized_image.toString(false);
         });
     }
     scaleCoordinates(source, x, y) {
-        const xScalingFactor = this.TargetResolution.width / this.screenDimensions.width;
-        const yScalingFactor = this.TargetResolution.height / this.screenDimensions.height;
+        if (!this.paddingInfo) {
+            throw new base_1.ToolError('Padding information not initialized');
+        }
+        const { scaleFactor, scaledWidth, scaledHeight, padLeft, padTop } = this.paddingInfo;
         if (source === 'api') {
-            if (x > this.TargetResolution.width || y > this.TargetResolution.height || x < 0 || y < 0) {
+            if (x > this.targetResolution.width || y > this.targetResolution.height || x < 0 || y < 0) {
                 throw new base_1.ToolError(`Coordinates ${x}, ${y} are outside screen bounds `
-                    + `(${this.TargetResolution.width}x${this.TargetResolution.height})`);
+                    + `(${this.targetResolution.width}x${this.targetResolution.height})`);
+            }
+            const adjustedX = x - padLeft;
+            const adjustedY = y - padTop;
+            if (adjustedX < 0 || adjustedX > scaledWidth || adjustedY < 0 || adjustedY > scaledHeight) {
+                throw new base_1.ToolError(`Coordinates ${x}, ${y} are outside the scaled image area `
+                    + `(${scaledWidth}x${scaledHeight} with padding ${padLeft},${padTop})`);
             }
             return [
-                Math.round(x / xScalingFactor),
-                Math.round(y / yScalingFactor),
+                Math.round(adjustedX / scaleFactor),
+                Math.round(adjustedY / scaleFactor),
             ];
         }
-        return [
-            Math.round(x * xScalingFactor),
-            Math.round(y * yScalingFactor),
-        ];
+        const apiX = Math.round(x * scaleFactor) + padLeft;
+        const apiY = Math.round(y * scaleFactor) + padTop;
+        return [apiX, apiY];
     }
     requestControl(controlCommand) {
         return __awaiter(this, void 0, void 0, function* () {
             for (const action of controlCommand.actions) {
-                [action.position.x, action.position.y] = this.scaleCoordinates('api', action.position.x, action.position.y);
+                if (action.inputEvent === ui_control_commands_1.InputEvent.MOUSE_MOVE || action.inputEvent === ui_control_commands_1.InputEvent.MOUSE_SCROLL) {
+                    [action.position.x, action.position.y] = this.scaleCoordinates('api', action.position.x, action.position.y);
+                }
             }
             yield this.AgentOsClient.requestControl(controlCommand);
         });
     }
+    mouseMove(x, y) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.MOUSE_MOVE, { x, y }, '', {})]);
+            yield this.requestControl(controlCommand);
+        });
+    }
+    mouseClick(button, doubleClick) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let action = ui_control_commands_1.InputEvent.MOUSE_CLICK_LEFT;
+            if (doubleClick) {
+                if (button === "left") {
+                    action = ui_control_commands_1.InputEvent.MOUSE_CLICK_DOUBLE_LEFT;
+                }
+                else if (button === "right") {
+                    action = ui_control_commands_1.InputEvent.MOUSE_CLICK_DOUBLE_RIGHT;
+                }
+                else if (button === "middle") {
+                    action = ui_control_commands_1.InputEvent.MOUSE_CLICK_DOUBLE_MIDDLE;
+                }
+            }
+            else {
+                if (button === "right") {
+                    action = ui_control_commands_1.InputEvent.MOUSE_CLICK_RIGHT;
+                }
+                else if (button === "middle") {
+                    action = ui_control_commands_1.InputEvent.MOUSE_CLICK_MIDDLE;
+                }
+            }
+            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(action, { x: 0, y: 0 }, '', {})]);
+            yield this.requestControl(controlCommand);
+        });
+    }
+    mouseScroll(dx, dy) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.MOUSE_SCROLL, { x: dx, y: dy }, '', {})]);
+            yield this.requestControl(controlCommand);
+        });
+    }
+    mouseHoldLeftButtonDown() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.MOUSE_DOWN, { x: 0, y: 0 }, '', {})]);
+            yield this.requestControl(controlCommand);
+        });
+    }
+    mouseReleaseLeftButton() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.MOUSE_UP, { x: 0, y: 0 }, '', {})]);
+            yield this.requestControl(controlCommand);
+        });
+    }
+    desktopKeyPressAndRelease(key_1) {
+        return __awaiter(this, arguments, void 0, function* (key, modifiers = []) {
+            let keyString = key;
+            if (modifiers.length > 0) {
+                keyString = `${modifiers.join('+')}+${key}`;
+            }
+            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.PRESS_KEY_SEQUENCE, { x: 0, y: 0 }, keyString, {})]);
+            yield this.requestControl(controlCommand);
+        });
+    }
+    desktopKeyHoldDown(key_1) {
+        return __awaiter(this, arguments, void 0, function* (key, modifiers = []) {
+            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.KEY_PRESS, { x: 0, y: 0 }, '', {
+                    key: key,
+                    modifiers: modifiers,
+                })]);
+            yield this.requestControl(controlCommand);
+        });
+    }
+    desktopKeyRelease(key_1) {
+        return __awaiter(this, arguments, void 0, function* (key, modifiers = []) {
+            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.KEY_RELEASE, { x: 0, y: 0 }, '', {
+                    key: key,
+                    modifiers: modifiers,
+                })]);
+            yield this.requestControl(controlCommand);
+        });
+    }
+    typeText(text) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.TYPE, { x: 0, y: 0 }, text, {})]);
+            yield this.requestControl(controlCommand);
+        });
+    }
 }
 exports.OsAgentHandler = OsAgentHandler;
 class ScreenShotTool extends base_1.BaseAgentTool {
@@ -95,7 +224,7 @@ class ScreenShotTool extends base_1.BaseAgentTool {
     toParams() {
         return {
             name: 'screenshot_tool',
-            description: 'Takes a screenshot of the current screen and returns it as a base64 image',
+            description: 'Takes a screenshot of the current screen and returns it as a base64 image.',
             input_schema: { type: 'object', properties: {}, required: [] },
         };
     }
@@ -108,8 +237,7 @@ class MouseMoveTool extends base_1.BaseAgentTool {
     }
     execute(command) {
         return __awaiter(this, void 0, void 0, function* () {
-            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.MOUSE_MOVE, { x: command.x, y: command.y }, '', {})]);
-            yield this.osAgentHandler.requestControl(controlCommand);
+            yield this.osAgentHandler.mouseMove(command.x, command.y);
             return {
                 output: `Moved mouse to (${command.x}, ${command.y})`,
             };
@@ -124,11 +252,11 @@ class MouseMoveTool extends base_1.BaseAgentTool {
                 properties: {
                     x: {
                         type: 'number',
-                        description: 'The x coordinate of the element to click on',
+                        description: 'The x (pixels from the left edge) coordinate to move the mouse to',
                     },
                     y: {
                         type: 'number',
-                        description: 'The y coordinate of the element to click on',
+                        description: 'The y (pixels from the top edge) coordinate to move the mouse to',
                     },
                 },
             },
@@ -143,33 +271,7 @@ class MouseClickTool extends base_1.BaseAgentTool {
     }
     execute(command) {
         return __awaiter(this, void 0, void 0, function* () {
-            let controlCommand;
-            if (command.doubleClick) {
-                if (command.button === 'left') {
-                    controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.MOUSE_CLICK_DOUBLE_LEFT, { x: 0, y: 0 }, '', {})]);
-                }
-                if (command.button === 'right') {
-                    controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.MOUSE_CLICK_DOUBLE_RIGHT, { x: 0, y: 0 }, '', {})]);
-                }
-                if (command.button === 'middle') {
-                    controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.MOUSE_CLICK_DOUBLE_MIDDLE, { x: 0, y: 0 }, '', {})]);
-                }
-            }
-            else {
-                if (command.button === 'left') {
-                    controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.MOUSE_CLICK_LEFT, { x: 0, y: 0 }, '', {})]);
-                }
-                if (command.button === 'right') {
-                    controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.MOUSE_CLICK_RIGHT, { x: 0, y: 0 }, '', {})]);
-                }
-                if (command.button === 'middle') {
-                    controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.MOUSE_CLICK_MIDDLE, { x: 0, y: 0 }, '', {})]);
-                }
-            }
-            if (!controlCommand) {
-                throw new base_1.ToolError('Invalid input parameter for mouse click tool');
-            }
-            yield this.osAgentHandler.requestControl(controlCommand);
+            yield this.osAgentHandler.mouseClick(command.button, command.doubleClick);
             const returnedMessage = command.doubleClick ? `Double clicked ${command.button} button` : `Clicked ${command.button} button`;
             return {
                 output: returnedMessage,
@@ -206,8 +308,7 @@ class MouseScrollTool extends base_1.BaseAgentTool {
     }
     execute(command) {
         return __awaiter(this, void 0, void 0, function* () {
-            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.MOUSE_SCROLL, { x: command.dx, y: command.dy }, '', {})]);
-            yield this.osAgentHandler.requestControl(controlCommand);
+            yield this.osAgentHandler.mouseScroll(command.dx, command.dy);
             return {
                 output: `Scrolled by (${command.dx}, ${command.dy})`,
             };
@@ -222,11 +323,11 @@ class MouseScrollTool extends base_1.BaseAgentTool {
                 properties: {
                     dx: {
                         type: 'number',
-                        description: 'The amount to scroll horizontally',
+                        description: 'The amount to scroll horizontally (positive is right, negative is left)',
                     },
                     dy: {
                         type: 'number',
-                        description: 'The amount to scroll vertically',
+                        description: 'The amount to scroll vertically (positive is down, negative is up)',
                     },
                 },
                 required: ['dx', 'dy'],
@@ -235,20 +336,107 @@ class MouseScrollTool extends base_1.BaseAgentTool {
     }
 }
 exports.MouseScrollTool = MouseScrollTool;
-class DesktopKeyPressSequenceTool extends base_1.BaseAgentTool {
+class MouseDragAndDropTool extends base_1.BaseAgentTool {
     constructor(osAgentHandler) {
         super();
         this.osAgentHandler = osAgentHandler;
     }
     execute(command) {
         return __awaiter(this, void 0, void 0, function* () {
-            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.PRESS_KEY_SEQUENCE, { x: 0, y: 0 }, command.key, {
-                    firstModifier: command.firstModifier || '',
-                    secondModifier: command.secondModifier || '',
-                })]);
-            yield this.osAgentHandler.requestControl(controlCommand);
+            yield this.osAgentHandler.mouseMove(command.startX, command.startY);
+            yield this.osAgentHandler.mouseHoldLeftButtonDown();
+            yield this.osAgentHandler.mouseMove(command.endX, command.endY);
+            yield this.osAgentHandler.mouseReleaseLeftButton();
             return {
-                output: `Pressed key ${command.key} with modifiers ${command.firstModifier || ''} ${command.secondModifier || ''}`,
+                output: `Dragged from (${command.startX}, ${command.startY}) to (${command.endX}, ${command.endY})`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'mouse_drag_and_drop_tool',
+            description: 'Drags the mouse from the specified start coordinates to the specified end coordinates. The top left corner of the screen is (0,0)',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    startX: {
+                        type: 'number',
+                        description: 'The x (pixels from the left edge) coordinate of the start position',
+                    },
+                    startY: {
+                        type: 'number',
+                        description: 'The y (pixels from the top edge) coordinate of the start position',
+                    },
+                    endX: {
+                        type: 'number',
+                        description: 'The x (pixels from the left edge) coordinate of the end position',
+                    },
+                    endY: {
+                        type: 'number',
+                        description: 'The y (pixels from the top edge) coordinate of the end position',
+                    },
+                },
+                required: ['startX', 'startY', 'endX', 'endY'],
+            },
+        };
+    }
+}
+exports.MouseDragAndDropTool = MouseDragAndDropTool;
+class MouseHoldLeftButtonDownTool extends base_1.BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute() {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.osAgentHandler.mouseHoldLeftButtonDown();
+            return {
+                output: 'Holding down left mouse button',
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'mouse_hold_left_button_down_tool',
+            description: 'Hold down the left mouse button at the current position.',
+            input_schema: { type: 'object', properties: {}, required: [] },
+        };
+    }
+}
+exports.MouseHoldLeftButtonDownTool = MouseHoldLeftButtonDownTool;
+class MouseReleaseLeftButtonTool extends base_1.BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute() {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.osAgentHandler.mouseReleaseLeftButton();
+            return {
+                output: 'Released left mouse button',
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'mouse_release_left_button_tool',
+            description: 'Release the left mouse button at the current position.',
+            input_schema: { type: 'object', properties: {}, required: [] },
+        };
+    }
+}
+exports.MouseReleaseLeftButtonTool = MouseReleaseLeftButtonTool;
+class DesktopPressAndReleaseKeysTool extends base_1.BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const modifiers = command.modifiers || [];
+            yield this.osAgentHandler.desktopKeyPressAndRelease(command.key, modifiers);
+            return {
+                output: `Pressed key ${command.key} with modifiers ${modifiers.join(' ')}`,
             };
         });
     }
@@ -264,15 +452,54 @@ class DesktopKeyPressSequenceTool extends base_1.BaseAgentTool {
                         enum: dsl_1.PC_KEY_VALUES,
                         description: 'The key to press',
                     },
-                    firstModifier: {
-                        type: 'string',
-                        enum: dsl_1.MODIFIER_KEY_VALUES,
-                        description: 'The first modifier key',
+                    modifiers: {
+                        type: 'array',
+                        items: {
+                            type: 'string',
+                            enum: dsl_1.MODIFIER_KEY_VALUES,
+                        },
+                        description: 'The modifiers to press',
                     },
-                    secondModifier: {
+                },
+                required: ['key'],
+            },
+        };
+    }
+}
+exports.DesktopPressAndReleaseKeysTool = DesktopPressAndReleaseKeysTool;
+class DesktopKeyHoldDownTool extends base_1.BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const modifiers = command.modifiers || [];
+            yield this.osAgentHandler.desktopKeyHoldDown(command.key, modifiers);
+            return {
+                output: `Holding down key ${command.key} with modifiers ${modifiers.join(' ')}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'desktop_key_hold_down_tool',
+            description: 'Hold down a key and optional modifiers. Keys will be still pressed after the tool is finished.',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    key: {
                         type: 'string',
-                        enum: dsl_1.MODIFIER_KEY_VALUES,
-                        description: 'The second modifier key',
+                        enum: [...dsl_1.PC_KEY_VALUES, ...dsl_1.MODIFIER_KEY_VALUES],
+                        description: 'The key to hold down',
+                    },
+                    modifiers: {
+                        type: 'array',
+                        items: {
+                            type: 'string',
+                            enum: dsl_1.MODIFIER_KEY_VALUES,
+                        },
+                        description: 'The modifiers to hold down',
                     },
                 },
                 required: ['key'],
@@ -280,32 +507,40 @@ class DesktopKeyPressSequenceTool extends base_1.BaseAgentTool {
         };
     }
 }
-exports.DesktopKeyPressSequenceTool = DesktopKeyPressSequenceTool;
-class DesktopSingleKeyPressTool extends base_1.BaseAgentTool {
+exports.DesktopKeyHoldDownTool = DesktopKeyHoldDownTool;
+class DesktopKeyReleaseTool extends base_1.BaseAgentTool {
     constructor(osAgentHandler) {
         super();
         this.osAgentHandler = osAgentHandler;
     }
     execute(command) {
         return __awaiter(this, void 0, void 0, function* () {
-            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.PRESS_KEY_SEQUENCE, { x: 0, y: 0 }, command.key, {})]);
-            yield this.osAgentHandler.requestControl(controlCommand);
+            const modifiers = command.modifiers || [];
+            yield this.osAgentHandler.desktopKeyRelease(command.key, modifiers);
             return {
-                output: `Pressed key ${command.key}`,
+                output: `Released key ${command.key} with modifiers ${modifiers.join(' ')}`,
             };
         });
     }
     toParams() {
         return {
-            name: 'desktop_single_key_press_tool',
-            description: 'Presses a single key',
+            name: 'desktop_key_release_tool',
+            description: 'Releases a key and optional modifiers. This can be used after keys were held down with the desktop_key_hold_down_tool',
             input_schema: {
                 type: 'object',
                 properties: {
                     key: {
                         type: 'string',
                         enum: [...dsl_1.PC_KEY_VALUES, ...dsl_1.MODIFIER_KEY_VALUES],
-                        description: 'The key to press',
+                        description: 'The key to release',
+                    },
+                    modifiers: {
+                        type: 'array',
+                        items: {
+                            type: 'string',
+                            enum: dsl_1.MODIFIER_KEY_VALUES,
+                        },
+                        description: 'The modifiers to release',
                     },
                 },
                 required: ['key'],
@@ -313,7 +548,7 @@ class DesktopSingleKeyPressTool extends base_1.BaseAgentTool {
         };
     }
 }
-exports.DesktopSingleKeyPressTool = DesktopSingleKeyPressTool;
+exports.DesktopKeyReleaseTool = DesktopKeyReleaseTool;
 class TypeTool extends base_1.BaseAgentTool {
     constructor(osAgentHandler) {
         super();
@@ -474,3 +709,33 @@ class ExecuteShellCommandTool extends base_1.BaseAgentTool {
     }
 }
 exports.ExecuteShellCommandTool = ExecuteShellCommandTool;
+class WaitTool extends base_1.BaseAgentTool {
+    constructor() {
+        super();
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield new Promise(resolve => setTimeout(resolve, command.milliseconds));
+            return {
+                output: `Waited for ${command.milliseconds} milliseconds`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'wait_tool',
+            description: 'Waits for a specified number of milliseconds',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    milliseconds: {
+                        type: 'number',
+                        description: 'The number of milliseconds to wait',
+                    },
+                },
+                required: ['milliseconds'],
+            },
+        };
+    }
+}
+exports.WaitTool = WaitTool;

package/dist/cjs/core/ui-control-commands/input-event.d.ts

@@ -16,5 +16,7 @@ export declare enum InputEvent {
     MOUSE_MOVE = "MOUSE_MOVE",
     MOUSE_DOWN = "MOUSE_DOWN",
     MOUSE_UP = "MOUSE_UP",
-    EXECUTE_COMMAND = "EXECUTE_COMMAND"
+    EXECUTE_COMMAND = "EXECUTE_COMMAND",
+    KEY_PRESS = "KEY_PRESS",
+    KEY_RELEASE = "KEY_RELEASE"
 }

package/dist/cjs/core/ui-control-commands/input-event.js

@@ -21,4 +21,6 @@ var InputEvent;
     InputEvent["MOUSE_DOWN"] = "MOUSE_DOWN";
     InputEvent["MOUSE_UP"] = "MOUSE_UP";
     InputEvent["EXECUTE_COMMAND"] = "EXECUTE_COMMAND";
+    InputEvent["KEY_PRESS"] = "KEY_PRESS";
+    InputEvent["KEY_RELEASE"] = "KEY_RELEASE";
 })(InputEvent || (exports.InputEvent = InputEvent = {}));

package/dist/cjs/execution/execution-runtime.d.ts

@@ -47,5 +47,9 @@ export declare class ExecutionRuntime {
         system?: string;
         tools?: object[];
         betas?: string[];
+        tool_choice?: {
+            type: 'tool' | 'any' | 'auto';
+            name?: string;
+        };
     }): Promise<BetaMessage>;
 }

package/dist/cjs/execution/inference-client.d.ts

@@ -26,6 +26,10 @@ export declare class InferenceClient {
     predictVQAAnswer(prompt: string, image: string, config?: object): Promise<any>;
     predictActResponse(params: {
         max_tokens: number;
+        tool_choice?: {
+            type: 'tool' | 'any' | 'auto';
+            name?: string;
+        };
         messages: BetaMessageParam[];
         model: string;
         system?: string;