askui 0.25.1 → 0.26.0

package/dist/esm/core/models/anthropic/tools/os-agent-tools.js

@@ -15,9 +15,40 @@ import { AgentError } from './agent-errors';
 export class OsAgentHandler {
     constructor(AgentOsClient, screenDimensions) {
         this.AgentOsClient = AgentOsClient;
-        this.TargetResolution = { width: 1280, height: 800 };
+        this.targetResolution = { width: 1280, height: 800 };
+        this.paddingInfo = null;
         this.screenDimensions = screenDimensions;
+        this.updatePaddingInfo();
+    }
+    updatePaddingInfo() {
+        const targetAspectRatio = this.targetResolution.width / this.targetResolution.height;
+        const screenAspectRatio = this.screenDimensions.width / this.screenDimensions.height;
+        let scaledWidth;
+        let scaledHeight;
+        let scaleFactor;
+        let padLeft = 0;
+        let padTop = 0;
+        if (targetAspectRatio > screenAspectRatio) {
+            scaleFactor = this.targetResolution.height / this.screenDimensions.height;
+            scaledWidth = Math.floor(this.screenDimensions.width * scaleFactor);
+            scaledHeight = this.targetResolution.height;
+            padLeft = Math.floor((this.targetResolution.width - scaledWidth) / 2);
+        }
+        else {
+            scaleFactor = this.targetResolution.width / this.screenDimensions.width;
+            scaledWidth = this.targetResolution.width;
+            scaledHeight = Math.floor(this.screenDimensions.height * scaleFactor);
+            padTop = Math.floor((this.targetResolution.height - scaledHeight) / 2);
+        }
+        this.paddingInfo = {
+            scaleFactor,
+            scaledWidth,
+            scaledHeight,
+            padLeft,
+            padTop
+        };
     }
+    // Add image support to act, an check for function overload in typescript.
     static createInstance(AgentOsClient) {
         return __awaiter(this, void 0, void 0, function* () {
             const base64ImageString = yield AgentOsClient.getScreenshot();
@@ -29,10 +60,14 @@ export class OsAgentHandler {
         });
     }
     getTargetResolution() {
-        return this.TargetResolution;
+        return this.targetResolution;
+    }
+    getScreenDimensions() {
+        return this.screenDimensions;
     }
     setTargetResolution(width, height) {
-        this.TargetResolution = { width, height };
+        this.targetResolution = { width, height };
+        this.updatePaddingInfo();
     }
     takeScreenshot() {
         return __awaiter(this, void 0, void 0, function* () {
@@ -43,36 +78,130 @@ export class OsAgentHandler {
                 width: image_info.width,
                 height: image_info.height,
             };
-            const resized_image = yield base64Image.resizeWithSameAspectRatio(this.TargetResolution.width, this.TargetResolution.height);
+            this.updatePaddingInfo();
+            const resized_image = yield base64Image.resizeWithSameAspectRatio(this.targetResolution.width, this.targetResolution.height);
             return resized_image.toString(false);
         });
     }
     scaleCoordinates(source, x, y) {
-        const xScalingFactor = this.TargetResolution.width / this.screenDimensions.width;
-        const yScalingFactor = this.TargetResolution.height / this.screenDimensions.height;
+        if (!this.paddingInfo) {
+            throw new ToolError('Padding information not initialized');
+        }
+        const { scaleFactor, scaledWidth, scaledHeight, padLeft, padTop } = this.paddingInfo;
         if (source === 'api') {
-            if (x > this.TargetResolution.width || y > this.TargetResolution.height || x < 0 || y < 0) {
+            if (x > this.targetResolution.width || y > this.targetResolution.height || x < 0 || y < 0) {
                 throw new ToolError(`Coordinates ${x}, ${y} are outside screen bounds `
-                    + `(${this.TargetResolution.width}x${this.TargetResolution.height})`);
+                    + `(${this.targetResolution.width}x${this.targetResolution.height})`);
+            }
+            const adjustedX = x - padLeft;
+            const adjustedY = y - padTop;
+            if (adjustedX < 0 || adjustedX > scaledWidth || adjustedY < 0 || adjustedY > scaledHeight) {
+                throw new ToolError(`Coordinates ${x}, ${y} are outside the scaled image area `
+                    + `(${scaledWidth}x${scaledHeight} with padding ${padLeft},${padTop})`);
             }
             return [
-                Math.round(x / xScalingFactor),
-                Math.round(y / yScalingFactor),
+                Math.round(adjustedX / scaleFactor),
+                Math.round(adjustedY / scaleFactor),
             ];
         }
-        return [
-            Math.round(x * xScalingFactor),
-            Math.round(y * yScalingFactor),
-        ];
+        const apiX = Math.round(x * scaleFactor) + padLeft;
+        const apiY = Math.round(y * scaleFactor) + padTop;
+        return [apiX, apiY];
     }
     requestControl(controlCommand) {
         return __awaiter(this, void 0, void 0, function* () {
             for (const action of controlCommand.actions) {
-                [action.position.x, action.position.y] = this.scaleCoordinates('api', action.position.x, action.position.y);
+                if (action.inputEvent === InputEvent.MOUSE_MOVE || action.inputEvent === InputEvent.MOUSE_SCROLL) {
+                    [action.position.x, action.position.y] = this.scaleCoordinates('api', action.position.x, action.position.y);
+                }
             }
             yield this.AgentOsClient.requestControl(controlCommand);
         });
     }
+    mouseMove(x, y) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.MOUSE_MOVE, { x, y }, '', {})]);
+            yield this.requestControl(controlCommand);
+        });
+    }
+    mouseClick(button, doubleClick) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let action = InputEvent.MOUSE_CLICK_LEFT;
+            if (doubleClick) {
+                if (button === "left") {
+                    action = InputEvent.MOUSE_CLICK_DOUBLE_LEFT;
+                }
+                else if (button === "right") {
+                    action = InputEvent.MOUSE_CLICK_DOUBLE_RIGHT;
+                }
+                else if (button === "middle") {
+                    action = InputEvent.MOUSE_CLICK_DOUBLE_MIDDLE;
+                }
+            }
+            else {
+                if (button === "right") {
+                    action = InputEvent.MOUSE_CLICK_RIGHT;
+                }
+                else if (button === "middle") {
+                    action = InputEvent.MOUSE_CLICK_MIDDLE;
+                }
+            }
+            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(action, { x: 0, y: 0 }, '', {})]);
+            yield this.requestControl(controlCommand);
+        });
+    }
+    mouseScroll(dx, dy) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.MOUSE_SCROLL, { x: dx, y: dy }, '', {})]);
+            yield this.requestControl(controlCommand);
+        });
+    }
+    mouseHoldLeftButtonDown() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.MOUSE_DOWN, { x: 0, y: 0 }, '', {})]);
+            yield this.requestControl(controlCommand);
+        });
+    }
+    mouseReleaseLeftButton() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.MOUSE_UP, { x: 0, y: 0 }, '', {})]);
+            yield this.requestControl(controlCommand);
+        });
+    }
+    desktopKeyPressAndRelease(key_1) {
+        return __awaiter(this, arguments, void 0, function* (key, modifiers = []) {
+            let keyString = key;
+            if (modifiers.length > 0) {
+                keyString = `${modifiers.join('+')}+${key}`;
+            }
+            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.PRESS_KEY_SEQUENCE, { x: 0, y: 0 }, keyString, {})]);
+            yield this.requestControl(controlCommand);
+        });
+    }
+    desktopKeyHoldDown(key_1) {
+        return __awaiter(this, arguments, void 0, function* (key, modifiers = []) {
+            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.KEY_PRESS, { x: 0, y: 0 }, '', {
+                    key: key,
+                    modifiers: modifiers,
+                })]);
+            yield this.requestControl(controlCommand);
+        });
+    }
+    desktopKeyRelease(key_1) {
+        return __awaiter(this, arguments, void 0, function* (key, modifiers = []) {
+            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.KEY_RELEASE, { x: 0, y: 0 }, '', {
+                    key: key,
+                    modifiers: modifiers,
+                })]);
+            yield this.requestControl(controlCommand);
+        });
+    }
+    typeText(text) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.TYPE, { x: 0, y: 0 }, text, {})]);
+            yield this.requestControl(controlCommand);
+        });
+    }
 }
 export class ScreenShotTool extends BaseAgentTool {
     constructor(osAgentHandler) {
@@ -91,7 +220,7 @@ export class ScreenShotTool extends BaseAgentTool {
     toParams() {
         return {
             name: 'screenshot_tool',
-            description: 'Takes a screenshot of the current screen and returns it as a base64 image',
+            description: 'Takes a screenshot of the current screen and returns it as a base64 image.',
             input_schema: { type: 'object', properties: {}, required: [] },
         };
     }
@@ -103,8 +232,7 @@ export class MouseMoveTool extends BaseAgentTool {
     }
     execute(command) {
         return __awaiter(this, void 0, void 0, function* () {
-            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.MOUSE_MOVE, { x: command.x, y: command.y }, '', {})]);
-            yield this.osAgentHandler.requestControl(controlCommand);
+            yield this.osAgentHandler.mouseMove(command.x, command.y);
             return {
                 output: `Moved mouse to (${command.x}, ${command.y})`,
             };
@@ -119,11 +247,11 @@ export class MouseMoveTool extends BaseAgentTool {
                 properties: {
                     x: {
                         type: 'number',
-                        description: 'The x coordinate of the element to click on',
+                        description: 'The x (pixels from the left edge) coordinate to move the mouse to',
                     },
                     y: {
                         type: 'number',
-                        description: 'The y coordinate of the element to click on',
+                        description: 'The y (pixels from the top edge) coordinate to move the mouse to',
                     },
                 },
             },
@@ -137,33 +265,7 @@ export class MouseClickTool extends BaseAgentTool {
     }
     execute(command) {
         return __awaiter(this, void 0, void 0, function* () {
-            let controlCommand;
-            if (command.doubleClick) {
-                if (command.button === 'left') {
-                    controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.MOUSE_CLICK_DOUBLE_LEFT, { x: 0, y: 0 }, '', {})]);
-                }
-                if (command.button === 'right') {
-                    controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.MOUSE_CLICK_DOUBLE_RIGHT, { x: 0, y: 0 }, '', {})]);
-                }
-                if (command.button === 'middle') {
-                    controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.MOUSE_CLICK_DOUBLE_MIDDLE, { x: 0, y: 0 }, '', {})]);
-                }
-            }
-            else {
-                if (command.button === 'left') {
-                    controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.MOUSE_CLICK_LEFT, { x: 0, y: 0 }, '', {})]);
-                }
-                if (command.button === 'right') {
-                    controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.MOUSE_CLICK_RIGHT, { x: 0, y: 0 }, '', {})]);
-                }
-                if (command.button === 'middle') {
-                    controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.MOUSE_CLICK_MIDDLE, { x: 0, y: 0 }, '', {})]);
-                }
-            }
-            if (!controlCommand) {
-                throw new ToolError('Invalid input parameter for mouse click tool');
-            }
-            yield this.osAgentHandler.requestControl(controlCommand);
+            yield this.osAgentHandler.mouseClick(command.button, command.doubleClick);
             const returnedMessage = command.doubleClick ? `Double clicked ${command.button} button` : `Clicked ${command.button} button`;
             return {
                 output: returnedMessage,
@@ -199,8 +301,7 @@ export class MouseScrollTool extends BaseAgentTool {
     }
     execute(command) {
         return __awaiter(this, void 0, void 0, function* () {
-            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.MOUSE_SCROLL, { x: command.dx, y: command.dy }, '', {})]);
-            yield this.osAgentHandler.requestControl(controlCommand);
+            yield this.osAgentHandler.mouseScroll(command.dx, command.dy);
             return {
                 output: `Scrolled by (${command.dx}, ${command.dy})`,
             };
@@ -215,11 +316,11 @@ export class MouseScrollTool extends BaseAgentTool {
                 properties: {
                     dx: {
                         type: 'number',
-                        description: 'The amount to scroll horizontally',
+                        description: 'The amount to scroll horizontally (positive is right, negative is left)',
                     },
                     dy: {
                         type: 'number',
-                        description: 'The amount to scroll vertically',
+                        description: 'The amount to scroll vertically (positive is down, negative is up)',
                     },
                 },
                 required: ['dx', 'dy'],
@@ -227,20 +328,104 @@ export class MouseScrollTool extends BaseAgentTool {
         };
     }
 }
-export class DesktopKeyPressSequenceTool extends BaseAgentTool {
+export class MouseDragAndDropTool extends BaseAgentTool {
     constructor(osAgentHandler) {
         super();
         this.osAgentHandler = osAgentHandler;
     }
     execute(command) {
         return __awaiter(this, void 0, void 0, function* () {
-            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.PRESS_KEY_SEQUENCE, { x: 0, y: 0 }, command.key, {
-                    firstModifier: command.firstModifier || '',
-                    secondModifier: command.secondModifier || '',
-                })]);
-            yield this.osAgentHandler.requestControl(controlCommand);
+            yield this.osAgentHandler.mouseMove(command.startX, command.startY);
+            yield this.osAgentHandler.mouseHoldLeftButtonDown();
+            yield this.osAgentHandler.mouseMove(command.endX, command.endY);
+            yield this.osAgentHandler.mouseReleaseLeftButton();
             return {
-                output: `Pressed key ${command.key} with modifiers ${command.firstModifier || ''} ${command.secondModifier || ''}`,
+                output: `Dragged from (${command.startX}, ${command.startY}) to (${command.endX}, ${command.endY})`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'mouse_drag_and_drop_tool',
+            description: 'Drags the mouse from the specified start coordinates to the specified end coordinates. The top left corner of the screen is (0,0)',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    startX: {
+                        type: 'number',
+                        description: 'The x (pixels from the left edge) coordinate of the start position',
+                    },
+                    startY: {
+                        type: 'number',
+                        description: 'The y (pixels from the top edge) coordinate of the start position',
+                    },
+                    endX: {
+                        type: 'number',
+                        description: 'The x (pixels from the left edge) coordinate of the end position',
+                    },
+                    endY: {
+                        type: 'number',
+                        description: 'The y (pixels from the top edge) coordinate of the end position',
+                    },
+                },
+                required: ['startX', 'startY', 'endX', 'endY'],
+            },
+        };
+    }
+}
+export class MouseHoldLeftButtonDownTool extends BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute() {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.osAgentHandler.mouseHoldLeftButtonDown();
+            return {
+                output: 'Holding down left mouse button',
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'mouse_hold_left_button_down_tool',
+            description: 'Hold down the left mouse button at the current position.',
+            input_schema: { type: 'object', properties: {}, required: [] },
+        };
+    }
+}
+export class MouseReleaseLeftButtonTool extends BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute() {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.osAgentHandler.mouseReleaseLeftButton();
+            return {
+                output: 'Released left mouse button',
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'mouse_release_left_button_tool',
+            description: 'Release the left mouse button at the current position.',
+            input_schema: { type: 'object', properties: {}, required: [] },
+        };
+    }
+}
+export class DesktopPressAndReleaseKeysTool extends BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const modifiers = command.modifiers || [];
+            yield this.osAgentHandler.desktopKeyPressAndRelease(command.key, modifiers);
+            return {
+                output: `Pressed key ${command.key} with modifiers ${modifiers.join(' ')}`,
             };
         });
     }
@@ -256,15 +441,53 @@ export class DesktopKeyPressSequenceTool extends BaseAgentTool {
                         enum: PC_KEY_VALUES,
                         description: 'The key to press',
                     },
-                    firstModifier: {
-                        type: 'string',
-                        enum: MODIFIER_KEY_VALUES,
-                        description: 'The first modifier key',
+                    modifiers: {
+                        type: 'array',
+                        items: {
+                            type: 'string',
+                            enum: MODIFIER_KEY_VALUES,
+                        },
+                        description: 'The modifiers to press',
                     },
-                    secondModifier: {
+                },
+                required: ['key'],
+            },
+        };
+    }
+}
+export class DesktopKeyHoldDownTool extends BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const modifiers = command.modifiers || [];
+            yield this.osAgentHandler.desktopKeyHoldDown(command.key, modifiers);
+            return {
+                output: `Holding down key ${command.key} with modifiers ${modifiers.join(' ')}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'desktop_key_hold_down_tool',
+            description: 'Hold down a key and optional modifiers. Keys will be still pressed after the tool is finished.',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    key: {
                         type: 'string',
-                        enum: MODIFIER_KEY_VALUES,
-                        description: 'The second modifier key',
+                        enum: [...PC_KEY_VALUES, ...MODIFIER_KEY_VALUES],
+                        description: 'The key to hold down',
+                    },
+                    modifiers: {
+                        type: 'array',
+                        items: {
+                            type: 'string',
+                            enum: MODIFIER_KEY_VALUES,
+                        },
+                        description: 'The modifiers to hold down',
                     },
                 },
                 required: ['key'],
@@ -272,31 +495,39 @@ export class DesktopKeyPressSequenceTool extends BaseAgentTool {
         };
     }
 }
-export class DesktopSingleKeyPressTool extends BaseAgentTool {
+export class DesktopKeyReleaseTool extends BaseAgentTool {
     constructor(osAgentHandler) {
         super();
         this.osAgentHandler = osAgentHandler;
     }
     execute(command) {
         return __awaiter(this, void 0, void 0, function* () {
-            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.PRESS_KEY_SEQUENCE, { x: 0, y: 0 }, command.key, {})]);
-            yield this.osAgentHandler.requestControl(controlCommand);
+            const modifiers = command.modifiers || [];
+            yield this.osAgentHandler.desktopKeyRelease(command.key, modifiers);
             return {
-                output: `Pressed key ${command.key}`,
+                output: `Released key ${command.key} with modifiers ${modifiers.join(' ')}`,
             };
         });
     }
     toParams() {
         return {
-            name: 'desktop_single_key_press_tool',
-            description: 'Presses a single key',
+            name: 'desktop_key_release_tool',
+            description: 'Releases a key and optional modifiers. This can be used after keys were held down with the desktop_key_hold_down_tool',
             input_schema: {
                 type: 'object',
                 properties: {
                     key: {
                         type: 'string',
                         enum: [...PC_KEY_VALUES, ...MODIFIER_KEY_VALUES],
-                        description: 'The key to press',
+                        description: 'The key to release',
+                    },
+                    modifiers: {
+                        type: 'array',
+                        items: {
+                            type: 'string',
+                            enum: MODIFIER_KEY_VALUES,
+                        },
+                        description: 'The modifiers to release',
                     },
                 },
                 required: ['key'],
@@ -459,3 +690,32 @@ export class ExecuteShellCommandTool extends BaseAgentTool {
         };
     }
 }
+export class WaitTool extends BaseAgentTool {
+    constructor() {
+        super();
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield new Promise(resolve => setTimeout(resolve, command.milliseconds));
+            return {
+                output: `Waited for ${command.milliseconds} milliseconds`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'wait_tool',
+            description: 'Waits for a specified number of milliseconds',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    milliseconds: {
+                        type: 'number',
+                        description: 'The number of milliseconds to wait',
+                    },
+                },
+                required: ['milliseconds'],
+            },
+        };
+    }
+}

package/dist/esm/core/ui-control-commands/input-event.d.ts

@@ -16,5 +16,7 @@ export declare enum InputEvent {
     MOUSE_MOVE = "MOUSE_MOVE",
     MOUSE_DOWN = "MOUSE_DOWN",
     MOUSE_UP = "MOUSE_UP",
-    EXECUTE_COMMAND = "EXECUTE_COMMAND"
+    EXECUTE_COMMAND = "EXECUTE_COMMAND",
+    KEY_PRESS = "KEY_PRESS",
+    KEY_RELEASE = "KEY_RELEASE"
 }

package/dist/esm/core/ui-control-commands/input-event.js

@@ -18,4 +18,6 @@ export var InputEvent;
     InputEvent["MOUSE_DOWN"] = "MOUSE_DOWN";
     InputEvent["MOUSE_UP"] = "MOUSE_UP";
     InputEvent["EXECUTE_COMMAND"] = "EXECUTE_COMMAND";
+    InputEvent["KEY_PRESS"] = "KEY_PRESS";
+    InputEvent["KEY_RELEASE"] = "KEY_RELEASE";
 })(InputEvent || (InputEvent = {}));

package/dist/esm/execution/execution-runtime.d.ts

@@ -47,5 +47,9 @@ export declare class ExecutionRuntime {
         system?: string;
         tools?: object[];
         betas?: string[];
+        tool_choice?: {
+            type: 'tool' | 'any' | 'auto';
+            name?: string;
+        };
     }): Promise<BetaMessage>;
 }

package/dist/esm/execution/inference-client.d.ts

@@ -26,6 +26,10 @@ export declare class InferenceClient {
     predictVQAAnswer(prompt: string, image: string, config?: object): Promise<any>;
     predictActResponse(params: {
         max_tokens: number;
+        tool_choice?: {
+            type: 'tool' | 'any' | 'auto';
+            name?: string;
+        };
         messages: BetaMessageParam[];
         model: string;
         system?: string;