askui 0.24.0 → 0.25.0

package/dist/esm/core/models/anthropic/tools/os-agent-tools.js

@@ -0,0 +1,461 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { PC_KEY_VALUES, MODIFIER_KEY_VALUES, ANDROID_KEY_VALUES, } from '../../../../execution/dsl';
+import { BaseAgentTool, ToolError } from './base';
+import { ControlCommand, ControlCommandCode, InputEvent, Action, } from '../../../ui-control-commands';
+import { Base64Image } from '../../../../utils/base_64_image/base-64-image';
+import { AgentError } from './agent-errors';
+export class OsAgentHandler {
+    constructor(AgentOsClient, screenDimensions) {
+        this.AgentOsClient = AgentOsClient;
+        this.TargetResolution = { width: 1280, height: 800 };
+        this.screenDimensions = screenDimensions;
+    }
+    static createInstance(AgentOsClient) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const base64ImageString = yield AgentOsClient.getScreenshot();
+            const image_info = yield (yield Base64Image.fromString(base64ImageString)).getInfo();
+            return new OsAgentHandler(AgentOsClient, {
+                width: image_info.width,
+                height: image_info.height,
+            });
+        });
+    }
+    getTargetResolution() {
+        return this.TargetResolution;
+    }
+    setTargetResolution(width, height) {
+        this.TargetResolution = { width, height };
+    }
+    takeScreenshot() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const base64ImageString = yield this.AgentOsClient.getScreenshot();
+            const base64Image = yield Base64Image.fromString(base64ImageString);
+            const image_info = yield base64Image.getInfo();
+            this.screenDimensions = {
+                width: image_info.width,
+                height: image_info.height,
+            };
+            const resized_image = yield base64Image.resizeWithSameAspectRatio(this.TargetResolution.width, this.TargetResolution.height);
+            return resized_image.toString(false);
+        });
+    }
+    scaleCoordinates(source, x, y) {
+        const xScalingFactor = this.TargetResolution.width / this.screenDimensions.width;
+        const yScalingFactor = this.TargetResolution.height / this.screenDimensions.height;
+        if (source === 'api') {
+            if (x > this.TargetResolution.width || y > this.TargetResolution.height || x < 0 || y < 0) {
+                throw new ToolError(`Coordinates ${x}, ${y} are outside screen bounds `
+                    + `(${this.TargetResolution.width}x${this.TargetResolution.height})`);
+            }
+            return [
+                Math.round(x / xScalingFactor),
+                Math.round(y / yScalingFactor),
+            ];
+        }
+        return [
+            Math.round(x * xScalingFactor),
+            Math.round(y * yScalingFactor),
+        ];
+    }
+    requestControl(controlCommand) {
+        return __awaiter(this, void 0, void 0, function* () {
+            for (const action of controlCommand.actions) {
+                [action.position.x, action.position.y] = this.scaleCoordinates('api', action.position.x, action.position.y);
+            }
+            yield this.AgentOsClient.requestControl(controlCommand);
+        });
+    }
+}
+export class ScreenShotTool extends BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const screenshot = yield this.osAgentHandler.takeScreenshot();
+            return {
+                base64Images: [screenshot],
+                output: `Screenshot was taken, with resolution width ${this.osAgentHandler.getTargetResolution().width} and height ${this.osAgentHandler.getTargetResolution().height}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'screenshot_tool',
+            description: 'Takes a screenshot of the current screen and returns it as a base64 image',
+            input_schema: { type: 'object', properties: {}, required: [] },
+        };
+    }
+}
+export class MouseMoveTool extends BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.MOUSE_MOVE, { x: command.x, y: command.y }, '', {})]);
+            yield this.osAgentHandler.requestControl(controlCommand);
+            return {
+                output: `Moved mouse to (${command.x}, ${command.y})`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'mouse_move_tool',
+            description: 'Moves the mouse to the specified absolute coordinates. The top left corner of the screen is (0,0)',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    x: {
+                        type: 'number',
+                        description: 'The x coordinate of the element to click on',
+                    },
+                    y: {
+                        type: 'number',
+                        description: 'The y coordinate of the element to click on',
+                    },
+                },
+            },
+        };
+    }
+}
+export class MouseClickTool extends BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let controlCommand;
+            if (command.doubleClick) {
+                if (command.button === 'left') {
+                    controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.MOUSE_CLICK_DOUBLE_LEFT, { x: 0, y: 0 }, '', {})]);
+                }
+                if (command.button === 'right') {
+                    controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.MOUSE_CLICK_DOUBLE_RIGHT, { x: 0, y: 0 }, '', {})]);
+                }
+                if (command.button === 'middle') {
+                    controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.MOUSE_CLICK_DOUBLE_MIDDLE, { x: 0, y: 0 }, '', {})]);
+                }
+            }
+            else {
+                if (command.button === 'left') {
+                    controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.MOUSE_CLICK_LEFT, { x: 0, y: 0 }, '', {})]);
+                }
+                if (command.button === 'right') {
+                    controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.MOUSE_CLICK_RIGHT, { x: 0, y: 0 }, '', {})]);
+                }
+                if (command.button === 'middle') {
+                    controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.MOUSE_CLICK_MIDDLE, { x: 0, y: 0 }, '', {})]);
+                }
+            }
+            if (!controlCommand) {
+                throw new ToolError('Invalid input parameter for mouse click tool');
+            }
+            yield this.osAgentHandler.requestControl(controlCommand);
+            const returnedMessage = command.doubleClick ? `Double clicked ${command.button} button` : `Clicked ${command.button} button`;
+            return {
+                output: returnedMessage,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'mouse_click_tool',
+            description: 'Clicks the specified button on the mouse',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    button: {
+                        type: 'string',
+                        enum: ['left', 'right', 'middle'],
+                        description: 'The button to click',
+                    },
+                    doubleClick: {
+                        type: 'boolean',
+                        description: 'Whether to double click the button',
+                    },
+                },
+                required: ['button', 'doubleClick'],
+            },
+        };
+    }
+}
+export class MouseScrollTool extends BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.MOUSE_SCROLL, { x: command.dx, y: command.dy }, '', {})]);
+            yield this.osAgentHandler.requestControl(controlCommand);
+            return {
+                output: `Scrolled by (${command.dx}, ${command.dy})`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'mouse_scroll_tool',
+            description: 'Scrolls the mouse by the specified amount',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    dx: {
+                        type: 'number',
+                        description: 'The amount to scroll horizontally',
+                    },
+                    dy: {
+                        type: 'number',
+                        description: 'The amount to scroll vertically',
+                    },
+                },
+                required: ['dx', 'dy'],
+            },
+        };
+    }
+}
+export class DesktopKeyPressSequenceTool extends BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.PRESS_KEY_SEQUENCE, { x: 0, y: 0 }, command.key, {
+                    firstModifier: command.firstModifier || '',
+                    secondModifier: command.secondModifier || '',
+                })]);
+            yield this.osAgentHandler.requestControl(controlCommand);
+            return {
+                output: `Pressed key ${command.key} with modifiers ${command.firstModifier || ''} ${command.secondModifier || ''}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'desktop_key_press_sequence_tool',
+            description: 'Presses a key with optional modifiers',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    key: {
+                        type: 'string',
+                        enum: PC_KEY_VALUES,
+                        description: 'The key to press',
+                    },
+                    firstModifier: {
+                        type: 'string',
+                        enum: MODIFIER_KEY_VALUES,
+                        description: 'The first modifier key',
+                    },
+                    secondModifier: {
+                        type: 'string',
+                        enum: MODIFIER_KEY_VALUES,
+                        description: 'The second modifier key',
+                    },
+                },
+                required: ['key'],
+            },
+        };
+    }
+}
+export class DesktopSingleKeyPressTool extends BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.PRESS_KEY_SEQUENCE, { x: 0, y: 0 }, command.key, {})]);
+            yield this.osAgentHandler.requestControl(controlCommand);
+            return {
+                output: `Pressed key ${command.key}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'desktop_single_key_press_tool',
+            description: 'Presses a single key',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    key: {
+                        type: 'string',
+                        enum: [...PC_KEY_VALUES, ...MODIFIER_KEY_VALUES],
+                        description: 'The key to press',
+                    },
+                },
+                required: ['key'],
+            },
+        };
+    }
+}
+export class TypeTool extends BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.TYPE, { x: 0, y: 0 }, command.text, {})]);
+            yield this.osAgentHandler.requestControl(controlCommand);
+            return {
+                output: `Typed text: ${command.text}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'type_tool',
+            description: 'Types the specified text',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    text: {
+                        type: 'string',
+                        description: 'The text to type',
+                    },
+                },
+                required: ['text'],
+            },
+        };
+    }
+}
+export class AndroidSingleKeyPressTool extends BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.PRESS_KEY_SEQUENCE, { x: 0, y: 0 }, command.key, {})]);
+            yield this.osAgentHandler.requestControl(controlCommand);
+            return {
+                output: `Pressed Android key ${command.key}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'android_single_key_press_tool',
+            description: 'Presses a single Android key',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    key: {
+                        type: 'string',
+                        enum: ANDROID_KEY_VALUES,
+                        description: 'The Android key to press',
+                    },
+                },
+                required: ['key'],
+            },
+        };
+    }
+}
+export class AndroidSequenceKeyPressTool extends BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.PRESS_KEY_SEQUENCE, { x: 0, y: 0 }, command.keys.join(' '), {})]);
+            yield this.osAgentHandler.requestControl(controlCommand);
+            return {
+                output: `Pressed Android keys: ${command.keys.join(', ')}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'android_sequence_key_press_tool',
+            description: 'Presses a sequence of Android keys',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    keys: {
+                        type: 'array',
+                        items: {
+                            type: 'string',
+                            enum: ANDROID_KEY_VALUES,
+                        },
+                        description: 'The sequence of Android keys to press',
+                    },
+                },
+                required: ['keys'],
+            },
+        };
+    }
+}
+export class AgentErrorTool extends BaseAgentTool {
+    constructor() {
+        super();
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            throw new AgentError(command.error);
+        });
+    }
+    toParams() {
+        return {
+            name: 'agent_error_tool',
+            description: 'Raises an error in the agent',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    error: {
+                        type: 'string',
+                        description: 'The error message to raise',
+                    },
+                },
+                required: ['error'],
+            },
+        };
+    }
+}
+export class ExecuteShellCommandTool extends BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.EXECUTE_COMMAND, { x: 0, y: 0 }, command.command, {})]);
+            yield this.osAgentHandler.requestControl(controlCommand);
+            return {
+                output: `Executed shell command: ${command.command}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'execute_shell_command_tool',
+            description: 'Executes a shell command',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    command: {
+                        type: 'string',
+                        description: 'The shell command to execute',
+                    },
+                },
+                required: ['command'],
+            },
+        };
+    }
+}

package/dist/esm/core/ui-control-commands/index.d.ts

@@ -1,3 +1,5 @@
 export { ControlCommand } from './control-command';
 export { ControlCommandCode } from './control-command-code';
 export { InferenceResponse } from '../inference-response/inference-response';
+export { Action } from './action';
+export { InputEvent } from './input-event';

package/dist/esm/core/ui-control-commands/index.js

@@ -1,3 +1,5 @@
 export { ControlCommand } from './control-command';
 export { ControlCommandCode } from './control-command-code';
 export { InferenceResponse } from '../inference-response/inference-response';
+export { Action } from './action';
+export { InputEvent } from './input-event';

package/dist/esm/execution/dsl.d.ts

@@ -5,11 +5,14 @@ export declare enum Separators {
     STRING = "<|string|>"
 }
 export type INTERSECTION_AREA = 'element_center_line' | 'element_edge_area' | 'display_edge_area';
-export type PC_KEY = 'backspace' | 'delete' | 'enter' | 'tab' | 'escape' | 'up' | 'down' | 'right' | 'left' | 'home' | 'end' | 'pageup' | 'pagedown' | 'f1' | 'f2' | 'f3' | 'f4' | 'f5' | 'f6' | 'f7' | 'f8' | 'f9' | 'f10' | 'f11' | 'f12' | 'space' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '!' | '"' | '#' | '$' | '%' | '&' | "'" | '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | ':' | ';' | '<' | '=' | '>' | '?' | '@' | '[' | '\\' | ']' | '^' | '_' | '`' | '{' | '|' | '}' | '~ ';
-export type ANDROID_KEY = 'home' | 'back' | 'call' | 'endcall' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | 'star' | 'pound' | 'dpad_up' | 'dpad_down' | 'dpad_left' | 'dpad_right' | 'dpad_center' | 'volume_up' | 'volume_down' | 'power' | 'camera' | 'clear' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | 'comma' | 'period' | 'alt_left' | 'alt_right' | 'shift_left' | 'shift_right' | 'tab' | 'space' | 'sym' | 'explorer' | 'envelope' | 'enter' | 'del' | 'grave' | 'minus' | 'equals' | 'left_bracket' | 'right_bracket' | 'backslash' | 'semicolon' | 'apostrophe' | 'slash' | 'at' | 'num' | 'headsethook' | 'focus' | 'plus' | 'menu' | 'notification' | 'search' | 'media_play_pause' | 'media_stop' | 'media_next' | 'media_previous' | 'media_rewind' | 'media_fast_forward' | 'mute' | 'page_up' | 'page_down' | 'switch_charset' | 'escape' | 'forward_del' | 'ctrl_left' | 'ctrl_right' | 'caps_lock' | 'scroll_lock' | 'function' | 'break' | 'move_home' | 'move_end' | 'insert' | 'forward' | 'media_play' | 'media_pause' | 'media_close' | 'media_eject' | 'media_record' | 'f1' | 'f2' | 'f3' | 'f4' | 'f5' | 'f6' | 'f7' | 'f8' | 'f9' | 'f10' | 'f11' | 'f12' | 'num_lock' | 'numpad_0' | 'numpad_1' | 'numpad_2' | 'numpad_3' | 'numpad_4' | 'numpad_5' | 'numpad_6' | 'numpad_7' | 'numpad_8' | 'numpad_9' | 'numpad_divide' | 'numpad_multiply' | 'numpad_subtract' | 'numpad_add' | 'numpad_dot' | 'numpad_comma' | 'numpad_enter' | 'numpad_equals' | 'numpad_left_paren' | 'numpad_right_paren' | 'volume_mute' | 'info' | 'channel_up' | 'channel_down' | 'zoom_in' | 'zoom_out' | 'window' | 'guide' | 'bookmark' | 'captions' | 'settings' | 'app_switch' | 'language_switch' | 'contacts' | 'calendar' | 'music' | 'calculator' | 'assist' | 'brightness_down' | 'brightness_up' | 'media_audio_track' | 'sleep' | 'wakeup' | 'pairing' | 'media_top_menu' | 'last_channel' | 'tv_data_service' | 'voice_assist' | 'help' | 'navigate_previous' | 'navigate_next' | 'navigate_in' | 'navigate_out' | 'dpad_up_left' | 'dpad_down_left' | 'dpad_up_right' | 'dpad_down_right' | 'media_skip_forward' | 'media_skip_backward' | 'media_step_forward' | 'media_step_backward' | 'soft_sleep' | 'cut' | 'copy' | 'paste' | 'all_apps' | 'refresh';
-export type MODIFIER_KEY = 'command' | 'alt' | 'control' | 'shift' | 'right_shift';
+export declare const PC_KEY_VALUES: readonly ["backspace", "delete", "enter", "tab", "escape", "up", "down", "right", "left", "home", "end", "pageup", "pagedown", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "f9", "f10", "f11", "f12", "space", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "!", "\"", "#", "$", "%", "&", "'", "(", ")", "*", "+", ",", "-", ".", "/", ":", ";", "<", "=", ">", "?", "@", "[", "\\", "]", "^", "_", "`", "{", "|", "}", "~ "];
+export type PC_KEY = typeof PC_KEY_VALUES[number];
+export declare const ANDROID_KEY_VALUES: readonly ["backspace", "delete", "enter", "tab", "escape", "up", "down", "right", "left", "home", "end", "pageup", "pagedown", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "f9", "f10", "f11", "f12", "space", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "!", "\"", "#", "$", "%", "&", "'", "(", ")", "*", "+", ",", "-", ".", "/", ":", ";", "<", "=", ">", "?", "@", "[", "\\", "]", "^", "_", "`", "{", "|", "}", "~ "];
+export type ANDROID_KEY = typeof ANDROID_KEY_VALUES[number];
+export declare const MODIFIER_KEY_VALUES: readonly ["command", "alt", "control", "shift", "right_shift"];
+export type MODIFIER_KEY = typeof MODIFIER_KEY_VALUES[number];
+export type PC_AND_MODIFIER_KEY = PC_KEY | MODIFIER_KEY;
 export type COLOR = 'black' | 'white' | 'red' | 'green' | 'yellow green' | 'orange' | 'yellow' | 'purple' | 'pink' | 'gray' | 'lime green' | 'royal blue';
-export type PC_AND_MODIFIER_KEY = 'command' | 'alt' | 'control' | 'shift' | 'right_shift' | 'backspace' | 'delete' | 'enter' | 'tab' | 'escape' | 'up' | 'down' | 'right' | 'left' | 'home' | 'end' | 'pageup' | 'pagedown' | 'f1' | 'f2' | 'f3' | 'f4' | 'f5' | 'f6' | 'f7' | 'f8' | 'f9' | 'f10' | 'f11' | 'f12' | 'space' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '!' | '"' | '#' | '$' | '%' | '&' | "'" | '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | ':' | ';' | '<' | '=' | '>' | '?' | '@' | '[' | '\\' | ']' | '^' | '_' | '`' | '{' | '|' | '}' | '~ ';
 export interface CommandExecutorContext {
     customElementsJson: CustomElementJson[];
     aiElementNames: string[];

package/dist/esm/execution/dsl.js

@@ -34,6 +34,9 @@ export var Separators;
 (function (Separators) {
     Separators["STRING"] = "<|string|>";
 })(Separators || (Separators = {}));
+export const PC_KEY_VALUES = ['backspace', 'delete', 'enter', 'tab', 'escape', 'up', 'down', 'right', 'left', 'home', 'end', 'pageup', 'pagedown', 'f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'f10', 'f11', 'f12', 'space', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', '{', '|', '}', '~ '];
+export const ANDROID_KEY_VALUES = ['backspace', 'delete', 'enter', 'tab', 'escape', 'up', 'down', 'right', 'left', 'home', 'end', 'pageup', 'pagedown', 'f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'f10', 'f11', 'f12', 'space', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', '{', '|', '}', '~ '];
+export const MODIFIER_KEY_VALUES = ['command', 'alt', 'control', 'shift', 'right_shift'];
 class FluentBase {
     constructor(prev) {
         this.prev = prev;

package/dist/esm/execution/execution-runtime.d.ts

@@ -1,3 +1,5 @@
+import { BetaMessage, BetaMessageParam } from '@anthropic-ai/sdk/resources/beta/messages';
+import { ControlCommand } from '../core/ui-control-commands';
 import { UiControllerClient } from './ui-controller-client';
 import { InferenceClient } from './inference-client';
 import { Annotation } from '../core/annotation/annotation';
@@ -18,7 +20,7 @@ export declare class ExecutionRuntime {
     startVideoRecording(): Promise<void>;
     stopVideoRecording(): Promise<void>;
     readVideoRecording(): Promise<string>;
-    private requestControl;
+    requestControl(controlCommand: ControlCommand): Promise<void>;
     executeInstruction(instruction: Instruction, modelComposition: ModelCompositionBranch[]): Promise<void>;
     private readonly EXEC_REPETITION_COUNT;
     private executeCommandRepeatedly;
@@ -38,4 +40,12 @@ export declare class ExecutionRuntime {
     getDetectedElements(instruction: string, customElementJson?: CustomElementJson[]): Promise<DetectedElement[]>;
     annotateImage(imagePath?: string, customElementJson?: CustomElementJson[], elements?: DetectedElement[]): Promise<Annotation>;
     predictVQA(prompt: string, config?: object): Promise<any>;
+    predictActResponse(params: {
+        max_tokens: number;
+        messages: BetaMessageParam[];
+        model: string;
+        system?: string;
+        tools?: object[];
+        betas?: string[];
+    }): Promise<BetaMessage>;
 }

package/dist/esm/execution/execution-runtime.js

@@ -212,4 +212,9 @@ export class ExecutionRuntime {
             return this.inferenceClient.predictVQAAnswer(prompt, base64Image, config);
         });
     }
+    predictActResponse(params) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.inferenceClient.predictActResponse(params);
+        });
+    }
 }

package/dist/esm/execution/inference-client.d.ts

@@ -1,3 +1,4 @@
+import { BetaMessage, BetaMessageParam } from '@anthropic-ai/sdk/resources/beta/messages';
 import { HttpClientGot } from '../utils/http/http-client-got';
 import { ControlCommand } from '../core/ui-control-commands';
 import { CustomElement } from '../core/model/custom-element';
@@ -23,4 +24,12 @@ export declare class InferenceClient {
     getDetectedElements(instruction: string, image: string, customElements?: CustomElement[]): Promise<DetectedElement[]>;
     predictImageAnnotation(image: string, customElements?: CustomElement[]): Promise<Annotation>;
     predictVQAAnswer(prompt: string, image: string, config?: object): Promise<any>;
+    predictActResponse(params: {
+        max_tokens: number;
+        messages: BetaMessageParam[];
+        model: string;
+        system?: string;
+        tools?: object[];
+        betas?: string[];
+    }): Promise<BetaMessage>;
 }

package/dist/esm/execution/inference-client.js

@@ -27,6 +27,7 @@ export class InferenceClient {
             ? urljoin(versionedBaseUrl, 'workspaces', workspaceId)
             : versionedBaseUrl;
         this.urls = {
+            actEndpoint: urljoin(url, 'act', 'inference'),
             inference: urljoin(url, 'inference'),
             isImageRequired: urljoin(url, 'instruction', 'is-image-required'),
             vqaInference: urljoin(url, 'vqa', 'inference'),
@@ -68,7 +69,7 @@ export class InferenceClient {
                     instruction,
                     modelComposition: modelComposition.length > 0 ? modelComposition : this.modelComposition,
                 });
-            InferenceClient.logMetaInformation(response);
+            InferenceClient.logMetaInformation(response.headers);
             return InferenceResponse.fromJson(response.body, resizedImage.resizeRatio, image);
         });
     }
@@ -79,13 +80,13 @@ export class InferenceClient {
                 image,
                 prompt,
             });
-            InferenceClient.logMetaInformation(response);
+            InferenceClient.logMetaInformation(response.headers);
             return response.body;
         });
     }
-    static logMetaInformation(response) {
-        if (response.headers['askui-usage-warnings'] !== undefined) {
-            logger.warn(response.headers['askui-usage-warnings']);
+    static logMetaInformation(headers) {
+        if (headers['askui-usage-warnings'] !== undefined) {
+            logger.warn(headers['askui-usage-warnings']);
         }
     }
     predictControlCommand(instruction_1, modelComposition_1) {
@@ -128,4 +129,11 @@ export class InferenceClient {
             return response;
         });
     }
+    predictActResponse(params) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const response = yield this.httpClient.post(this.urls.actEndpoint, params);
+            InferenceClient.logMetaInformation(response.headers);
+            return response.body;
+        });
+    }
 }

package/dist/esm/execution/ui-control-client-dependency-builder.js

@@ -59,7 +59,7 @@ export class UiControlClientDependencyBuilder {
                     onLocationNotExist: (_d = (_c = clientArgs.aiElementArgs) === null || _c === void 0 ? void 0 : _c.onLocationNotExist) !== null && _d !== void 0 ? _d : 'error',
                 }, context: {
                     isCi: (_f = (_e = clientArgs.context) === null || _e === void 0 ? void 0 : _e.isCi) !== null && _f !== void 0 ? _f : isCI,
-                }, credentials: readCredentials(clientArgs), inferenceServerApiVersion: (_g = clientArgs.inferenceServerApiVersion) !== null && _g !== void 0 ? _g : 'v3', inferenceServerUrl: (_h = clientArgs.inferenceServerUrl) !== null && _h !== void 0 ? _h : 'https://inference.askui.com', proxyAgents: (_j = clientArgs.proxyAgents) !== null && _j !== void 0 ? _j : (yield envProxyAgents()), uiControllerUrl: (_k = clientArgs.uiControllerUrl) !== null && _k !== void 0 ? _k : 'http://127.0.0.1:6769' });
+                }, credentials: readCredentials(clientArgs), inferenceServerApiVersion: (_g = clientArgs.inferenceServerApiVersion) !== null && _g !== void 0 ? _g : 'v1', inferenceServerUrl: (_h = clientArgs.inferenceServerUrl) !== null && _h !== void 0 ? _h : 'https://inference.askui.com', proxyAgents: (_j = clientArgs.proxyAgents) !== null && _j !== void 0 ? _j : (yield envProxyAgents()), uiControllerUrl: (_k = clientArgs.uiControllerUrl) !== null && _k !== void 0 ? _k : 'http://127.0.0.1:6769' });
         });
     }
 }