askui 0.24.1 → 0.25.1

package/dist/cjs/core/models/anthropic/tools/os-agent-tools.js

@@ -0,0 +1,476 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ExecuteShellCommandTool = exports.AgentErrorTool = exports.AndroidSequenceKeyPressTool = exports.AndroidSingleKeyPressTool = exports.TypeTool = exports.DesktopSingleKeyPressTool = exports.DesktopKeyPressSequenceTool = exports.MouseScrollTool = exports.MouseClickTool = exports.MouseMoveTool = exports.ScreenShotTool = exports.OsAgentHandler = void 0;
+const dsl_1 = require("../../../../execution/dsl");
+const base_1 = require("./base");
+const ui_control_commands_1 = require("../../../ui-control-commands");
+const base_64_image_1 = require("../../../../utils/base_64_image/base-64-image");
+const agent_errors_1 = require("./agent-errors");
+class OsAgentHandler {
+    constructor(AgentOsClient, screenDimensions) {
+        this.AgentOsClient = AgentOsClient;
+        this.TargetResolution = { width: 1280, height: 800 };
+        this.screenDimensions = screenDimensions;
+    }
+    static createInstance(AgentOsClient) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const base64ImageString = yield AgentOsClient.getScreenshot();
+            const image_info = yield (yield base_64_image_1.Base64Image.fromString(base64ImageString)).getInfo();
+            return new OsAgentHandler(AgentOsClient, {
+                width: image_info.width,
+                height: image_info.height,
+            });
+        });
+    }
+    getTargetResolution() {
+        return this.TargetResolution;
+    }
+    setTargetResolution(width, height) {
+        this.TargetResolution = { width, height };
+    }
+    takeScreenshot() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const base64ImageString = yield this.AgentOsClient.getScreenshot();
+            const base64Image = yield base_64_image_1.Base64Image.fromString(base64ImageString);
+            const image_info = yield base64Image.getInfo();
+            this.screenDimensions = {
+                width: image_info.width,
+                height: image_info.height,
+            };
+            const resized_image = yield base64Image.resizeWithSameAspectRatio(this.TargetResolution.width, this.TargetResolution.height);
+            return resized_image.toString(false);
+        });
+    }
+    scaleCoordinates(source, x, y) {
+        const xScalingFactor = this.TargetResolution.width / this.screenDimensions.width;
+        const yScalingFactor = this.TargetResolution.height / this.screenDimensions.height;
+        if (source === 'api') {
+            if (x > this.TargetResolution.width || y > this.TargetResolution.height || x < 0 || y < 0) {
+                throw new base_1.ToolError(`Coordinates ${x}, ${y} are outside screen bounds `
+                    + `(${this.TargetResolution.width}x${this.TargetResolution.height})`);
+            }
+            return [
+                Math.round(x / xScalingFactor),
+                Math.round(y / yScalingFactor),
+            ];
+        }
+        return [
+            Math.round(x * xScalingFactor),
+            Math.round(y * yScalingFactor),
+        ];
+    }
+    requestControl(controlCommand) {
+        return __awaiter(this, void 0, void 0, function* () {
+            for (const action of controlCommand.actions) {
+                [action.position.x, action.position.y] = this.scaleCoordinates('api', action.position.x, action.position.y);
+            }
+            yield this.AgentOsClient.requestControl(controlCommand);
+        });
+    }
+}
+exports.OsAgentHandler = OsAgentHandler;
+class ScreenShotTool extends base_1.BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const screenshot = yield this.osAgentHandler.takeScreenshot();
+            return {
+                base64Images: [screenshot],
+                output: `Screenshot was taken, with resolution width ${this.osAgentHandler.getTargetResolution().width} and height ${this.osAgentHandler.getTargetResolution().height}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'screenshot_tool',
+            description: 'Takes a screenshot of the current screen and returns it as a base64 image',
+            input_schema: { type: 'object', properties: {}, required: [] },
+        };
+    }
+}
+exports.ScreenShotTool = ScreenShotTool;
+class MouseMoveTool extends base_1.BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.MOUSE_MOVE, { x: command.x, y: command.y }, '', {})]);
+            yield this.osAgentHandler.requestControl(controlCommand);
+            return {
+                output: `Moved mouse to (${command.x}, ${command.y})`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'mouse_move_tool',
+            description: 'Moves the mouse to the specified absolute coordinates. The top left corner of the screen is (0,0)',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    x: {
+                        type: 'number',
+                        description: 'The x coordinate of the element to click on',
+                    },
+                    y: {
+                        type: 'number',
+                        description: 'The y coordinate of the element to click on',
+                    },
+                },
+            },
+        };
+    }
+}
+exports.MouseMoveTool = MouseMoveTool;
+class MouseClickTool extends base_1.BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let controlCommand;
+            if (command.doubleClick) {
+                if (command.button === 'left') {
+                    controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.MOUSE_CLICK_DOUBLE_LEFT, { x: 0, y: 0 }, '', {})]);
+                }
+                if (command.button === 'right') {
+                    controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.MOUSE_CLICK_DOUBLE_RIGHT, { x: 0, y: 0 }, '', {})]);
+                }
+                if (command.button === 'middle') {
+                    controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.MOUSE_CLICK_DOUBLE_MIDDLE, { x: 0, y: 0 }, '', {})]);
+                }
+            }
+            else {
+                if (command.button === 'left') {
+                    controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.MOUSE_CLICK_LEFT, { x: 0, y: 0 }, '', {})]);
+                }
+                if (command.button === 'right') {
+                    controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.MOUSE_CLICK_RIGHT, { x: 0, y: 0 }, '', {})]);
+                }
+                if (command.button === 'middle') {
+                    controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.MOUSE_CLICK_MIDDLE, { x: 0, y: 0 }, '', {})]);
+                }
+            }
+            if (!controlCommand) {
+                throw new base_1.ToolError('Invalid input parameter for mouse click tool');
+            }
+            yield this.osAgentHandler.requestControl(controlCommand);
+            const returnedMessage = command.doubleClick ? `Double clicked ${command.button} button` : `Clicked ${command.button} button`;
+            return {
+                output: returnedMessage,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'mouse_click_tool',
+            description: 'Clicks the specified button on the mouse',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    button: {
+                        type: 'string',
+                        enum: ['left', 'right', 'middle'],
+                        description: 'The button to click',
+                    },
+                    doubleClick: {
+                        type: 'boolean',
+                        description: 'Whether to double click the button',
+                    },
+                },
+                required: ['button', 'doubleClick'],
+            },
+        };
+    }
+}
+exports.MouseClickTool = MouseClickTool;
+class MouseScrollTool extends base_1.BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.MOUSE_SCROLL, { x: command.dx, y: command.dy }, '', {})]);
+            yield this.osAgentHandler.requestControl(controlCommand);
+            return {
+                output: `Scrolled by (${command.dx}, ${command.dy})`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'mouse_scroll_tool',
+            description: 'Scrolls the mouse by the specified amount',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    dx: {
+                        type: 'number',
+                        description: 'The amount to scroll horizontally',
+                    },
+                    dy: {
+                        type: 'number',
+                        description: 'The amount to scroll vertically',
+                    },
+                },
+                required: ['dx', 'dy'],
+            },
+        };
+    }
+}
+exports.MouseScrollTool = MouseScrollTool;
+class DesktopKeyPressSequenceTool extends base_1.BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.PRESS_KEY_SEQUENCE, { x: 0, y: 0 }, command.key, {
+                    firstModifier: command.firstModifier || '',
+                    secondModifier: command.secondModifier || '',
+                })]);
+            yield this.osAgentHandler.requestControl(controlCommand);
+            return {
+                output: `Pressed key ${command.key} with modifiers ${command.firstModifier || ''} ${command.secondModifier || ''}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'desktop_key_press_sequence_tool',
+            description: 'Presses a key with optional modifiers',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    key: {
+                        type: 'string',
+                        enum: dsl_1.PC_KEY_VALUES,
+                        description: 'The key to press',
+                    },
+                    firstModifier: {
+                        type: 'string',
+                        enum: dsl_1.MODIFIER_KEY_VALUES,
+                        description: 'The first modifier key',
+                    },
+                    secondModifier: {
+                        type: 'string',
+                        enum: dsl_1.MODIFIER_KEY_VALUES,
+                        description: 'The second modifier key',
+                    },
+                },
+                required: ['key'],
+            },
+        };
+    }
+}
+exports.DesktopKeyPressSequenceTool = DesktopKeyPressSequenceTool;
+class DesktopSingleKeyPressTool extends base_1.BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.PRESS_KEY_SEQUENCE, { x: 0, y: 0 }, command.key, {})]);
+            yield this.osAgentHandler.requestControl(controlCommand);
+            return {
+                output: `Pressed key ${command.key}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'desktop_single_key_press_tool',
+            description: 'Presses a single key',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    key: {
+                        type: 'string',
+                        enum: [...dsl_1.PC_KEY_VALUES, ...dsl_1.MODIFIER_KEY_VALUES],
+                        description: 'The key to press',
+                    },
+                },
+                required: ['key'],
+            },
+        };
+    }
+}
+exports.DesktopSingleKeyPressTool = DesktopSingleKeyPressTool;
+class TypeTool extends base_1.BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.TYPE, { x: 0, y: 0 }, command.text, {})]);
+            yield this.osAgentHandler.requestControl(controlCommand);
+            return {
+                output: `Typed text: ${command.text}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'type_tool',
+            description: 'Types the specified text',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    text: {
+                        type: 'string',
+                        description: 'The text to type',
+                    },
+                },
+                required: ['text'],
+            },
+        };
+    }
+}
+exports.TypeTool = TypeTool;
+class AndroidSingleKeyPressTool extends base_1.BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.PRESS_KEY_SEQUENCE, { x: 0, y: 0 }, command.key, {})]);
+            yield this.osAgentHandler.requestControl(controlCommand);
+            return {
+                output: `Pressed Android key ${command.key}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'android_single_key_press_tool',
+            description: 'Presses a single Android key',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    key: {
+                        type: 'string',
+                        enum: dsl_1.ANDROID_KEY_VALUES,
+                        description: 'The Android key to press',
+                    },
+                },
+                required: ['key'],
+            },
+        };
+    }
+}
+exports.AndroidSingleKeyPressTool = AndroidSingleKeyPressTool;
+class AndroidSequenceKeyPressTool extends base_1.BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.PRESS_KEY_SEQUENCE, { x: 0, y: 0 }, command.keys.join(' '), {})]);
+            yield this.osAgentHandler.requestControl(controlCommand);
+            return {
+                output: `Pressed Android keys: ${command.keys.join(', ')}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'android_sequence_key_press_tool',
+            description: 'Presses a sequence of Android keys',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    keys: {
+                        type: 'array',
+                        items: {
+                            type: 'string',
+                            enum: dsl_1.ANDROID_KEY_VALUES,
+                        },
+                        description: 'The sequence of Android keys to press',
+                    },
+                },
+                required: ['keys'],
+            },
+        };
+    }
+}
+exports.AndroidSequenceKeyPressTool = AndroidSequenceKeyPressTool;
+class AgentErrorTool extends base_1.BaseAgentTool {
+    constructor() {
+        super();
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            throw new agent_errors_1.AgentError(command.error);
+        });
+    }
+    toParams() {
+        return {
+            name: 'agent_error_tool',
+            description: 'Raises an error in the agent',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    error: {
+                        type: 'string',
+                        description: 'The error message to raise',
+                    },
+                },
+                required: ['error'],
+            },
+        };
+    }
+}
+exports.AgentErrorTool = AgentErrorTool;
+class ExecuteShellCommandTool extends base_1.BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.EXECUTE_COMMAND, { x: 0, y: 0 }, command.command, {})]);
+            yield this.osAgentHandler.requestControl(controlCommand);
+            return {
+                output: `Executed shell command: ${command.command}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'execute_shell_command_tool',
+            description: 'Executes a shell command',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    command: {
+                        type: 'string',
+                        description: 'The shell command to execute',
+                    },
+                },
+                required: ['command'],
+            },
+        };
+    }
+}
+exports.ExecuteShellCommandTool = ExecuteShellCommandTool;

package/dist/cjs/core/ui-control-commands/index.d.ts

@@ -1,3 +1,5 @@
 export { ControlCommand } from './control-command';
 export { ControlCommandCode } from './control-command-code';
 export { InferenceResponse } from '../inference-response/inference-response';
+export { Action } from './action';
+export { InputEvent } from './input-event';

package/dist/cjs/core/ui-control-commands/index.js

@@ -1,9 +1,13 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.InferenceResponse = exports.ControlCommandCode = exports.ControlCommand = void 0;
+exports.InputEvent = exports.Action = exports.InferenceResponse = exports.ControlCommandCode = exports.ControlCommand = void 0;
 var control_command_1 = require("./control-command");
 Object.defineProperty(exports, "ControlCommand", { enumerable: true, get: function () { return control_command_1.ControlCommand; } });
 var control_command_code_1 = require("./control-command-code");
 Object.defineProperty(exports, "ControlCommandCode", { enumerable: true, get: function () { return control_command_code_1.ControlCommandCode; } });
 var inference_response_1 = require("../inference-response/inference-response");
 Object.defineProperty(exports, "InferenceResponse", { enumerable: true, get: function () { return inference_response_1.InferenceResponse; } });
+var action_1 = require("./action");
+Object.defineProperty(exports, "Action", { enumerable: true, get: function () { return action_1.Action; } });
+var input_event_1 = require("./input-event");
+Object.defineProperty(exports, "InputEvent", { enumerable: true, get: function () { return input_event_1.InputEvent; } });

package/dist/cjs/execution/dsl.d.ts

@@ -5,11 +5,14 @@ export declare enum Separators {
     STRING = "<|string|>"
 }
 export type INTERSECTION_AREA = 'element_center_line' | 'element_edge_area' | 'display_edge_area';
-export type PC_KEY = 'backspace' | 'delete' | 'enter' | 'tab' | 'escape' | 'up' | 'down' | 'right' | 'left' | 'home' | 'end' | 'pageup' | 'pagedown' | 'f1' | 'f2' | 'f3' | 'f4' | 'f5' | 'f6' | 'f7' | 'f8' | 'f9' | 'f10' | 'f11' | 'f12' | 'space' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '!' | '"' | '#' | '$' | '%' | '&' | "'" | '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | ':' | ';' | '<' | '=' | '>' | '?' | '@' | '[' | '\\' | ']' | '^' | '_' | '`' | '{' | '|' | '}' | '~ ';
-export type ANDROID_KEY = 'home' | 'back' | 'call' | 'endcall' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | 'star' | 'pound' | 'dpad_up' | 'dpad_down' | 'dpad_left' | 'dpad_right' | 'dpad_center' | 'volume_up' | 'volume_down' | 'power' | 'camera' | 'clear' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | 'comma' | 'period' | 'alt_left' | 'alt_right' | 'shift_left' | 'shift_right' | 'tab' | 'space' | 'sym' | 'explorer' | 'envelope' | 'enter' | 'del' | 'grave' | 'minus' | 'equals' | 'left_bracket' | 'right_bracket' | 'backslash' | 'semicolon' | 'apostrophe' | 'slash' | 'at' | 'num' | 'headsethook' | 'focus' | 'plus' | 'menu' | 'notification' | 'search' | 'media_play_pause' | 'media_stop' | 'media_next' | 'media_previous' | 'media_rewind' | 'media_fast_forward' | 'mute' | 'page_up' | 'page_down' | 'switch_charset' | 'escape' | 'forward_del' | 'ctrl_left' | 'ctrl_right' | 'caps_lock' | 'scroll_lock' | 'function' | 'break' | 'move_home' | 'move_end' | 'insert' | 'forward' | 'media_play' | 'media_pause' | 'media_close' | 'media_eject' | 'media_record' | 'f1' | 'f2' | 'f3' | 'f4' | 'f5' | 'f6' | 'f7' | 'f8' | 'f9' | 'f10' | 'f11' | 'f12' | 'num_lock' | 'numpad_0' | 'numpad_1' | 'numpad_2' | 'numpad_3' | 'numpad_4' | 'numpad_5' | 'numpad_6' | 'numpad_7' | 'numpad_8' | 'numpad_9' | 'numpad_divide' | 'numpad_multiply' | 'numpad_subtract' | 'numpad_add' | 'numpad_dot' | 'numpad_comma' | 'numpad_enter' | 'numpad_equals' | 'numpad_left_paren' | 'numpad_right_paren' | 'volume_mute' | 'info' | 'channel_up' | 'channel_down' | 'zoom_in' | 'zoom_out' | 'window' | 'guide' | 'bookmark' | 'captions' | 'settings' | 'app_switch' | 'language_switch' | 'contacts' | 'calendar' | 'music' | 'calculator' | 'assist' | 'brightness_down' | 'brightness_up' | 'media_audio_track' | 'sleep' | 'wakeup' | 'pairing' | 'media_top_menu' | 'last_channel' | 'tv_data_service' | 'voice_assist' | 'help' | 'navigate_previous' | 'navigate_next' | 'navigate_in' | 'navigate_out' | 'dpad_up_left' | 'dpad_down_left' | 'dpad_up_right' | 'dpad_down_right' | 'media_skip_forward' | 'media_skip_backward' | 'media_step_forward' | 'media_step_backward' | 'soft_sleep' | 'cut' | 'copy' | 'paste' | 'all_apps' | 'refresh';
-export type MODIFIER_KEY = 'command' | 'alt' | 'control' | 'shift' | 'right_shift';
+export declare const PC_KEY_VALUES: readonly ["backspace", "delete", "enter", "tab", "escape", "up", "down", "right", "left", "home", "end", "pageup", "pagedown", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "f9", "f10", "f11", "f12", "space", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "!", "\"", "#", "$", "%", "&", "'", "(", ")", "*", "+", ",", "-", ".", "/", ":", ";", "<", "=", ">", "?", "@", "[", "\\", "]", "^", "_", "`", "{", "|", "}", "~ "];
+export type PC_KEY = typeof PC_KEY_VALUES[number];
+export declare const ANDROID_KEY_VALUES: readonly ["backspace", "delete", "enter", "tab", "escape", "up", "down", "right", "left", "home", "end", "pageup", "pagedown", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "f9", "f10", "f11", "f12", "space", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "!", "\"", "#", "$", "%", "&", "'", "(", ")", "*", "+", ",", "-", ".", "/", ":", ";", "<", "=", ">", "?", "@", "[", "\\", "]", "^", "_", "`", "{", "|", "}", "~ "];
+export type ANDROID_KEY = typeof ANDROID_KEY_VALUES[number];
+export declare const MODIFIER_KEY_VALUES: readonly ["command", "alt", "control", "shift", "right_shift"];
+export type MODIFIER_KEY = typeof MODIFIER_KEY_VALUES[number];
+export type PC_AND_MODIFIER_KEY = PC_KEY | MODIFIER_KEY;
 export type COLOR = 'black' | 'white' | 'red' | 'green' | 'yellow green' | 'orange' | 'yellow' | 'purple' | 'pink' | 'gray' | 'lime green' | 'royal blue';
-export type PC_AND_MODIFIER_KEY = 'command' | 'alt' | 'control' | 'shift' | 'right_shift' | 'backspace' | 'delete' | 'enter' | 'tab' | 'escape' | 'up' | 'down' | 'right' | 'left' | 'home' | 'end' | 'pageup' | 'pagedown' | 'f1' | 'f2' | 'f3' | 'f4' | 'f5' | 'f6' | 'f7' | 'f8' | 'f9' | 'f10' | 'f11' | 'f12' | 'space' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '!' | '"' | '#' | '$' | '%' | '&' | "'" | '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | ':' | ';' | '<' | '=' | '>' | '?' | '@' | '[' | '\\' | ']' | '^' | '_' | '`' | '{' | '|' | '}' | '~ ';
 export interface CommandExecutorContext {
     customElementsJson: CustomElementJson[];
     aiElementNames: string[];

package/dist/cjs/execution/dsl.js

@@ -5,7 +5,7 @@
 /* eslint-disable max-len */
 // Autogenerated from typescript.template file
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ApiCommands = exports.Getter = exports.FluentFiltersOrRelationsGetter = exports.FluentFiltersGetter = exports.ExecGetter = exports.FluentCommand = exports.FluentFiltersOrRelationsCondition = exports.FluentFiltersCondition = exports.FluentFiltersOrRelations = exports.FluentFilters = exports.Exec = exports.Separators = void 0;
+exports.ApiCommands = exports.Getter = exports.FluentFiltersOrRelationsGetter = exports.FluentFiltersGetter = exports.ExecGetter = exports.FluentCommand = exports.FluentFiltersOrRelationsCondition = exports.FluentFiltersCondition = exports.FluentFiltersOrRelations = exports.FluentFilters = exports.Exec = exports.MODIFIER_KEY_VALUES = exports.ANDROID_KEY_VALUES = exports.PC_KEY_VALUES = exports.Separators = void 0;
 function isStackTraceCodeline(line) {
     return /[ \t]+at .+/.test(line);
 }
@@ -37,6 +37,9 @@ var Separators;
 (function (Separators) {
     Separators["STRING"] = "<|string|>";
 })(Separators || (exports.Separators = Separators = {}));
+exports.PC_KEY_VALUES = ['backspace', 'delete', 'enter', 'tab', 'escape', 'up', 'down', 'right', 'left', 'home', 'end', 'pageup', 'pagedown', 'f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'f10', 'f11', 'f12', 'space', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', '{', '|', '}', '~ '];
+exports.ANDROID_KEY_VALUES = ['backspace', 'delete', 'enter', 'tab', 'escape', 'up', 'down', 'right', 'left', 'home', 'end', 'pageup', 'pagedown', 'f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'f10', 'f11', 'f12', 'space', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', '{', '|', '}', '~ '];
+exports.MODIFIER_KEY_VALUES = ['command', 'alt', 'control', 'shift', 'right_shift'];
 class FluentBase {
     constructor(prev) {
         this.prev = prev;

package/dist/cjs/execution/execution-runtime.d.ts

@@ -1,3 +1,5 @@
+import { BetaMessage, BetaMessageParam } from '@anthropic-ai/sdk/resources/beta/messages';
+import { ControlCommand } from '../core/ui-control-commands';
 import { UiControllerClient } from './ui-controller-client';
 import { InferenceClient } from './inference-client';
 import { Annotation } from '../core/annotation/annotation';
@@ -18,7 +20,7 @@ export declare class ExecutionRuntime {
     startVideoRecording(): Promise<void>;
     stopVideoRecording(): Promise<void>;
     readVideoRecording(): Promise<string>;
-    private requestControl;
+    requestControl(controlCommand: ControlCommand): Promise<void>;
     executeInstruction(instruction: Instruction, modelComposition: ModelCompositionBranch[]): Promise<void>;
     private readonly EXEC_REPETITION_COUNT;
     private executeCommandRepeatedly;
@@ -38,4 +40,12 @@ export declare class ExecutionRuntime {
     getDetectedElements(instruction: string, customElementJson?: CustomElementJson[]): Promise<DetectedElement[]>;
     annotateImage(imagePath?: string, customElementJson?: CustomElementJson[], elements?: DetectedElement[]): Promise<Annotation>;
     predictVQA(prompt: string, config?: object): Promise<any>;
+    predictActResponse(params: {
+        max_tokens: number;
+        messages: BetaMessageParam[];
+        model: string;
+        system?: string;
+        tools?: object[];
+        betas?: string[];
+    }): Promise<BetaMessage>;
 }

package/dist/cjs/execution/execution-runtime.js

@@ -215,5 +215,10 @@ class ExecutionRuntime {
             return this.inferenceClient.predictVQAAnswer(prompt, base64Image, config);
         });
     }
+    predictActResponse(params) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.inferenceClient.predictActResponse(params);
+        });
+    }
 }
 exports.ExecutionRuntime = ExecutionRuntime;

package/dist/cjs/execution/inference-client.d.ts

@@ -1,3 +1,4 @@
+import { BetaMessage, BetaMessageParam } from '@anthropic-ai/sdk/resources/beta/messages';
 import { HttpClientGot } from '../utils/http/http-client-got';
 import { ControlCommand } from '../core/ui-control-commands';
 import { CustomElement } from '../core/model/custom-element';
@@ -23,4 +24,12 @@ export declare class InferenceClient {
     getDetectedElements(instruction: string, image: string, customElements?: CustomElement[]): Promise<DetectedElement[]>;
     predictImageAnnotation(image: string, customElements?: CustomElement[]): Promise<Annotation>;
     predictVQAAnswer(prompt: string, image: string, config?: object): Promise<any>;
+    predictActResponse(params: {
+        max_tokens: number;
+        messages: BetaMessageParam[];
+        model: string;
+        system?: string;
+        tools?: object[];
+        betas?: string[];
+    }): Promise<BetaMessage>;
 }