askui 0.29.0 → 0.31.0

package/dist/cjs/core/models/anthropic/askui-agent.js

@@ -16,6 +16,7 @@ class AskUIAgent extends claude_agent_1.ClaudeAgent {
     constructor(executionRuntime) {
         super((params) => executionRuntime.predictActResponse(params));
         this.osAgentHandler = undefined;
+        this.runtime = 'desktop';
         this.executionRuntime = executionRuntime;
     }
     isConnected() {
@@ -24,6 +25,7 @@ class AskUIAgent extends claude_agent_1.ClaudeAgent {
     initializeOsAgentHandler() {
         return __awaiter(this, void 0, void 0, function* () {
             this.osAgentHandler = yield os_agent_tools_1.OsAgentHandler.createInstance(this.executionRuntime);
+            this.runtime = this.osAgentHandler.runtime;
         });
     }
     getOsAgentHandler() {
@@ -32,50 +34,48 @@ class AskUIAgent extends claude_agent_1.ClaudeAgent {
         }
         return this.osAgentHandler;
     }
-    configureAsDesktopAgent() {
+    configureAgent() {
         return __awaiter(this, void 0, void 0, function* () {
             if (!this.osAgentHandler) {
                 throw new Error('Agent OS client is not connected');
             }
-            const tools = [
+            let systemPrompt = AskUIAgent.DesktopSystemPrompt;
+            let tools = [
                 new os_agent_tools_1.AgentErrorTool(),
                 new os_agent_tools_1.PrintTool(),
-                new os_agent_tools_1.ScreenShotTool(this.osAgentHandler),
-                new os_agent_tools_1.MouseMoveTool(this.osAgentHandler),
-                new os_agent_tools_1.MouseClickTool(this.osAgentHandler),
-                new os_agent_tools_1.MouseScrollTool(this.osAgentHandler),
-                new os_agent_tools_1.TypeTool(this.osAgentHandler),
-                new os_agent_tools_1.DesktopPressAndReleaseKeysTool(this.osAgentHandler),
-                new os_agent_tools_1.DesktopKeyHoldDownTool(this.osAgentHandler),
-                new os_agent_tools_1.DesktopKeyReleaseTool(this.osAgentHandler),
-                new os_agent_tools_1.MouseHoldLeftButtonDownTool(this.osAgentHandler),
-                new os_agent_tools_1.MouseReleaseLeftButtonTool(this.osAgentHandler),
-                new os_agent_tools_1.MouseDragAndDropTool(this.osAgentHandler),
                 new os_agent_tools_1.WaitTool(),
-            ];
-            this.setTools(tools);
-            this.setSystemPrompt(AskUIAgent.DesktopSystemPrompt);
-        });
-    }
-    configureAsAndroidAgent() {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.osAgentHandler) {
-                throw new Error('Agent OS client is not connected');
-            }
-            const tools = [
-                new os_agent_tools_1.AgentErrorTool(),
                 new os_agent_tools_1.ScreenShotTool(this.osAgentHandler),
-                new os_agent_tools_1.MouseMoveTool(this.osAgentHandler),
-                new os_agent_tools_1.MouseClickTool(this.osAgentHandler),
-                new os_agent_tools_1.MouseScrollTool(this.osAgentHandler),
-                new os_agent_tools_1.AndroidSingleKeyPressTool(this.osAgentHandler),
-                new os_agent_tools_1.AndroidSequenceKeyPressTool(this.osAgentHandler),
                 new os_agent_tools_1.TypeTool(this.osAgentHandler),
-                new os_agent_tools_1.ExecuteShellCommandTool(this.osAgentHandler),
-                new os_agent_tools_1.WaitTool(),
             ];
+            if (this.runtime === 'desktop') {
+                tools = [
+                    ...tools,
+                    new os_agent_tools_1.MouseMoveTool(this.osAgentHandler),
+                    new os_agent_tools_1.MouseClickTool(this.osAgentHandler),
+                    new os_agent_tools_1.MouseScrollTool(this.osAgentHandler),
+                    new os_agent_tools_1.DesktopPressAndReleaseKeysTool(this.osAgentHandler),
+                    new os_agent_tools_1.DesktopKeyHoldDownTool(this.osAgentHandler),
+                    new os_agent_tools_1.DesktopKeyReleaseTool(this.osAgentHandler),
+                    new os_agent_tools_1.MouseHoldLeftButtonDownTool(this.osAgentHandler),
+                    new os_agent_tools_1.MouseReleaseLeftButtonTool(this.osAgentHandler),
+                    new os_agent_tools_1.MouseDragAndDropTool(this.osAgentHandler),
+                    new os_agent_tools_1.ExecuteShellCommandTool(this.osAgentHandler),
+                ];
+            }
+            if (this.runtime === 'android') {
+                tools = [
+                    ...tools,
+                    new os_agent_tools_1.AndroidSingleKeyPressTool(this.osAgentHandler),
+                    new os_agent_tools_1.AndroidSequenceKeyPressTool(this.osAgentHandler),
+                    new os_agent_tools_1.AndroidSwipeTool(this.osAgentHandler),
+                    new os_agent_tools_1.AndroidDragAndDropTool(this.osAgentHandler),
+                    new os_agent_tools_1.AndroidTapTool(this.osAgentHandler),
+                    new os_agent_tools_1.AndroidShellCommandTool(this.osAgentHandler),
+                ];
+                systemPrompt = AskUIAgent.AndroidSystemPrompt;
+            }
             this.setTools(tools);
-            this.setSystemPrompt(AskUIAgent.AndroidSystemPrompt);
+            this.setSystemPrompt(systemPrompt);
         });
     }
 }

package/dist/cjs/core/models/anthropic/tools/os-agent-tools.d.ts

@@ -5,13 +5,14 @@ import { ExecutionRuntime } from '../../../../execution/execution-runtime';
 import { ControlCommand } from '../../../ui-control-commands';
 export declare class OsAgentHandler {
     private AgentOsClient;
-    private targetResolution;
     private screenDimensions;
+    runtime: 'android' | 'desktop';
+    private targetResolution;
     private paddingInfo;
     constructor(AgentOsClient: ExecutionRuntime, screenDimensions: {
         width: number;
         height: number;
-    });
+    }, runtime: 'android' | 'desktop');
     private updatePaddingInfo;
     static createInstance(AgentOsClient: ExecutionRuntime): Promise<OsAgentHandler>;
     getTargetResolution(): {
@@ -27,7 +28,7 @@ export declare class OsAgentHandler {
     scaleCoordinates(source: 'api' | 'computer', x: number, y: number): [number, number];
     requestControl(controlCommand: ControlCommand): Promise<void>;
     mouseMove(x: number, y: number): Promise<void>;
-    mouseClick(button: "left" | "right" | "middle", doubleClick: boolean): Promise<void>;
+    mouseClick(button: 'left' | 'right' | 'middle', doubleClick: boolean): Promise<void>;
     mouseScroll(dx: number, dy: number): Promise<void>;
     mouseHoldLeftButtonDown(): Promise<void>;
     mouseReleaseLeftButton(): Promise<void>;
@@ -38,6 +39,10 @@ export declare class OsAgentHandler {
     androidKeyPress(key: ANDROID_KEY): Promise<void>;
     androidKeySequencePress(keys: ANDROID_KEY[]): Promise<void>;
     executeShellCommand(command: string): Promise<void>;
+    AndroidSwipeTool(startX: number, startY: number, endX: number, endY: number): Promise<void>;
+    AndroidDragAndDropTool(startX: number, startY: number, endX: number, endY: number): Promise<void>;
+    AndroidTapTool(x: number, y: number): Promise<void>;
+    executeAndroidShellCommand(command: string): Promise<void>;
 }
 export declare class ScreenShotTool extends BaseAgentTool {
     private osAgentHandler;
@@ -175,3 +180,42 @@ export declare class PrintTool extends BaseAgentTool {
     }): Promise<ToolResult>;
     toParams(): BetaTool;
 }
+export declare class AndroidSwipeTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(command: {
+        startX: number;
+        startY: number;
+        endX: number;
+        endY: number;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class AndroidDragAndDropTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(command: {
+        startX: number;
+        startY: number;
+        endX: number;
+        endY: number;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class AndroidTapTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(command: {
+        x: number;
+        y: number;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class AndroidShellCommandTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(command: {
+        command: string;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}

package/dist/cjs/core/models/anthropic/tools/os-agent-tools.js

@@ -9,15 +9,17 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.PrintTool = exports.WaitTool = exports.ExecuteShellCommandTool = exports.AgentErrorTool = exports.AndroidSequenceKeyPressTool = exports.AndroidSingleKeyPressTool = exports.TypeTool = exports.DesktopKeyReleaseTool = exports.DesktopKeyHoldDownTool = exports.DesktopPressAndReleaseKeysTool = exports.MouseReleaseLeftButtonTool = exports.MouseHoldLeftButtonDownTool = exports.MouseDragAndDropTool = exports.MouseScrollTool = exports.MouseClickTool = exports.MouseMoveTool = exports.ScreenShotTool = exports.OsAgentHandler = void 0;
+exports.AndroidShellCommandTool = exports.AndroidTapTool = exports.AndroidDragAndDropTool = exports.AndroidSwipeTool = exports.PrintTool = exports.WaitTool = exports.ExecuteShellCommandTool = exports.AgentErrorTool = exports.AndroidSequenceKeyPressTool = exports.AndroidSingleKeyPressTool = exports.TypeTool = exports.DesktopKeyReleaseTool = exports.DesktopKeyHoldDownTool = exports.DesktopPressAndReleaseKeysTool = exports.MouseReleaseLeftButtonTool = exports.MouseHoldLeftButtonDownTool = exports.MouseDragAndDropTool = exports.MouseScrollTool = exports.MouseClickTool = exports.MouseMoveTool = exports.ScreenShotTool = exports.OsAgentHandler = void 0;
 const dsl_1 = require("../../../../execution/dsl");
 const base_1 = require("./base");
 const ui_control_commands_1 = require("../../../ui-control-commands");
 const base_64_image_1 = require("../../../../utils/base_64_image/base-64-image");
 const agent_errors_1 = require("./agent-errors");
 class OsAgentHandler {
-    constructor(AgentOsClient, screenDimensions) {
+    constructor(AgentOsClient, screenDimensions, runtime) {
         this.AgentOsClient = AgentOsClient;
+        this.screenDimensions = screenDimensions;
+        this.runtime = runtime;
         this.targetResolution = { width: 1280, height: 800 };
         this.paddingInfo = null;
         this.screenDimensions = screenDimensions;
@@ -48,7 +50,7 @@ class OsAgentHandler {
             scaledWidth,
             scaledHeight,
             padLeft,
-            padTop
+            padTop,
         };
     }
     // Add image support to act, an check for function overload in typescript.
@@ -56,10 +58,12 @@ class OsAgentHandler {
         return __awaiter(this, void 0, void 0, function* () {
             const base64ImageString = yield AgentOsClient.getScreenshot();
             const image_info = yield (yield base_64_image_1.Base64Image.fromString(base64ImageString)).getInfo();
+            const startingArguments = yield AgentOsClient.getStartingArguments();
+            const runtime = startingArguments['runtime'] === 'android' ? 'android' : 'desktop';
             return new OsAgentHandler(AgentOsClient, {
                 width: image_info.width,
                 height: image_info.height,
-            });
+            }, runtime);
         });
     }
     getTargetResolution() {
@@ -90,7 +94,7 @@ class OsAgentHandler {
         if (!this.paddingInfo) {
             throw new base_1.ToolError('Padding information not initialized');
         }
-        const { scaleFactor, scaledWidth, scaledHeight, padLeft, padTop } = this.paddingInfo;
+        const { scaleFactor, scaledWidth, scaledHeight, padLeft, padTop, } = this.paddingInfo;
         if (source === 'api') {
             if (x > this.targetResolution.width || y > this.targetResolution.height || x < 0 || y < 0) {
                 throw new base_1.ToolError(`Coordinates ${x}, ${y} are outside screen bounds `
@@ -131,23 +135,21 @@ class OsAgentHandler {
         return __awaiter(this, void 0, void 0, function* () {
             let action = ui_control_commands_1.InputEvent.MOUSE_CLICK_LEFT;
             if (doubleClick) {
-                if (button === "left") {
+                if (button === 'left') {
                     action = ui_control_commands_1.InputEvent.MOUSE_CLICK_DOUBLE_LEFT;
                 }
-                else if (button === "right") {
+                else if (button === 'right') {
                     action = ui_control_commands_1.InputEvent.MOUSE_CLICK_DOUBLE_RIGHT;
                 }
-                else if (button === "middle") {
+                else if (button === 'middle') {
                     action = ui_control_commands_1.InputEvent.MOUSE_CLICK_DOUBLE_MIDDLE;
                 }
             }
-            else {
-                if (button === "right") {
-                    action = ui_control_commands_1.InputEvent.MOUSE_CLICK_RIGHT;
-                }
-                else if (button === "middle") {
-                    action = ui_control_commands_1.InputEvent.MOUSE_CLICK_MIDDLE;
-                }
+            else if (button === 'right') {
+                action = ui_control_commands_1.InputEvent.MOUSE_CLICK_RIGHT;
+            }
+            else if (button === 'middle') {
+                action = ui_control_commands_1.InputEvent.MOUSE_CLICK_MIDDLE;
             }
             const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(action, { x: 0, y: 0 }, '', {})]);
             yield this.requestControl(controlCommand);
@@ -184,8 +186,8 @@ class OsAgentHandler {
     desktopKeyHoldDown(key_1) {
         return __awaiter(this, arguments, void 0, function* (key, modifiers = []) {
             const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.KEY_PRESS, { x: 0, y: 0 }, '', {
-                    key: key,
-                    modifiers: modifiers,
+                    key,
+                    modifiers,
                 })]);
             yield this.requestControl(controlCommand);
         });
@@ -193,8 +195,8 @@ class OsAgentHandler {
     desktopKeyRelease(key_1) {
         return __awaiter(this, arguments, void 0, function* (key, modifiers = []) {
             const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.KEY_RELEASE, { x: 0, y: 0 }, '', {
-                    key: key,
-                    modifiers: modifiers,
+                    key,
+                    modifiers,
                 })]);
             yield this.requestControl(controlCommand);
         });
@@ -223,6 +225,47 @@ class OsAgentHandler {
             yield this.requestControl(controlCommand);
         });
     }
+    AndroidSwipeTool(startX, startY, endX, endY) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.runtime !== 'android') {
+                throw new base_1.ToolError('This tool is only available on Android devices');
+            }
+            [startX, startY] = this.scaleCoordinates('api', startX, startY);
+            [endX, endY] = this.scaleCoordinates('api', endX, endY);
+            const adbCommand = `input swipe ${startX} ${startY} ${endX} ${endY}`;
+            yield this.executeShellCommand(adbCommand);
+        });
+    }
+    AndroidDragAndDropTool(startX, startY, endX, endY) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.runtime !== 'android') {
+                throw new base_1.ToolError('This tool is only available on Android devices');
+            }
+            [startX, startY] = this.scaleCoordinates('api', startX, startY);
+            [endX, endY] = this.scaleCoordinates('api', endX, endY);
+            const adbCommand = `input draganddrop ${startX} ${startY} ${endX} ${endY}`;
+            yield this.executeShellCommand(adbCommand);
+        });
+    }
+    AndroidTapTool(x, y) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.runtime !== 'android') {
+                throw new base_1.ToolError('This tool is only available on Android devices');
+            }
+            [x, y] = this.scaleCoordinates('api', x, y);
+            const adbCommand = `input tap ${x} ${y}`;
+            yield this.executeShellCommand(adbCommand);
+        });
+    }
+    executeAndroidShellCommand(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.runtime !== 'android') {
+                throw new base_1.ToolError('This tool is only available on Android devices');
+            }
+            command = command.replace(/^adb shell /, '');
+            yield this.executeShellCommand(command);
+        });
+    }
 }
 exports.OsAgentHandler = OsAgentHandler;
 class ScreenShotTool extends base_1.BaseAgentTool {
@@ -605,7 +648,8 @@ class AndroidSingleKeyPressTool extends base_1.BaseAgentTool {
     }
     execute(command) {
         return __awaiter(this, void 0, void 0, function* () {
-            yield this.osAgentHandler.androidKeyPress(command.key);
+            const adbCommand = `input keyevent ${command.key.toUpperCase()}`;
+            yield this.osAgentHandler.executeShellCommand(adbCommand);
             return {
                 output: `Pressed Android key ${command.key}`,
             };
@@ -637,7 +681,8 @@ class AndroidSequenceKeyPressTool extends base_1.BaseAgentTool {
     }
     execute(command) {
         return __awaiter(this, void 0, void 0, function* () {
-            yield this.osAgentHandler.androidKeySequencePress(command.keys);
+            const adbCommand = `input keyevent ${command.keys.map((key) => key.toUpperCase()).join(' ')}`;
+            yield this.osAgentHandler.executeShellCommand(adbCommand);
             return {
                 output: `Pressed Android keys: ${command.keys.join(', ')}`,
             };
@@ -708,7 +753,7 @@ class ExecuteShellCommandTool extends base_1.BaseAgentTool {
     toParams() {
         return {
             name: 'execute_shell_command_tool',
-            description: 'Executes a shell command',
+            description: 'Executes a shell command. It does not return the output of the command.',
             input_schema: {
                 type: 'object',
                 properties: {
@@ -729,7 +774,7 @@ class WaitTool extends base_1.BaseAgentTool {
     }
     execute(command) {
         return __awaiter(this, void 0, void 0, function* () {
-            yield new Promise(resolve => setTimeout(resolve, command.milliseconds));
+            yield new Promise((resolve) => setTimeout(resolve, command.milliseconds));
             return {
                 output: `Waited for ${command.milliseconds} milliseconds`,
             };
@@ -783,3 +828,155 @@ class PrintTool extends base_1.BaseAgentTool {
     }
 }
 exports.PrintTool = PrintTool;
+class AndroidSwipeTool extends base_1.BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.osAgentHandler.AndroidSwipeTool(command.startX, command.startY, command.endX, command.endY);
+            return {
+                output: `Swiped from ${command.startX}, ${command.startY} to ${command.endX}, ${command.endY} on the screen`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'android_swipe_tool',
+            description: 'Swipes from a starting point to an ending point on the screen',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    startX: {
+                        type: 'number',
+                        description: 'The x (pixels from the left edge) coordinate of the start position',
+                    },
+                    startY: {
+                        type: 'number',
+                        description: 'The y (pixels from the top edge) coordinate of the start position',
+                    },
+                    endX: {
+                        type: 'number',
+                        description: 'The x (pixels from the left edge) coordinate of the end position',
+                    },
+                    endY: {
+                        type: 'number',
+                        description: 'The y (pixels from the top edge) coordinate of the end position',
+                    },
+                },
+                required: ['startX', 'startY', 'endX', 'endY'],
+            },
+        };
+    }
+}
+exports.AndroidSwipeTool = AndroidSwipeTool;
+class AndroidDragAndDropTool extends base_1.BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.osAgentHandler.AndroidDragAndDropTool(command.startX, command.startY, command.endX, command.endY);
+            return {
+                output: `Dragged and dropped from ${command.startX}, ${command.startY} to ${command.endX}, ${command.endY} on the screen`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'android_drag_and_drop_tool',
+            description: 'Drags and drops from a starting point to an ending point on the screen',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    startX: {
+                        type: 'number',
+                        description: 'The x (pixels from the left edge) coordinate of the start position',
+                    },
+                    startY: {
+                        type: 'number',
+                        description: 'The y (pixels from the top edge) coordinate of the start position',
+                    },
+                    endX: {
+                        type: 'number',
+                        description: 'The x (pixels from the left edge) coordinate of the end position',
+                    },
+                    endY: {
+                        type: 'number',
+                        description: 'The y (pixels from the top edge) coordinate of the end position',
+                    },
+                },
+                required: ['startX', 'startY', 'endX', 'endY'],
+            },
+        };
+    }
+}
+exports.AndroidDragAndDropTool = AndroidDragAndDropTool;
+class AndroidTapTool extends base_1.BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.osAgentHandler.AndroidTapTool(command.x, command.y);
+            return {
+                output: `Tapped the screen at ${command.x}, ${command.y}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'android_tap_tool',
+            description: 'Taps the screen at the specified coordinates',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    x: {
+                        type: 'number',
+                        description: 'The x (pixels from the left edge) coordinate of the tap position',
+                    },
+                    y: {
+                        type: 'number',
+                        description: 'The y (pixels from the top edge) coordinate of the tap position',
+                    },
+                },
+                required: ['x', 'y'],
+            },
+        };
+    }
+}
+exports.AndroidTapTool = AndroidTapTool;
+class AndroidShellCommandTool extends base_1.BaseAgentTool {
+    constructor(osAgentHandler) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.osAgentHandler.executeAndroidShellCommand(command.command);
+            return {
+                output: `Executed shell command: ${command.command}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'android_shell_command_tool',
+            description: 'Executes a shell command on the Android device. It does not return the output of the command.',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    command: {
+                        type: 'string',
+                        description: 'The shell command to execute without the "adb shell" prefix',
+                    },
+                },
+                required: ['command'],
+            },
+        };
+    }
+}
+exports.AndroidShellCommandTool = AndroidShellCommandTool;

package/dist/cjs/core/ui-control-commands/action.d.ts

@@ -15,4 +15,5 @@ export declare class Action {
         y: number;
     }, text: string, parameters?: ActionParameters);
     static fromJson(action: Action, resizeRatio?: number): Action;
+    toJson(): object;
 }

package/dist/cjs/core/ui-control-commands/action.js

@@ -11,9 +11,17 @@ class Action {
     }
     static fromJson(action, resizeRatio = 1) {
         return new Action(input_event_1.InputEvent[action.inputEvent], {
-            x: action.position.x * resizeRatio,
-            y: action.position.y * resizeRatio,
+            x: Math.round(action.position.x * resizeRatio),
+            y: Math.round(action.position.y * resizeRatio),
         }, action.text, action.parameters ? action.parameters : {});
     }
+    toJson() {
+        return {
+            inputEvent: this.inputEvent,
+            parameters: this.parameters,
+            position: this.position,
+            text: this.text,
+        };
+    }
 }
 exports.Action = Action;

package/dist/cjs/core/ui-control-commands/control-command.d.ts

@@ -7,4 +7,5 @@ export declare class ControlCommand {
     constructor(code: ControlCommandCode, actions: Action[], tryToRepeat?: boolean);
     static fromJson(json: unknown, resizeRatio?: number): ControlCommand;
     setTextToBeTyped(text: string): void;
+    toJson(): object;
 }

package/dist/cjs/core/ui-control-commands/control-command.js

@@ -18,5 +18,12 @@ class ControlCommand {
         this.actions = this.actions.map((action) => ([input_event_1.InputEvent.TYPE, input_event_1.InputEvent.TYPE_TEXT].includes(action.inputEvent)
             ? new action_1.Action(action.inputEvent, action.position, text, action.parameters) : action));
     }
+    toJson() {
+        return {
+            actions: this.actions.map((action) => action.toJson()),
+            code: this.code,
+            tryToRepeat: this.tryToRepeat,
+        };
+    }
 }
 exports.ControlCommand = ControlCommand;

package/dist/cjs/execution/dsl.d.ts

@@ -1,6 +1,7 @@
 import { CustomElementJson } from '../core/model/custom-element-json';
 import { DetectedElement } from '../core/model/annotation-result/detected-element';
 import { ModelCompositionBranch } from './model-composition-branch';
+import { RetryStrategy } from './retry-strategies/retry-strategy';
 export declare enum Separators {
     STRING = "<|string|>"
 }
@@ -17,22 +18,27 @@ export interface CommandExecutorContext {
     customElementsJson: CustomElementJson[];
     aiElementNames: string[];
 }
+export interface ExecOptions {
+    modelComposition?: ModelCompositionBranch[];
+    skipCache?: boolean;
+    retryStrategy?: RetryStrategy;
+}
 declare abstract class FluentBase {
     protected prev?: FluentBase | undefined;
     constructor(prev?: FluentBase | undefined);
     protected _textStr: string;
     protected _params: Map<string, unknown>;
     protected static addParams(paramsList: Map<string, unknown[]>, params: Map<string, unknown>): Map<string, unknown[]>;
-    protected fluentCommandStringBuilder(modelComposition: ModelCompositionBranch[], currentInstruction?: string, paramsList?: Map<string, unknown[]>): Promise<void>;
+    protected fluentCommandStringBuilder(modelComposition?: ModelCompositionBranch[], skipCache?: boolean, retryStrategy?: RetryStrategy, currentInstruction?: string, paramsList?: Map<string, unknown[]>): Promise<void>;
     protected getterStringBuilder(currentInstruction?: string, paramsList?: Map<string, unknown[]>): Promise<DetectedElement[]>;
     protected get textStr(): string;
     protected get params(): Map<string, unknown>;
 }
 export interface Executable {
-    exec(): Promise<void>;
+    exec(execOptions?: ExecOptions): Promise<void>;
 }
 export declare class Exec extends FluentBase implements Executable {
-    exec(modelComposition?: ModelCompositionBranch[]): Promise<void>;
+    exec(execOptions?: ExecOptions): Promise<void>;
 }
 export declare class FluentFilters extends FluentBase {
     /**
@@ -808,7 +814,7 @@ export declare class FluentFiltersOrRelations extends FluentFilters {
      * @return {FluentFilters}
      */
     contains(): FluentFilters;
-    exec(modelComposition?: ModelCompositionBranch[]): Promise<void>;
+    exec(execOptions?: ExecOptions): Promise<void>;
 }
 export declare class FluentFiltersCondition extends FluentBase {
     /**
@@ -1634,6 +1640,7 @@ export declare class FluentFiltersOrRelationsCondition extends FluentFiltersCond
     notExists(): ExecCondition;
 }
 declare class ExecCondition extends Exec {
+    exec(execOptions?: ExecOptions): Promise<void>;
 }
 export declare abstract class FluentCommand extends FluentBase {
     constructor();
@@ -2082,7 +2089,7 @@ export declare abstract class FluentCommand extends FluentBase {
      * @return {Exec}
      */
     pressAndroidKey(key: ANDROID_KEY): Exec;
-    abstract fluentCommandExecutor(instruction: string, modelComposition: ModelCompositionBranch[], context: CommandExecutorContext): Promise<void>;
+    abstract fluentCommandExecutor(instruction: string, modelComposition: ModelCompositionBranch[], context: CommandExecutorContext, skipCache: boolean, retryStrategy?: RetryStrategy): Promise<void>;
 }
 export interface ExecutableGetter {
     exec(): Promise<DetectedElement[]>;