askui 0.29.0 → 0.31.0

package/dist/esm/core/cache/cache-manager.js

@@ -0,0 +1,141 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { ControlCommandCode } from '../ui-control-commands';
+import { InputEvent } from '../ui-control-commands/input-event';
+import { Base64Image } from '../../utils/base_64_image/base-64-image';
+import { CacheEntry } from './cache-entry';
+import { ImageReference } from './image-reference';
+import { CacheEntryReference } from './cache-entry-reference';
+import { CacheFile } from './cahe-file';
+export class CacheManager {
+    constructor(config) {
+        var _a;
+        this.relevantData = {};
+        this.referenceWidth = 32;
+        this.referenceHeight = 16;
+        this.defaultValidationType = 'PixelPerfect';
+        this.validationType = (_a = config.validationType) !== null && _a !== void 0 ? _a : this.defaultValidationType;
+        this.cacheFile = new CacheFile(config.cacheFilePath);
+    }
+    loadFromFile() {
+        this.cacheFile.loadFromFile();
+        this.relevantData = this.cacheFile.getDataForValidationType(this.validationType);
+    }
+    saveToFile() {
+        this.cacheFile.saveToFile(this.validationType, this.relevantData);
+    }
+    add(key, value) {
+        if (this.relevantData[key] === undefined) {
+            this.relevantData[key] = [];
+        }
+        this.relevantData[key].push(value);
+    }
+    addCacheEntryFromControlCommand(instruction, controlCommand, customElements, image) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let imageReference;
+            if (image !== undefined) {
+                const moveAction = controlCommand.actions.find((action) => action.inputEvent === InputEvent.MOUSE_MOVE);
+                const x = moveAction === null || moveAction === void 0 ? void 0 : moveAction.position.x;
+                const y = moveAction === null || moveAction === void 0 ? void 0 : moveAction.position.y;
+                if (x !== undefined && y !== undefined) {
+                    const xTopLeft = Math.max(0, (x - this.referenceWidth / 2));
+                    const yTopLeft = Math.max(0, (y - this.referenceHeight / 2));
+                    const base64Image = yield Base64Image.fromString(image);
+                    const croppedImage = yield base64Image.cropRegion(xTopLeft, yTopLeft, this.referenceWidth, this.referenceHeight);
+                    imageReference = new ImageReference(croppedImage.toString(true), this.referenceWidth, this.referenceHeight, xTopLeft, yTopLeft);
+                }
+            }
+            const cacheEntry = new CacheEntry(image === undefined, controlCommand, new CacheEntryReference(imageReference));
+            const cacheKey = this.encodeKey(instruction, customElements);
+            this.add(cacheKey, cacheEntry);
+        });
+    }
+    isImageRequired(instruction, customElements) {
+        const cacheKey = this.encodeKey(instruction, customElements);
+        const cacheEntries = this.getCacheEntriesByKey(cacheKey);
+        if (cacheEntries.length === 0) {
+            return undefined;
+        }
+        return cacheEntries.find((entry) => entry.alwaysValid === false) !== undefined;
+    }
+    getCachedControlCommand(instruction, customElements, image) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const cacheKey = this.encodeKey(instruction, customElements);
+            const cacheEntry = yield this.getValidCacheEntry(cacheKey, image);
+            if (cacheEntry === undefined || cacheEntry.controlCommand.code !== ControlCommandCode.OK) {
+                return undefined;
+            }
+            return cacheEntry.controlCommand;
+        });
+    }
+    // eslint-disable-next-line class-methods-use-this
+    encodeKey(instruction, customElements) {
+        let key = instruction;
+        if (customElements.length > 0) {
+            key += ` with CustomElements:${customElements.map((element, index) => `${index}:${element.asString()}`).join(',')}`;
+        }
+        return key;
+    }
+    getCacheEntriesByKey(key) {
+        if (this.relevantData[key] === undefined) {
+            return [];
+        }
+        const entries = this.relevantData[key];
+        if (entries === undefined) {
+            return [];
+        }
+        return entries;
+    }
+    getValidCacheEntry(key, screenshot) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const cacheEntries = this.getCacheEntriesByKey(key);
+            if (cacheEntries.length === 0) {
+                return undefined;
+            }
+            if (this.validationType === 'PixelPerfect') {
+                /* eslint-disable no-restricted-syntax, no-await-in-loop */
+                for (const cacheEntry of cacheEntries) {
+                    if (cacheEntry.alwaysValid) {
+                        return cacheEntry;
+                    }
+                    const isValid = yield this.validateAccordingToPixelPerfect(cacheEntry, screenshot);
+                    if (isValid) {
+                        /* eslint-enable no-restricted-syntax, no-await-in-loop */
+                        return cacheEntry;
+                    }
+                }
+                /* eslint-enable no-restricted-syntax, no-await-in-loop */
+            }
+            return undefined;
+        });
+    }
+    // eslint-disable-next-line class-methods-use-this
+    validateAccordingToPixelPerfect(cacheEntry, screenshot) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            if (screenshot === undefined) {
+                return false;
+            }
+            if (((_a = cacheEntry.reference) === null || _a === void 0 ? void 0 : _a.image) === undefined) {
+                return false;
+            }
+            try {
+                const referenceImage = yield Base64Image.fromString(screenshot);
+                const croppedScreenshot = yield referenceImage.cropRegion(cacheEntry.reference.image.xTopLeft, cacheEntry.reference.image.yTopLeft, cacheEntry.reference.image.width, cacheEntry.reference.image.height);
+                const croppedReferenceImageString = croppedScreenshot.toString(true);
+                const isValid = croppedReferenceImageString === cacheEntry.reference.image.base64Image;
+                return isValid;
+            }
+            catch (error) {
+                return false;
+            }
+        });
+    }
+}

package/dist/esm/core/cache/cahe-file.d.ts

@@ -0,0 +1,19 @@
+import { CacheEntry } from './cache-entry';
+import { ValidationType } from './cache-config';
+export type CacheDataByInstruction = Record<string, CacheEntry[]>;
+export type CacheDataByValidationType = Record<string, CacheDataByInstruction>;
+export declare class CacheFile {
+    private createAt;
+    private version;
+    private type;
+    private data;
+    private filePath;
+    constructor(filePath?: string);
+    loadFromFileVersion1(json: CacheFile): void;
+    loadFromFile(): void;
+    getDataForValidationType(validationType: ValidationType): CacheDataByInstruction;
+    getData(): CacheDataByValidationType;
+    saveToFile(validationType: ValidationType, relevantData: CacheDataByInstruction): void;
+    private setRelevantData;
+    private asJsonObject;
+}

package/dist/esm/core/cache/cahe-file.js

@@ -0,0 +1,121 @@
+import fs from 'fs';
+import path from 'path';
+import { logger } from '../../lib';
+import { CacheEntry } from './cache-entry';
+export class CacheFile {
+    constructor(filePath) {
+        this.createAt = new Date();
+        this.version = 1;
+        this.type = 'AskUI-Cache';
+        this.data = {};
+        this.filePath = filePath;
+    }
+    loadFromFileVersion1(json) {
+        if (json.version !== 1) {
+            throw new Error(`Unsupported key 'version' in the cache file: '${json.version}'. Only version 1 is supported.`);
+        }
+        if (json.data === undefined) {
+            throw new Error("Key 'data' is required in the cache file.");
+        }
+        if (json.createAt === undefined) {
+            throw new Error("Key 'createAt' is required in the cache file.");
+        }
+        if (json.version === undefined) {
+            throw new Error("Key 'version' is required in the cache file.");
+        }
+        if (json.type !== 'AskUI-Cache') {
+            throw new Error(`Unsupported key 'type' in the cache file: '${json.type}'. Only 'AskUI-Cache' is supported.`);
+        }
+        this.createAt = new Date(json.createAt);
+        this.version = json.version;
+        this.type = json.type;
+        this.data = Object.keys(json.data).reduce((acc, validationType) => {
+            var _a, _b;
+            const nextAcc = Object.assign({}, acc);
+            const validationEntries = (_b = (_a = json.data) === null || _a === void 0 ? void 0 : _a[validationType]) !== null && _b !== void 0 ? _b : {};
+            nextAcc[validationType] = Object.keys(validationEntries).reduce((innerAcc, instruction) => {
+                var _a;
+                const nextInnerAcc = Object.assign({}, innerAcc);
+                const rawEntries = (_a = validationEntries[instruction]) !== null && _a !== void 0 ? _a : [];
+                nextInnerAcc[instruction] = rawEntries
+                    .map((entry) => CacheEntry.fromJson(entry))
+                    .filter((entry) => entry !== undefined);
+                return nextInnerAcc;
+            }, {});
+            return nextAcc;
+        }, {});
+    }
+    loadFromFile() {
+        if (this.filePath === undefined) {
+            logger.debug('The cache file path is not set. Skipping the cache load.');
+            return;
+        }
+        logger.debug(`Loading the cache from file '${this.filePath}'.`);
+        if (!fs.existsSync(this.filePath)) {
+            logger.warn(`The cache file does not exist: '${this.filePath}'. Skipping the cache load.`);
+            return;
+        }
+        const data = fs.readFileSync(this.filePath, 'utf8');
+        const json = JSON.parse(data);
+        if (json.version !== 1) {
+            throw new Error(`Unsupported key 'version' in the cache file: '${json.version}'. Only version 1 is supported.`);
+        }
+        this.loadFromFileVersion1(json);
+    }
+    getDataForValidationType(validationType) {
+        var _a;
+        return (_a = this.data[validationType]) !== null && _a !== void 0 ? _a : {};
+    }
+    getData() {
+        return this.data;
+    }
+    saveToFile(validationType, relevantData) {
+        if (this.filePath === undefined) {
+            logger.debug('The cache file path is not set. Skipping the cache save.');
+            return;
+        }
+        try {
+            logger.debug(`Saving the cache to file '${this.filePath}'.`);
+            this.setRelevantData(validationType, relevantData);
+            const jsonObject = this.asJsonObject();
+            const jsonString = JSON.stringify(jsonObject, null, 2);
+            // create the directory if it doesn't exist
+            if (!fs.existsSync(path.dirname(this.filePath))) {
+                fs.mkdirSync(path.dirname(this.filePath), { recursive: true });
+            }
+            fs.writeFileSync(this.filePath, jsonString, 'utf8');
+            logger.info(`The cache was saved successfully to '${this.filePath}'.`);
+        }
+        catch (error) {
+            logger.error(`An error occurred while saving the cache to file '${this.filePath}': '${error}'`);
+            throw error;
+        }
+    }
+    setRelevantData(validationType, relevantData) {
+        this.data[validationType] = relevantData;
+    }
+    asJsonObject() {
+        return {
+            createAt: this.createAt.toISOString(),
+            data: Object.keys(this.data).reduce((acc, validationType) => {
+                const validationData = this.data[validationType];
+                if (validationData === undefined) {
+                    return acc;
+                }
+                const nextAcc = Object.assign({}, acc);
+                nextAcc[validationType] = Object.keys(validationData).reduce((innerAcc, instruction) => {
+                    const entries = validationData[instruction];
+                    if (entries !== undefined) {
+                        const nextInnerAcc = Object.assign({}, innerAcc);
+                        nextInnerAcc[instruction] = entries;
+                        return nextInnerAcc;
+                    }
+                    return innerAcc;
+                }, {});
+                return nextAcc;
+            }, {}),
+            type: this.type,
+            version: this.version,
+        };
+    }
+}

package/dist/esm/core/cache/dummy-cache-manager.d.ts

@@ -0,0 +1,12 @@
+import { CacheInterface } from './cache-interface';
+import { CacheEntry } from './cache-entry';
+import { ControlCommand } from '../ui-control-commands';
+import { CustomElement } from '../model/custom-element';
+export declare class DummyCacheManager implements CacheInterface {
+    add(_key: string, _value: CacheEntry): void;
+    addCacheEntryFromControlCommand(_instruction: string, _controlCommand: ControlCommand, _customElements: CustomElement[], _image?: string): Promise<void>;
+    isImageRequired(_instruction: string, _customElements: CustomElement[]): boolean | undefined;
+    getCachedControlCommand(_instruction: string, _customElements: CustomElement[], _image?: string): Promise<ControlCommand | undefined>;
+    loadFromFile(): void;
+    saveToFile(): void;
+}

package/dist/esm/core/cache/dummy-cache-manager.js

@@ -0,0 +1,23 @@
+export class DummyCacheManager {
+    // eslint-disable-next-line class-methods-use-this
+    add(_key, _value) {
+    }
+    // eslint-disable-next-line class-methods-use-this
+    addCacheEntryFromControlCommand(_instruction, _controlCommand, _customElements, _image) {
+        return Promise.resolve();
+    }
+    // eslint-disable-next-line class-methods-use-this
+    isImageRequired(_instruction, _customElements) {
+        return undefined;
+    }
+    // eslint-disable-next-line class-methods-use-this
+    getCachedControlCommand(_instruction, _customElements, _image) {
+        return Promise.resolve(undefined);
+    }
+    // eslint-disable-next-line class-methods-use-this
+    loadFromFile() {
+    }
+    // eslint-disable-next-line class-methods-use-this
+    saveToFile() {
+    }
+}

package/dist/esm/core/cache/image-reference.d.ts

@@ -0,0 +1,10 @@
+export declare class ImageReference {
+    base64Image: string;
+    width: number;
+    height: number;
+    xTopLeft: number;
+    yTopLeft: number;
+    constructor(base64Image: string, width: number, height: number, xTopLeft: number, yTopLeft: number);
+    static fromJson(json: ImageReference): ImageReference;
+    toJson(): Record<string, unknown>;
+}

package/dist/esm/core/cache/image-reference.js

@@ -0,0 +1,36 @@
+export class ImageReference {
+    constructor(base64Image, width, height, xTopLeft, yTopLeft) {
+        this.base64Image = base64Image;
+        this.width = width;
+        this.height = height;
+        this.xTopLeft = xTopLeft;
+        this.yTopLeft = yTopLeft;
+    }
+    static fromJson(json) {
+        if (json.base64Image === undefined) {
+            throw new Error('the key "base64Image" is required');
+        }
+        if (json.width === undefined) {
+            throw new Error('the key "width" is required');
+        }
+        if (json.height === undefined) {
+            throw new Error('the key "height" is required');
+        }
+        if (json.xTopLeft === undefined) {
+            throw new Error('the key "xTopLeft" is required');
+        }
+        if (json.yTopLeft === undefined) {
+            throw new Error('the key "yTopLeft" is required');
+        }
+        return new ImageReference(json.base64Image, json.width, json.height, json.xTopLeft, json.yTopLeft);
+    }
+    toJson() {
+        return {
+            base64Image: this.base64Image,
+            height: this.height,
+            width: this.width,
+            xTopLeft: this.xTopLeft,
+            yTopLeft: this.yTopLeft,
+        };
+    }
+}

package/dist/esm/core/cache/index.d.ts

@@ -0,0 +1,7 @@
+export { ImageReference } from './image-reference';
+export { CacheEntryReference } from './cache-entry-reference';
+export { CacheEntry } from './cache-entry';
+export { CacheConfig, ValidationType } from './cache-config';
+export { CacheInterface } from './cache-interface';
+export { DummyCacheManager } from './dummy-cache-manager';
+export { CacheManager } from './cache-manager';

package/dist/esm/core/cache/index.js

@@ -0,0 +1,5 @@
+export { ImageReference } from './image-reference';
+export { CacheEntryReference } from './cache-entry-reference';
+export { CacheEntry } from './cache-entry';
+export { DummyCacheManager } from './dummy-cache-manager';
+export { CacheManager } from './cache-manager';

package/dist/esm/core/model/annotation-result/boundary-box.js

@@ -17,7 +17,7 @@ export class BoundingBox {
         this.ymax = ymax;
     }
     static fromJson(boundinBox, resizeRatio = 1) {
-        return new BoundingBox(boundinBox.xmin * resizeRatio, boundinBox.ymin * resizeRatio, boundinBox.xmax * resizeRatio, boundinBox.ymax * resizeRatio);
+        return new BoundingBox(Math.round(boundinBox.xmin * resizeRatio), Math.round(boundinBox.ymin * resizeRatio), Math.round(boundinBox.xmax * resizeRatio), Math.round(boundinBox.ymax * resizeRatio));
     }
     /**
     *

package/dist/esm/core/model/custom-element.d.ts

@@ -19,4 +19,5 @@ export declare class CustomElement implements CustomElementJson {
     static fromJsonWithImagePathOrImage(ceJson: CustomElementJson): Promise<CustomElement>;
     static fromJson(ceJson: CustomElementJson): CustomElement;
     validate(): void;
+    asString(): string;
 }

package/dist/esm/core/model/custom-element.js

@@ -43,6 +43,9 @@ export class CustomElement {
             throw new ValidationError(e.errors.join(', '));
         }
     }
+    asString() {
+        return `name:${this.name}-customImage:${this.customImage}-threshold:${this.threshold}-stopThreshold:${this.stopThreshold}-rotationDegreePerStep:${this.rotationDegreePerStep}-imageCompareFormat:${this.imageCompareFormat}-mask:${this.mask}`;
+    }
 }
 CustomElement.schema = object({
     mask: array().optional().min(3, 'mask must contain at least 3 points'),

package/dist/esm/core/models/anthropic/askui-agent.d.ts

@@ -4,12 +4,12 @@ import { ExecutionRuntime } from '../../../execution/execution-runtime';
 export declare class AskUIAgent extends ClaudeAgent {
     private osAgentHandler;
     private executionRuntime;
+    private runtime;
     constructor(executionRuntime: ExecutionRuntime);
     isConnected(): boolean;
     initializeOsAgentHandler(): Promise<void>;
     getOsAgentHandler(): OsAgentHandler;
-    configureAsDesktopAgent(): Promise<void>;
-    configureAsAndroidAgent(): Promise<void>;
+    configureAgent(): Promise<void>;
     private static DesktopSystemPrompt;
     private static AndroidSystemPrompt;
 }

package/dist/esm/core/models/anthropic/askui-agent.js

@@ -7,12 +7,13 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-import { DesktopPressAndReleaseKeysTool, MouseClickTool, MouseMoveTool, MouseScrollTool, OsAgentHandler, ScreenShotTool, TypeTool, AgentErrorTool, AndroidSequenceKeyPressTool, AndroidSingleKeyPressTool, ExecuteShellCommandTool, DesktopKeyHoldDownTool, DesktopKeyReleaseTool, MouseReleaseLeftButtonTool, MouseHoldLeftButtonDownTool, MouseDragAndDropTool, WaitTool, PrintTool, } from './tools/os-agent-tools';
+import { DesktopPressAndReleaseKeysTool, MouseClickTool, MouseMoveTool, MouseScrollTool, OsAgentHandler, ScreenShotTool, TypeTool, AgentErrorTool, AndroidSequenceKeyPressTool, AndroidSingleKeyPressTool, ExecuteShellCommandTool, DesktopKeyHoldDownTool, DesktopKeyReleaseTool, MouseReleaseLeftButtonTool, MouseHoldLeftButtonDownTool, MouseDragAndDropTool, WaitTool, PrintTool, AndroidSwipeTool, AndroidDragAndDropTool, AndroidTapTool, AndroidShellCommandTool, } from './tools/os-agent-tools';
 import { ClaudeAgent } from './claude-agent';
 export class AskUIAgent extends ClaudeAgent {
     constructor(executionRuntime) {
         super((params) => executionRuntime.predictActResponse(params));
         this.osAgentHandler = undefined;
+        this.runtime = 'desktop';
         this.executionRuntime = executionRuntime;
     }
     isConnected() {
@@ -21,6 +22,7 @@ export class AskUIAgent extends ClaudeAgent {
     initializeOsAgentHandler() {
         return __awaiter(this, void 0, void 0, function* () {
             this.osAgentHandler = yield OsAgentHandler.createInstance(this.executionRuntime);
+            this.runtime = this.osAgentHandler.runtime;
         });
     }
     getOsAgentHandler() {
@@ -29,50 +31,48 @@ export class AskUIAgent extends ClaudeAgent {
         }
         return this.osAgentHandler;
     }
-    configureAsDesktopAgent() {
+    configureAgent() {
         return __awaiter(this, void 0, void 0, function* () {
             if (!this.osAgentHandler) {
                 throw new Error('Agent OS client is not connected');
             }
-            const tools = [
+            let systemPrompt = AskUIAgent.DesktopSystemPrompt;
+            let tools = [
                 new AgentErrorTool(),
                 new PrintTool(),
-                new ScreenShotTool(this.osAgentHandler),
-                new MouseMoveTool(this.osAgentHandler),
-                new MouseClickTool(this.osAgentHandler),
-                new MouseScrollTool(this.osAgentHandler),
-                new TypeTool(this.osAgentHandler),
-                new DesktopPressAndReleaseKeysTool(this.osAgentHandler),
-                new DesktopKeyHoldDownTool(this.osAgentHandler),
-                new DesktopKeyReleaseTool(this.osAgentHandler),
-                new MouseHoldLeftButtonDownTool(this.osAgentHandler),
-                new MouseReleaseLeftButtonTool(this.osAgentHandler),
-                new MouseDragAndDropTool(this.osAgentHandler),
                 new WaitTool(),
-            ];
-            this.setTools(tools);
-            this.setSystemPrompt(AskUIAgent.DesktopSystemPrompt);
-        });
-    }
-    configureAsAndroidAgent() {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.osAgentHandler) {
-                throw new Error('Agent OS client is not connected');
-            }
-            const tools = [
-                new AgentErrorTool(),
                 new ScreenShotTool(this.osAgentHandler),
-                new MouseMoveTool(this.osAgentHandler),
-                new MouseClickTool(this.osAgentHandler),
-                new MouseScrollTool(this.osAgentHandler),
-                new AndroidSingleKeyPressTool(this.osAgentHandler),
-                new AndroidSequenceKeyPressTool(this.osAgentHandler),
                 new TypeTool(this.osAgentHandler),
-                new ExecuteShellCommandTool(this.osAgentHandler),
-                new WaitTool(),
             ];
+            if (this.runtime === 'desktop') {
+                tools = [
+                    ...tools,
+                    new MouseMoveTool(this.osAgentHandler),
+                    new MouseClickTool(this.osAgentHandler),
+                    new MouseScrollTool(this.osAgentHandler),
+                    new DesktopPressAndReleaseKeysTool(this.osAgentHandler),
+                    new DesktopKeyHoldDownTool(this.osAgentHandler),
+                    new DesktopKeyReleaseTool(this.osAgentHandler),
+                    new MouseHoldLeftButtonDownTool(this.osAgentHandler),
+                    new MouseReleaseLeftButtonTool(this.osAgentHandler),
+                    new MouseDragAndDropTool(this.osAgentHandler),
+                    new ExecuteShellCommandTool(this.osAgentHandler),
+                ];
+            }
+            if (this.runtime === 'android') {
+                tools = [
+                    ...tools,
+                    new AndroidSingleKeyPressTool(this.osAgentHandler),
+                    new AndroidSequenceKeyPressTool(this.osAgentHandler),
+                    new AndroidSwipeTool(this.osAgentHandler),
+                    new AndroidDragAndDropTool(this.osAgentHandler),
+                    new AndroidTapTool(this.osAgentHandler),
+                    new AndroidShellCommandTool(this.osAgentHandler),
+                ];
+                systemPrompt = AskUIAgent.AndroidSystemPrompt;
+            }
             this.setTools(tools);
-            this.setSystemPrompt(AskUIAgent.AndroidSystemPrompt);
+            this.setSystemPrompt(systemPrompt);
         });
     }
 }

package/dist/esm/core/models/anthropic/tools/os-agent-tools.d.ts

@@ -5,13 +5,14 @@ import { ExecutionRuntime } from '../../../../execution/execution-runtime';
 import { ControlCommand } from '../../../ui-control-commands';
 export declare class OsAgentHandler {
     private AgentOsClient;
-    private targetResolution;
     private screenDimensions;
+    runtime: 'android' | 'desktop';
+    private targetResolution;
     private paddingInfo;
     constructor(AgentOsClient: ExecutionRuntime, screenDimensions: {
         width: number;
         height: number;
-    });
+    }, runtime: 'android' | 'desktop');
     private updatePaddingInfo;
     static createInstance(AgentOsClient: ExecutionRuntime): Promise<OsAgentHandler>;
     getTargetResolution(): {
@@ -27,7 +28,7 @@ export declare class OsAgentHandler {
     scaleCoordinates(source: 'api' | 'computer', x: number, y: number): [number, number];
     requestControl(controlCommand: ControlCommand): Promise<void>;
     mouseMove(x: number, y: number): Promise<void>;
-    mouseClick(button: "left" | "right" | "middle", doubleClick: boolean): Promise<void>;
+    mouseClick(button: 'left' | 'right' | 'middle', doubleClick: boolean): Promise<void>;
     mouseScroll(dx: number, dy: number): Promise<void>;
     mouseHoldLeftButtonDown(): Promise<void>;
     mouseReleaseLeftButton(): Promise<void>;
@@ -38,6 +39,10 @@ export declare class OsAgentHandler {
     androidKeyPress(key: ANDROID_KEY): Promise<void>;
     androidKeySequencePress(keys: ANDROID_KEY[]): Promise<void>;
     executeShellCommand(command: string): Promise<void>;
+    AndroidSwipeTool(startX: number, startY: number, endX: number, endY: number): Promise<void>;
+    AndroidDragAndDropTool(startX: number, startY: number, endX: number, endY: number): Promise<void>;
+    AndroidTapTool(x: number, y: number): Promise<void>;
+    executeAndroidShellCommand(command: string): Promise<void>;
 }
 export declare class ScreenShotTool extends BaseAgentTool {
     private osAgentHandler;
@@ -175,3 +180,42 @@ export declare class PrintTool extends BaseAgentTool {
     }): Promise<ToolResult>;
     toParams(): BetaTool;
 }
+export declare class AndroidSwipeTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(command: {
+        startX: number;
+        startY: number;
+        endX: number;
+        endY: number;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class AndroidDragAndDropTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(command: {
+        startX: number;
+        startY: number;
+        endX: number;
+        endY: number;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class AndroidTapTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(command: {
+        x: number;
+        y: number;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class AndroidShellCommandTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(command: {
+        command: string;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}