askui 0.25.1 → 0.27.0

package/dist/cjs/core/ai-element/ai-element-collection.d.ts

@@ -8,5 +8,6 @@ export declare class AIElementCollection {
     static collectAIElements(workspaceId: string | undefined, aiElementArgs: AIElementArgs): Promise<AIElementCollection>;
     getByName(name: string): CustomElementJson[];
     getByNames(names: string[]): CustomElementJson[];
+    getNames(): string[];
     private static CollectAiElementsFromLocation;
 }

package/dist/cjs/core/ai-element/ai-element-collection.js

@@ -73,6 +73,9 @@ class AIElementCollection {
         }
         return names.flatMap((name) => this.getByName(name));
     }
+    getNames() {
+        return [...new Set(this.elements.map((element) => element.name))];
+    }
     static CollectAiElementsFromLocation(aiElementLocation) {
         const files = fs_extra_1.default.readdirSync(aiElementLocation);
         if (files.length === 0) {

package/dist/cjs/core/models/anthropic/askui-agent.d.ts

@@ -1,3 +1,4 @@
+import { OsAgentHandler } from './tools/os-agent-tools';
 import { ClaudeAgent } from './claude-agent';
 import { ExecutionRuntime } from '../../../execution/execution-runtime';
 export declare class AskUIAgent extends ClaudeAgent {
@@ -6,6 +7,7 @@ export declare class AskUIAgent extends ClaudeAgent {
     constructor(executionRuntime: ExecutionRuntime);
     isConnected(): boolean;
     initializeOsAgentHandler(): Promise<void>;
+    getOsAgentHandler(): OsAgentHandler;
     configureAsDesktopAgent(): Promise<void>;
     configureAsAndroidAgent(): Promise<void>;
     private static DesktopSystemPrompt;

package/dist/cjs/core/models/anthropic/askui-agent.js

@@ -26,6 +26,12 @@ class AskUIAgent extends claude_agent_1.ClaudeAgent {
             this.osAgentHandler = yield os_agent_tools_1.OsAgentHandler.createInstance(this.executionRuntime);
         });
     }
+    getOsAgentHandler() {
+        if (!this.osAgentHandler) {
+            throw new Error('Agent OS client is not connected');
+        }
+        return this.osAgentHandler;
+    }
     configureAsDesktopAgent() {
         return __awaiter(this, void 0, void 0, function* () {
             if (!this.osAgentHandler) {
@@ -33,13 +39,19 @@ class AskUIAgent extends claude_agent_1.ClaudeAgent {
             }
             const tools = [
                 new os_agent_tools_1.AgentErrorTool(),
+                new os_agent_tools_1.PrintTool(),
                 new os_agent_tools_1.ScreenShotTool(this.osAgentHandler),
                 new os_agent_tools_1.MouseMoveTool(this.osAgentHandler),
                 new os_agent_tools_1.MouseClickTool(this.osAgentHandler),
                 new os_agent_tools_1.MouseScrollTool(this.osAgentHandler),
-                new os_agent_tools_1.DesktopKeyPressSequenceTool(this.osAgentHandler),
-                new os_agent_tools_1.DesktopSingleKeyPressTool(this.osAgentHandler),
                 new os_agent_tools_1.TypeTool(this.osAgentHandler),
+                new os_agent_tools_1.DesktopPressAndReleaseKeysTool(this.osAgentHandler),
+                new os_agent_tools_1.DesktopKeyHoldDownTool(this.osAgentHandler),
+                new os_agent_tools_1.DesktopKeyReleaseTool(this.osAgentHandler),
+                new os_agent_tools_1.MouseHoldLeftButtonDownTool(this.osAgentHandler),
+                new os_agent_tools_1.MouseReleaseLeftButtonTool(this.osAgentHandler),
+                new os_agent_tools_1.MouseDragAndDropTool(this.osAgentHandler),
+                new os_agent_tools_1.WaitTool(),
             ];
             this.setTools(tools);
             this.setSystemPrompt(AskUIAgent.DesktopSystemPrompt);
@@ -60,6 +72,7 @@ class AskUIAgent extends claude_agent_1.ClaudeAgent {
                 new os_agent_tools_1.AndroidSequenceKeyPressTool(this.osAgentHandler),
                 new os_agent_tools_1.TypeTool(this.osAgentHandler),
                 new os_agent_tools_1.ExecuteShellCommandTool(this.osAgentHandler),
+                new os_agent_tools_1.WaitTool(),
             ];
             this.setTools(tools);
             this.setSystemPrompt(AskUIAgent.AndroidSystemPrompt);
@@ -70,22 +83,13 @@ exports.AskUIAgent = AskUIAgent;
 AskUIAgent.DesktopSystemPrompt = `
 <SYSTEM_CAPABILITY>
 You are an autonomous AI assistant operating on a ${process.platform} machine with ${process.arch} architecture. You have full access to the system and internet connectivity.
-Your main goal is to mimic a human user interacting with a desktop computer. So you should try to use the tools in a way that a human would use a mouse and keyboard to interact with a computer.
-
-Key Capabilities:
-* Full system control through mouse and keyboard interactions
-* Screen capture and analysis
-* Web browser automation and navigation
-* File system access and manipulation
-* PDF document handling and text extraction
-* Error handling and recovery mechanisms
-
-Available Tools:
-* Mouse control (move, click, scroll)
-* Keyboard input (single keys, key combinations, typing)
-* Screen capture and analysis
-* Error reporting and recovery
-
+Your main goal is to mimic a human user interacting with a desktop computer.
+Use a mouse and keyboard to interact with a computer, and take screenshots.
+* This is an interface to a desktop GUI. You do not have access to a terminal or applications menu. You must click on desktop icons to start applications.
+* Some applications may take time to start or process actions, so you may need to wait and take successive screenshots to see the results of your actions. E.g. if you click on Firefox and a window doesn't open, try taking another screenshot.
+* Whenever you intend to move the cursor to click on an element like an icon, you should consult a screenshot to determine the coordinates of the element before moving the cursor.
+* If you tried clicking on a program or link but it failed to load, even after waiting, try adjusting your cursor position so that the tip of the cursor visually falls on the element that you want to click.
+* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges unless asked.
 Current Date: ${new Date().toUTCString()} UTC
 </SYSTEM_CAPABILITY>
 
@@ -134,20 +138,11 @@ AskUIAgent.AndroidSystemPrompt = `
 <SYSTEM_CAPABILITY>
 You are an autonomous AI assistant operating on an Android device via ADB. The host machine is ${process.platform} with ${process.arch} architecture and internet connectivity.
 Your main goal is to mimic a human user interacting with an Android device. So you should try to use the tools in a way that a human would use a touch screen to interact with an Android device.
-
-Key Capabilities:
-* Full Android device control through ADB
-* Screen capture and analysis
-* Touch input simulation
-* Android-specific key events
-* Error handling and recovery mechanisms
-
-Available Tools:
-* Touch control (click, swipe, scroll)
-* Android key events (single and sequence)
-* Screen capture and analysis
-* Error reporting and recovery
-
+Use a gestures and adb commands to interact with the android device, and take screenshots.
+* Some applications may take time to start or process actions, so you may need to wait and take successive screenshots to see the results of your actions. E.g. if you click on Firefox and a window doesn't open, try taking another screenshot.
+* Whenever you intend to move the cursor to click on an element like an icon, you should consult a screenshot to determine the coordinates of the element before moving the cursor.
+* If you tried clicking on a program or link but it failed to load, even after waiting, try adjusting your cursor position so that the tip of the cursor visually falls on the element that you want to click.
+* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges unless asked.
 Current Date: ${new Date().toUTCString()} UTC
 </SYSTEM_CAPABILITY>
 

package/dist/cjs/core/models/anthropic/claude-agent.d.ts

@@ -1,5 +1,6 @@
 import { Beta } from '@anthropic-ai/sdk/resources';
 import { BaseAgentTool } from './tools/base';
+import { BetaMessageParam } from '@anthropic-ai/sdk/resources/beta/messages';
 type PredictActResponseFunction = (params: {
     max_tokens: number;
     messages: Beta.BetaMessageParam[];
@@ -7,7 +8,16 @@ type PredictActResponseFunction = (params: {
     system?: string;
     tools?: any[];
     betas?: string[];
+    tool_choice?: {
+        type: 'tool' | 'any' | 'auto';
+        name?: string;
+    };
 }) => Promise<Beta.BetaMessage>;
+export type AgentHistory = BetaMessageParam[];
+export interface ActOptions {
+    chatId?: string;
+    agentHistory?: Beta.BetaMessageParam[];
+}
 export declare class ClaudeAgent {
     private predictActResponseFunction;
     private maxTokens;
@@ -19,7 +29,12 @@ export declare class ClaudeAgent {
     private _toolCollection;
     private tools;
     private history;
+    private toolChoice;
     constructor(predictActResponseFunction: PredictActResponseFunction);
+    setToolChoice(toolChoice: {
+        type: 'tool' | 'any' | 'auto';
+        name?: string;
+    }): void;
     setTools(tools: BaseAgentTool[]): void;
     addTool(tool: BaseAgentTool): void;
     listToolNames(): string[];
@@ -29,10 +44,7 @@ export declare class ClaudeAgent {
     private get toolCollection();
     private setHistory;
     private getHistory;
-    act(goal: string, options?: {
-        chatId?: string;
-        agentHistory?: Beta.BetaMessageParam[];
-    }): Promise<Beta.BetaMessageParam[]>;
+    act(goal: string, imagePathOrBase64String?: string, options?: ActOptions): Promise<Beta.BetaMessageParam[]>;
     private makeApiToolResult;
     private maybePrependSystemToolResult;
     private static filterNMostRecentImages;

package/dist/cjs/core/models/anthropic/claude-agent.js

@@ -12,6 +12,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.ClaudeAgent = void 0;
 const base_1 = require("./tools/base");
 const logger_1 = require("../../../lib/logger");
+const base_64_image_1 = require("../../../utils/base_64_image/base-64-image");
 class ClaudeAgent {
     constructor(predictActResponseFunction) {
         this.predictActResponseFunction = predictActResponseFunction;
@@ -19,11 +20,17 @@ class ClaudeAgent {
         this.onlyNMostRecentImages = 3;
         this.imageTruncationThreshold = 10;
         this.systemPrompt = '';
-        this.model = 'claude-3-5-sonnet-20241022';
-        this.betas = ['computer-use-2024-10-22'];
+        this.model = 'claude-sonnet-4-20250514';
+        this.betas = ['computer-use-2025-01-24'];
         this._toolCollection = undefined;
         this.tools = [];
         this.history = {};
+        this.toolChoice = {
+            type: 'auto',
+        };
+    }
+    setToolChoice(toolChoice) {
+        this.toolChoice = toolChoice;
     }
     setTools(tools) {
         this._toolCollection = undefined;
@@ -40,7 +47,14 @@ class ClaudeAgent {
         this.tools = this.tools.filter((tool) => tool.ToolName !== toolName);
     }
     setSystemPrompt(systemPrompt) {
-        this.systemPrompt = systemPrompt;
+        const enhancedPrompt = `${systemPrompt}
+        If you cannot complete a request due to safety concerns, please:
+        1. Explain what specific aspect is problematic
+        2. Suggest alternative approaches that would be acceptable
+        3. Provide partial assistance where possible within guidelines.
+        Raise an exception After you have provided the above information. include the error message in the exception.
+        `;
+        this.systemPrompt = enhancedPrompt;
     }
     IsConfigured() {
         return this.tools.length > 0 && this.systemPrompt !== '';
@@ -57,7 +71,7 @@ class ClaudeAgent {
     getHistory(key) {
         return this.history[key] || [];
     }
-    act(goal, options) {
+    act(goal, imagePathOrBase64String, options) {
         return __awaiter(this, void 0, void 0, function* () {
             if (!goal.trim()) {
                 throw new Error('Goal cannot be empty');
@@ -73,8 +87,24 @@ class ClaudeAgent {
                 messages.push(...this.getHistory(options.chatId));
             }
             // Add the new goal as a user message
+            const userContent = [{
+                    type: 'text',
+                    text: goal,
+                }];
+            if (imagePathOrBase64String !== undefined) {
+                const image = yield base_64_image_1.Base64Image.fromPathOrString(imagePathOrBase64String);
+                const imageString = image.toString(false);
+                userContent.push({
+                    type: 'image',
+                    source: {
+                        type: 'base64',
+                        media_type: 'image/png',
+                        data: imageString,
+                    },
+                });
+            }
             messages.push({
-                content: goal,
+                content: userContent,
                 role: 'user',
             });
             if (this.onlyNMostRecentImages) {
@@ -88,7 +118,15 @@ class ClaudeAgent {
                     system: this.systemPrompt,
                     tools: (new base_1.ToolCollection(this.tools).toParams()),
                     betas: this.betas,
+                    tool_choice: this.toolChoice,
                 });
+                if (response.stop_reason === 'refusal') {
+                    const refusalMessage = response.content
+                        .filter(block => block.type === 'text')
+                        .map(block => block.text)
+                        .join(' ');
+                    throw new Error(`Agent refused to answer: ${refusalMessage || 'The request violates agent\'s usage policies'}`);
+                }
                 messages.push({
                     content: response.content,
                     role: 'assistant',

package/dist/cjs/core/models/anthropic/index.d.ts

@@ -1,4 +1,3 @@
-import { BetaMessageParam } from '@anthropic-ai/sdk/resources/beta/messages';
-export type AgentHistory = BetaMessageParam[];
 export { AskUIAgent } from './askui-agent';
-export { ToolFailure, ToolError, BaseAgentTool } from './tools/base';
+export { ToolFailure, ToolError, BaseAgentTool, BetaTool, ToolResult, } from './tools/base';
+export { AgentHistory, ActOptions } from './claude-agent';

package/dist/cjs/core/models/anthropic/tools/askui-api-tools.d.ts

@@ -0,0 +1,19 @@
+import { DetectedElement } from '../../../model/annotation-result/detected-element';
+import { BaseAgentTool, ToolResult, BetaTool } from './base';
+import { OsAgentHandler } from './os-agent-tools';
+export declare class AskUIGetAskUIElementTool extends BaseAgentTool {
+    private osAgentHandler;
+    private locatorFunction;
+    private elementType;
+    constructor(osAgentHandler: OsAgentHandler, locatorFunction: (aiElementName: string) => Promise<DetectedElement[]>, elementType: string);
+    execute(params: {
+        elementName: string;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class AskUIListAIElementTool extends BaseAgentTool {
+    private listFunction;
+    constructor(listFunction: () => Promise<string[]>);
+    execute(): Promise<ToolResult>;
+    toParams(): BetaTool;
+}

package/dist/cjs/core/models/anthropic/tools/askui-api-tools.js

@@ -0,0 +1,81 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AskUIListAIElementTool = exports.AskUIGetAskUIElementTool = void 0;
+const base_1 = require("./base");
+class AskUIGetAskUIElementTool extends base_1.BaseAgentTool {
+    constructor(osAgentHandler, locatorFunction, elementType) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+        this.locatorFunction = locatorFunction;
+        this.elementType = elementType;
+    }
+    execute(params) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const { elementName } = params;
+            const detectedElements = yield this.locatorFunction(elementName);
+            const scaledElementsBoundingBoxes = detectedElements.map((element) => {
+                const xMid = (element.bndbox.xmin + element.bndbox.xmax) / 2;
+                const yMid = (element.bndbox.ymin + element.bndbox.ymax) / 2;
+                const [x, y] = this.osAgentHandler.scaleCoordinates('computer', xMid, yMid);
+                return {
+                    x, y,
+                };
+            });
+            return {
+                output: `Found ${scaledElementsBoundingBoxes.length} elements of type ${this.elementType}. center coordinates: ${JSON.stringify(scaledElementsBoundingBoxes)}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            description: `Locates and retrieves the bounding box coordinates of AskUI ${this.elementType} elements on the screen. ` +
+                `This tool is essential for UI automation as it provides the exact pixel coordinates needed to interact with UI elements. ` +
+                `The coordinates returned  can be used for clicking, hovering, or other mouse interactions. ` +
+                `Use this tool when you need to find and interact with specific ${this.elementType} UI elements by their semantic names.`,
+            input_schema: {
+                properties: {
+                    elementName: {
+                        type: 'string',
+                        description: `The semantic name or identifier of the ${this.elementType} element to locate on the screen. `
+                    },
+                },
+                required: ['elementName'],
+                type: 'object',
+            },
+            name: `get_askui_${this.elementType}_element_tool`,
+        };
+    }
+}
+exports.AskUIGetAskUIElementTool = AskUIGetAskUIElementTool;
+class AskUIListAIElementTool extends base_1.BaseAgentTool {
+    constructor(listFunction) {
+        super();
+        this.listFunction = listFunction;
+    }
+    execute() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const elementNames = yield this.listFunction();
+            return {
+                output: `Found ${elementNames.length} element names that can be used to retrieve bounding boxes. Names: ${JSON.stringify(elementNames)}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            description: 'Retrieves a comprehensive list of all valid AskUI AI element names that can be used for element location and interaction. ' +
+                'The returned names can be used as input for the get_askui_aiElement_element_tool to locate specific ai elements. ',
+            input_schema: { type: 'object', properties: {}, required: [] },
+            name: 'list_ai_element_names_tool',
+        };
+    }
+}
+exports.AskUIListAIElementTool = AskUIListAIElementTool;

package/dist/cjs/core/models/anthropic/tools/base.d.ts

@@ -1,3 +1,5 @@
+import { BetaTool as AnthropicBetaTool } from '@anthropic-ai/sdk/resources/beta/messages/messages';
+export type BetaTool = AnthropicBetaTool;
 export interface ToolResult {
     output?: string;
     error?: string;

package/dist/cjs/core/models/anthropic/tools/os-agent-tools.d.ts

@@ -5,21 +5,39 @@ import { ExecutionRuntime } from '../../../../execution/execution-runtime';
 import { ControlCommand } from '../../../ui-control-commands';
 export declare class OsAgentHandler {
     private AgentOsClient;
-    private TargetResolution;
+    private targetResolution;
     private screenDimensions;
+    private paddingInfo;
     constructor(AgentOsClient: ExecutionRuntime, screenDimensions: {
         width: number;
         height: number;
     });
+    private updatePaddingInfo;
     static createInstance(AgentOsClient: ExecutionRuntime): Promise<OsAgentHandler>;
     getTargetResolution(): {
         width: number;
         height: number;
     };
+    getScreenDimensions(): {
+        width: number;
+        height: number;
+    };
     setTargetResolution(width: number, height: number): void;
     takeScreenshot(): Promise<string>;
-    private scaleCoordinates;
+    scaleCoordinates(source: 'api' | 'computer', x: number, y: number): [number, number];
     requestControl(controlCommand: ControlCommand): Promise<void>;
+    mouseMove(x: number, y: number): Promise<void>;
+    mouseClick(button: "left" | "right" | "middle", doubleClick: boolean): Promise<void>;
+    mouseScroll(dx: number, dy: number): Promise<void>;
+    mouseHoldLeftButtonDown(): Promise<void>;
+    mouseReleaseLeftButton(): Promise<void>;
+    desktopKeyPressAndRelease(key: PC_AND_MODIFIER_KEY, modifiers?: MODIFIER_KEY[]): Promise<void>;
+    desktopKeyHoldDown(key: PC_AND_MODIFIER_KEY, modifiers?: MODIFIER_KEY[]): Promise<void>;
+    desktopKeyRelease(key: PC_AND_MODIFIER_KEY, modifiers?: MODIFIER_KEY[]): Promise<void>;
+    typeText(text: string): Promise<void>;
+    androidKeyPress(key: ANDROID_KEY): Promise<void>;
+    androidKeySequencePress(keys: ANDROID_KEY[]): Promise<void>;
+    executeShellCommand(command: string): Promise<void>;
 }
 export declare class ScreenShotTool extends BaseAgentTool {
     private osAgentHandler;
@@ -54,21 +72,53 @@ export declare class MouseScrollTool extends BaseAgentTool {
     }): Promise<ToolResult>;
     toParams(): BetaTool;
 }
-export declare class DesktopKeyPressSequenceTool extends BaseAgentTool {
+export declare class MouseDragAndDropTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(command: {
+        startX: number;
+        startY: number;
+        endX: number;
+        endY: number;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class MouseHoldLeftButtonDownTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class MouseReleaseLeftButtonTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class DesktopPressAndReleaseKeysTool extends BaseAgentTool {
     private osAgentHandler;
     constructor(osAgentHandler: OsAgentHandler);
     execute(command: {
         key: PC_KEY;
-        firstModifier?: MODIFIER_KEY;
-        secondModifier?: MODIFIER_KEY;
+        modifiers?: MODIFIER_KEY[];
     }): Promise<ToolResult>;
     toParams(): BetaTool;
 }
-export declare class DesktopSingleKeyPressTool extends BaseAgentTool {
+export declare class DesktopKeyHoldDownTool extends BaseAgentTool {
     private osAgentHandler;
     constructor(osAgentHandler: OsAgentHandler);
     execute(command: {
         key: PC_AND_MODIFIER_KEY;
+        modifiers?: MODIFIER_KEY[];
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class DesktopKeyReleaseTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(command: {
+        key: PC_AND_MODIFIER_KEY;
+        modifiers?: MODIFIER_KEY[];
     }): Promise<ToolResult>;
     toParams(): BetaTool;
 }
@@ -111,3 +161,17 @@ export declare class ExecuteShellCommandTool extends BaseAgentTool {
     }): Promise<ToolResult>;
     toParams(): BetaTool;
 }
+export declare class WaitTool extends BaseAgentTool {
+    constructor();
+    execute(command: {
+        milliseconds: number;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class PrintTool extends BaseAgentTool {
+    constructor();
+    execute(command: {
+        text: string;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}