askui 0.25.1 → 0.26.0

package/dist/cjs/core/models/anthropic/askui-agent.js

@@ -37,9 +37,14 @@ class AskUIAgent extends claude_agent_1.ClaudeAgent {
                 new os_agent_tools_1.MouseMoveTool(this.osAgentHandler),
                 new os_agent_tools_1.MouseClickTool(this.osAgentHandler),
                 new os_agent_tools_1.MouseScrollTool(this.osAgentHandler),
-                new os_agent_tools_1.DesktopKeyPressSequenceTool(this.osAgentHandler),
-                new os_agent_tools_1.DesktopSingleKeyPressTool(this.osAgentHandler),
                 new os_agent_tools_1.TypeTool(this.osAgentHandler),
+                new os_agent_tools_1.DesktopPressAndReleaseKeysTool(this.osAgentHandler),
+                new os_agent_tools_1.DesktopKeyHoldDownTool(this.osAgentHandler),
+                new os_agent_tools_1.DesktopKeyReleaseTool(this.osAgentHandler),
+                new os_agent_tools_1.MouseHoldLeftButtonDownTool(this.osAgentHandler),
+                new os_agent_tools_1.MouseReleaseLeftButtonTool(this.osAgentHandler),
+                new os_agent_tools_1.MouseDragAndDropTool(this.osAgentHandler),
+                new os_agent_tools_1.WaitTool(),
             ];
             this.setTools(tools);
             this.setSystemPrompt(AskUIAgent.DesktopSystemPrompt);
@@ -60,6 +65,7 @@ class AskUIAgent extends claude_agent_1.ClaudeAgent {
                 new os_agent_tools_1.AndroidSequenceKeyPressTool(this.osAgentHandler),
                 new os_agent_tools_1.TypeTool(this.osAgentHandler),
                 new os_agent_tools_1.ExecuteShellCommandTool(this.osAgentHandler),
+                new os_agent_tools_1.WaitTool(),
             ];
             this.setTools(tools);
             this.setSystemPrompt(AskUIAgent.AndroidSystemPrompt);
@@ -70,22 +76,13 @@ exports.AskUIAgent = AskUIAgent;
 AskUIAgent.DesktopSystemPrompt = `
 <SYSTEM_CAPABILITY>
 You are an autonomous AI assistant operating on a ${process.platform} machine with ${process.arch} architecture. You have full access to the system and internet connectivity.
-Your main goal is to mimic a human user interacting with a desktop computer. So you should try to use the tools in a way that a human would use a mouse and keyboard to interact with a computer.
-
-Key Capabilities:
-* Full system control through mouse and keyboard interactions
-* Screen capture and analysis
-* Web browser automation and navigation
-* File system access and manipulation
-* PDF document handling and text extraction
-* Error handling and recovery mechanisms
-
-Available Tools:
-* Mouse control (move, click, scroll)
-* Keyboard input (single keys, key combinations, typing)
-* Screen capture and analysis
-* Error reporting and recovery
-
+Your main goal is to mimic a human user interacting with a desktop computer.
+Use a mouse and keyboard to interact with a computer, and take screenshots.
+* This is an interface to a desktop GUI. You do not have access to a terminal or applications menu. You must click on desktop icons to start applications.
+* Some applications may take time to start or process actions, so you may need to wait and take successive screenshots to see the results of your actions. E.g. if you click on Firefox and a window doesn't open, try taking another screenshot.
+* Whenever you intend to move the cursor to click on an element like an icon, you should consult a screenshot to determine the coordinates of the element before moving the cursor.
+* If you tried clicking on a program or link but it failed to load, even after waiting, try adjusting your cursor position so that the tip of the cursor visually falls on the element that you want to click.
+* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges unless asked.
 Current Date: ${new Date().toUTCString()} UTC
 </SYSTEM_CAPABILITY>
 
@@ -134,20 +131,11 @@ AskUIAgent.AndroidSystemPrompt = `
 <SYSTEM_CAPABILITY>
 You are an autonomous AI assistant operating on an Android device via ADB. The host machine is ${process.platform} with ${process.arch} architecture and internet connectivity.
 Your main goal is to mimic a human user interacting with an Android device. So you should try to use the tools in a way that a human would use a touch screen to interact with an Android device.
-
-Key Capabilities:
-* Full Android device control through ADB
-* Screen capture and analysis
-* Touch input simulation
-* Android-specific key events
-* Error handling and recovery mechanisms
-
-Available Tools:
-* Touch control (click, swipe, scroll)
-* Android key events (single and sequence)
-* Screen capture and analysis
-* Error reporting and recovery
-
+Use a gestures and adb commands to interact with the android device, and take screenshots.
+* Some applications may take time to start or process actions, so you may need to wait and take successive screenshots to see the results of your actions. E.g. if you click on Firefox and a window doesn't open, try taking another screenshot.
+* Whenever you intend to move the cursor to click on an element like an icon, you should consult a screenshot to determine the coordinates of the element before moving the cursor.
+* If you tried clicking on a program or link but it failed to load, even after waiting, try adjusting your cursor position so that the tip of the cursor visually falls on the element that you want to click.
+* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges unless asked.
 Current Date: ${new Date().toUTCString()} UTC
 </SYSTEM_CAPABILITY>
 

package/dist/cjs/core/models/anthropic/claude-agent.d.ts

@@ -1,5 +1,6 @@
 import { Beta } from '@anthropic-ai/sdk/resources';
 import { BaseAgentTool } from './tools/base';
+import { BetaMessageParam } from '@anthropic-ai/sdk/resources/beta/messages';
 type PredictActResponseFunction = (params: {
     max_tokens: number;
     messages: Beta.BetaMessageParam[];
@@ -7,7 +8,16 @@ type PredictActResponseFunction = (params: {
     system?: string;
     tools?: any[];
     betas?: string[];
+    tool_choice?: {
+        type: 'tool' | 'any' | 'auto';
+        name?: string;
+    };
 }) => Promise<Beta.BetaMessage>;
+export type AgentHistory = BetaMessageParam[];
+export interface ActOptions {
+    chatId?: string;
+    agentHistory?: Beta.BetaMessageParam[];
+}
 export declare class ClaudeAgent {
     private predictActResponseFunction;
     private maxTokens;
@@ -19,7 +29,12 @@ export declare class ClaudeAgent {
     private _toolCollection;
     private tools;
     private history;
+    private toolChoice;
     constructor(predictActResponseFunction: PredictActResponseFunction);
+    setToolChoice(toolChoice: {
+        type: 'tool' | 'any' | 'auto';
+        name?: string;
+    }): void;
     setTools(tools: BaseAgentTool[]): void;
     addTool(tool: BaseAgentTool): void;
     listToolNames(): string[];
@@ -29,10 +44,7 @@ export declare class ClaudeAgent {
     private get toolCollection();
     private setHistory;
     private getHistory;
-    act(goal: string, options?: {
-        chatId?: string;
-        agentHistory?: Beta.BetaMessageParam[];
-    }): Promise<Beta.BetaMessageParam[]>;
+    act(goal: string, imagePathOrBase64String?: string, options?: ActOptions): Promise<Beta.BetaMessageParam[]>;
     private makeApiToolResult;
     private maybePrependSystemToolResult;
     private static filterNMostRecentImages;

package/dist/cjs/core/models/anthropic/claude-agent.js

@@ -12,6 +12,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.ClaudeAgent = void 0;
 const base_1 = require("./tools/base");
 const logger_1 = require("../../../lib/logger");
+const base_64_image_1 = require("../../../utils/base_64_image/base-64-image");
 class ClaudeAgent {
     constructor(predictActResponseFunction) {
         this.predictActResponseFunction = predictActResponseFunction;
@@ -19,11 +20,17 @@ class ClaudeAgent {
         this.onlyNMostRecentImages = 3;
         this.imageTruncationThreshold = 10;
         this.systemPrompt = '';
-        this.model = 'claude-3-5-sonnet-20241022';
-        this.betas = ['computer-use-2024-10-22'];
+        this.model = 'claude-sonnet-4-20250514';
+        this.betas = ['computer-use-2025-01-24'];
         this._toolCollection = undefined;
         this.tools = [];
         this.history = {};
+        this.toolChoice = {
+            type: 'any',
+        };
+    }
+    setToolChoice(toolChoice) {
+        this.toolChoice = toolChoice;
     }
     setTools(tools) {
         this._toolCollection = undefined;
@@ -40,7 +47,14 @@ class ClaudeAgent {
         this.tools = this.tools.filter((tool) => tool.ToolName !== toolName);
     }
     setSystemPrompt(systemPrompt) {
-        this.systemPrompt = systemPrompt;
+        const enhancedPrompt = `${systemPrompt}
+        If you cannot complete a request due to safety concerns, please:
+        1. Explain what specific aspect is problematic
+        2. Suggest alternative approaches that would be acceptable
+        3. Provide partial assistance where possible within guidelines.
+        Raise an exception After you have provided the above information. include the error message in the exception.
+        `;
+        this.systemPrompt = enhancedPrompt;
     }
     IsConfigured() {
         return this.tools.length > 0 && this.systemPrompt !== '';
@@ -57,7 +71,7 @@ class ClaudeAgent {
     getHistory(key) {
         return this.history[key] || [];
     }
-    act(goal, options) {
+    act(goal, imagePathOrBase64String, options) {
         return __awaiter(this, void 0, void 0, function* () {
             if (!goal.trim()) {
                 throw new Error('Goal cannot be empty');
@@ -73,8 +87,24 @@ class ClaudeAgent {
                 messages.push(...this.getHistory(options.chatId));
             }
             // Add the new goal as a user message
+            const userContent = [{
+                    type: 'text',
+                    text: goal,
+                }];
+            if (imagePathOrBase64String !== undefined) {
+                const image = yield base_64_image_1.Base64Image.fromPathOrString(imagePathOrBase64String);
+                const imageString = image.toString(false);
+                userContent.push({
+                    type: 'image',
+                    source: {
+                        type: 'base64',
+                        media_type: 'image/png',
+                        data: imageString,
+                    },
+                });
+            }
             messages.push({
-                content: goal,
+                content: userContent,
                 role: 'user',
             });
             if (this.onlyNMostRecentImages) {
@@ -88,7 +118,15 @@ class ClaudeAgent {
                     system: this.systemPrompt,
                     tools: (new base_1.ToolCollection(this.tools).toParams()),
                     betas: this.betas,
+                    tool_choice: this.toolChoice,
                 });
+                if (response.stop_reason === 'refusal') {
+                    const refusalMessage = response.content
+                        .filter(block => block.type === 'text')
+                        .map(block => block.text)
+                        .join(' ');
+                    throw new Error(`Agent refused to answer: ${refusalMessage || 'The request violates agent\'s usage policies'}`);
+                }
                 messages.push({
                     content: response.content,
                     role: 'assistant',

package/dist/cjs/core/models/anthropic/index.d.ts

@@ -1,4 +1,3 @@
-import { BetaMessageParam } from '@anthropic-ai/sdk/resources/beta/messages';
-export type AgentHistory = BetaMessageParam[];
 export { AskUIAgent } from './askui-agent';
 export { ToolFailure, ToolError, BaseAgentTool } from './tools/base';
+export { AgentHistory, ActOptions } from './claude-agent';

package/dist/cjs/core/models/anthropic/tools/os-agent-tools.d.ts

@@ -5,21 +5,36 @@ import { ExecutionRuntime } from '../../../../execution/execution-runtime';
 import { ControlCommand } from '../../../ui-control-commands';
 export declare class OsAgentHandler {
     private AgentOsClient;
-    private TargetResolution;
+    private targetResolution;
     private screenDimensions;
+    private paddingInfo;
     constructor(AgentOsClient: ExecutionRuntime, screenDimensions: {
         width: number;
         height: number;
     });
+    private updatePaddingInfo;
     static createInstance(AgentOsClient: ExecutionRuntime): Promise<OsAgentHandler>;
     getTargetResolution(): {
         width: number;
         height: number;
     };
+    getScreenDimensions(): {
+        width: number;
+        height: number;
+    };
     setTargetResolution(width: number, height: number): void;
     takeScreenshot(): Promise<string>;
     private scaleCoordinates;
     requestControl(controlCommand: ControlCommand): Promise<void>;
+    mouseMove(x: number, y: number): Promise<void>;
+    mouseClick(button: "left" | "right" | "middle", doubleClick: boolean): Promise<void>;
+    mouseScroll(dx: number, dy: number): Promise<void>;
+    mouseHoldLeftButtonDown(): Promise<void>;
+    mouseReleaseLeftButton(): Promise<void>;
+    desktopKeyPressAndRelease(key: PC_AND_MODIFIER_KEY, modifiers?: MODIFIER_KEY[]): Promise<void>;
+    desktopKeyHoldDown(key: PC_AND_MODIFIER_KEY, modifiers?: MODIFIER_KEY[]): Promise<void>;
+    desktopKeyRelease(key: PC_AND_MODIFIER_KEY, modifiers?: MODIFIER_KEY[]): Promise<void>;
+    typeText(text: string): Promise<void>;
 }
 export declare class ScreenShotTool extends BaseAgentTool {
     private osAgentHandler;
@@ -54,21 +69,53 @@ export declare class MouseScrollTool extends BaseAgentTool {
     }): Promise<ToolResult>;
     toParams(): BetaTool;
 }
-export declare class DesktopKeyPressSequenceTool extends BaseAgentTool {
+export declare class MouseDragAndDropTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(command: {
+        startX: number;
+        startY: number;
+        endX: number;
+        endY: number;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class MouseHoldLeftButtonDownTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class MouseReleaseLeftButtonTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class DesktopPressAndReleaseKeysTool extends BaseAgentTool {
     private osAgentHandler;
     constructor(osAgentHandler: OsAgentHandler);
     execute(command: {
         key: PC_KEY;
-        firstModifier?: MODIFIER_KEY;
-        secondModifier?: MODIFIER_KEY;
+        modifiers?: MODIFIER_KEY[];
     }): Promise<ToolResult>;
     toParams(): BetaTool;
 }
-export declare class DesktopSingleKeyPressTool extends BaseAgentTool {
+export declare class DesktopKeyHoldDownTool extends BaseAgentTool {
     private osAgentHandler;
     constructor(osAgentHandler: OsAgentHandler);
     execute(command: {
         key: PC_AND_MODIFIER_KEY;
+        modifiers?: MODIFIER_KEY[];
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class DesktopKeyReleaseTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(command: {
+        key: PC_AND_MODIFIER_KEY;
+        modifiers?: MODIFIER_KEY[];
     }): Promise<ToolResult>;
     toParams(): BetaTool;
 }
@@ -111,3 +158,10 @@ export declare class ExecuteShellCommandTool extends BaseAgentTool {
     }): Promise<ToolResult>;
     toParams(): BetaTool;
 }
+export declare class WaitTool extends BaseAgentTool {
+    constructor();
+    execute(command: {
+        milliseconds: number;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}