askui 0.24.0 → 0.25.0

package/dist/cjs/execution/inference-client.js

@@ -33,6 +33,7 @@ class InferenceClient {
             ? (0, url_join_1.default)(versionedBaseUrl, 'workspaces', workspaceId)
             : versionedBaseUrl;
         this.urls = {
+            actEndpoint: (0, url_join_1.default)(url, 'act', 'inference'),
             inference: (0, url_join_1.default)(url, 'inference'),
             isImageRequired: (0, url_join_1.default)(url, 'instruction', 'is-image-required'),
             vqaInference: (0, url_join_1.default)(url, 'vqa', 'inference'),
@@ -74,7 +75,7 @@ class InferenceClient {
                     instruction,
                     modelComposition: modelComposition.length > 0 ? modelComposition : this.modelComposition,
                 });
-            InferenceClient.logMetaInformation(response);
+            InferenceClient.logMetaInformation(response.headers);
             return ui_control_commands_1.InferenceResponse.fromJson(response.body, resizedImage.resizeRatio, image);
         });
     }
@@ -85,13 +86,13 @@ class InferenceClient {
                 image,
                 prompt,
             });
-            InferenceClient.logMetaInformation(response);
+            InferenceClient.logMetaInformation(response.headers);
             return response.body;
         });
     }
-    static logMetaInformation(response) {
-        if (response.headers['askui-usage-warnings'] !== undefined) {
-            logger_1.logger.warn(response.headers['askui-usage-warnings']);
+    static logMetaInformation(headers) {
+        if (headers['askui-usage-warnings'] !== undefined) {
+            logger_1.logger.warn(headers['askui-usage-warnings']);
         }
     }
     predictControlCommand(instruction_1, modelComposition_1) {
@@ -134,5 +135,12 @@ class InferenceClient {
             return response;
         });
     }
+    predictActResponse(params) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const response = yield this.httpClient.post(this.urls.actEndpoint, params);
+            InferenceClient.logMetaInformation(response.headers);
+            return response.body;
+        });
+    }
 }
 exports.InferenceClient = InferenceClient;

package/dist/cjs/execution/ui-control-client-dependency-builder.js

@@ -65,7 +65,7 @@ class UiControlClientDependencyBuilder {
                     onLocationNotExist: (_d = (_c = clientArgs.aiElementArgs) === null || _c === void 0 ? void 0 : _c.onLocationNotExist) !== null && _d !== void 0 ? _d : 'error',
                 }, context: {
                     isCi: (_f = (_e = clientArgs.context) === null || _e === void 0 ? void 0 : _e.isCi) !== null && _f !== void 0 ? _f : is_ci_1.default,
-                }, credentials: (0, read_credentials_1.readCredentials)(clientArgs), inferenceServerApiVersion: (_g = clientArgs.inferenceServerApiVersion) !== null && _g !== void 0 ? _g : 'v3', inferenceServerUrl: (_h = clientArgs.inferenceServerUrl) !== null && _h !== void 0 ? _h : 'https://inference.askui.com', proxyAgents: (_j = clientArgs.proxyAgents) !== null && _j !== void 0 ? _j : (yield (0, proxy_builder_1.envProxyAgents)()), uiControllerUrl: (_k = clientArgs.uiControllerUrl) !== null && _k !== void 0 ? _k : 'http://127.0.0.1:6769' });
+                }, credentials: (0, read_credentials_1.readCredentials)(clientArgs), inferenceServerApiVersion: (_g = clientArgs.inferenceServerApiVersion) !== null && _g !== void 0 ? _g : 'v1', inferenceServerUrl: (_h = clientArgs.inferenceServerUrl) !== null && _h !== void 0 ? _h : 'https://inference.askui.com', proxyAgents: (_j = clientArgs.proxyAgents) !== null && _j !== void 0 ? _j : (yield (0, proxy_builder_1.envProxyAgents)()), uiControllerUrl: (_k = clientArgs.uiControllerUrl) !== null && _k !== void 0 ? _k : 'http://127.0.0.1:6769' });
         });
     }
 }

package/dist/cjs/execution/ui-control-client.d.ts

@@ -5,6 +5,7 @@ import { AnnotationRequest } from '../core/model/annotation-result/annotation-in
 import { DetectedElement } from '../core/model/annotation-result/detected-element';
 import { ClientArgs } from './ui-controller-client-interface';
 import { ModelCompositionBranch } from './model-composition-branch';
+import { AskUIAgent, AgentHistory } from '../core/models/anthropic';
 export type RelationsForConvenienceMethods = 'nearestTo' | 'leftOf' | 'above' | 'rightOf' | 'below' | 'contains';
 export type TextMatchingOption = 'similar' | 'exact' | 'regex';
 export type ElementExistsQueryType = 'otherElement' | 'switch' | 'element' | 'container' | 'checkbox' | 'element' | 'button' | 'table' | 'text' | 'icon' | 'image' | 'textfield';
@@ -33,6 +34,7 @@ export declare class UiControlClient extends ApiCommands {
     private executionRuntime;
     private stepReporter;
     private aiElementArgs;
+    agent: AskUIAgent;
     private constructor();
     static build(clientArgs?: ClientArgs): Promise<UiControlClient>;
     /**
@@ -473,4 +475,52 @@ export declare class UiControlClient extends ApiCommands {
      * @returns {ExpectAllExistResult.elements} - ExpectExistenceElement[].
      */
     expectAllExist(query: ElementExistsQuery[]): Promise<ExpectAllExistResult>;
+    /**
+     * Instructs the agent to achieve a specified goal through autonomous actions.
+     *
+     * The agent will analyze the screen, determine necessary steps, and perform actions
+     * to accomplish the goal. This may include clicking, typing, scrolling, and other
+     * interface interactions.
+     *
+     * The `options` parameter allows the caller to maintain contextual continuity across
+     * multiple `act` calls, either from the same or different agent interfaces.
+     *
+     * **Examples:**
+     *
+     * ```ts
+     * // Use chatId to maintain context across consecutive steps
+     * await aui.act("Search online for the current gold price", {
+     *   chatId: "session-gold-price"
+     * });
+     * await aui.act("Create a new text file and type the gold price result into it", {
+     *   chatId: "session-gold-price"
+     * });
+     *
+     * // Share history explicitly between separate agents (e.g., desktop and Android)
+     * // By default, the agent operates as a computer agent.
+     * // To control an Android device, you must configure it explicitly:
+     * await auiAndroid.agent.configureAsAndroidAgent();
+     * const history = await auiDesktop.act("Copy username from desktop app");
+     * await auiAndroid.act("Paste username into the mobile login screen", {
+     *   agentHistory: history
+     * });
+     * ```
+     *
+     * @param {string} goal - A description of what the agent should achieve.
+     * @param {Object} [options] - Optional parameters to maintain or share context.
+     * @param {string} [options.chatId] - A session identifier used to persist memory between
+     *                                    consecutive `act` calls. When multiple actions share the
+     *                                    same `chatId`, the agent retains knowledge of prior steps,
+     *                                    such as extracted data or navigation history.
+     * @param {AgentHistory} [options.agentHistory] - A shared interaction history object that can be
+     *                                           passed between different agent clients (e.g., between
+     *                                           `auiDesktop` and `auiAndroid`) to ensure continuity
+     *                                           of understanding and task flow.
+     * @returns {Promise<AgentHistory>} - Updated action history after executing the goal.
+     * @throws {Error} If the agent is not connected when the method is called.
+     */
+    act(goal: string, options?: {
+        chatId?: string;
+        agentHistory?: AgentHistory;
+    }): Promise<AgentHistory>;
 }

package/dist/cjs/execution/ui-control-client.js

@@ -18,13 +18,15 @@ const logger_1 = require("../lib/logger");
 const ui_control_client_dependency_builder_1 = require("./ui-control-client-dependency-builder");
 const ai_element_collection_1 = require("../core/ai-element/ai-element-collection");
 const retry_strategies_1 = require("./retry-strategies");
+const anthropic_1 = require("../core/models/anthropic");
 class UiControlClient extends dsl_1.ApiCommands {
-    constructor(workspaceId, executionRuntime, stepReporter, aiElementArgs) {
+    constructor(workspaceId, executionRuntime, stepReporter, aiElementArgs, agent) {
         super();
         this.workspaceId = workspaceId;
         this.executionRuntime = executionRuntime;
         this.stepReporter = stepReporter;
         this.aiElementArgs = aiElementArgs;
+        this.agent = agent;
         this.secretText = undefined;
     }
     static build() {
@@ -32,7 +34,8 @@ class UiControlClient extends dsl_1.ApiCommands {
             const builder = ui_control_client_dependency_builder_1.UiControlClientDependencyBuilder;
             const clientArgsWithDefaults = yield builder.getClientArgsWithDefaults(clientArgs);
             const { workspaceId, executionRuntime, stepReporter, } = yield builder.build(clientArgsWithDefaults);
-            return new UiControlClient(workspaceId, executionRuntime, stepReporter, clientArgsWithDefaults.aiElementArgs);
+            const agent = new anthropic_1.AskUIAgent(executionRuntime);
+            return new UiControlClient(workspaceId, executionRuntime, stepReporter, clientArgsWithDefaults.aiElementArgs, agent);
         });
     }
     /**
@@ -40,7 +43,10 @@ class UiControlClient extends dsl_1.ApiCommands {
      */
     connect() {
         return __awaiter(this, void 0, void 0, function* () {
-            return this.executionRuntime.connect();
+            const connectionState = yield this.executionRuntime.connect();
+            yield this.agent.initializeOsAgentHandler();
+            yield this.agent.configureAsDesktopAgent();
+            return connectionState;
         });
     }
     /**
@@ -732,5 +738,57 @@ class UiControlClient extends dsl_1.ApiCommands {
             };
         });
     }
+    /**
+     * Instructs the agent to achieve a specified goal through autonomous actions.
+     *
+     * The agent will analyze the screen, determine necessary steps, and perform actions
+     * to accomplish the goal. This may include clicking, typing, scrolling, and other
+     * interface interactions.
+     *
+     * The `options` parameter allows the caller to maintain contextual continuity across
+     * multiple `act` calls, either from the same or different agent interfaces.
+     *
+     * **Examples:**
+     *
+     * ```ts
+     * // Use chatId to maintain context across consecutive steps
+     * await aui.act("Search online for the current gold price", {
+     *   chatId: "session-gold-price"
+     * });
+     * await aui.act("Create a new text file and type the gold price result into it", {
+     *   chatId: "session-gold-price"
+     * });
+     *
+     * // Share history explicitly between separate agents (e.g., desktop and Android)
+     * // By default, the agent operates as a computer agent.
+     * // To control an Android device, you must configure it explicitly:
+     * await auiAndroid.agent.configureAsAndroidAgent();
+     * const history = await auiDesktop.act("Copy username from desktop app");
+     * await auiAndroid.act("Paste username into the mobile login screen", {
+     *   agentHistory: history
+     * });
+     * ```
+     *
+     * @param {string} goal - A description of what the agent should achieve.
+     * @param {Object} [options] - Optional parameters to maintain or share context.
+     * @param {string} [options.chatId] - A session identifier used to persist memory between
+     *                                    consecutive `act` calls. When multiple actions share the
+     *                                    same `chatId`, the agent retains knowledge of prior steps,
+     *                                    such as extracted data or navigation history.
+     * @param {AgentHistory} [options.agentHistory] - A shared interaction history object that can be
+     *                                           passed between different agent clients (e.g., between
+     *                                           `auiDesktop` and `auiAndroid`) to ensure continuity
+     *                                           of understanding and task flow.
+     * @returns {Promise<AgentHistory>} - Updated action history after executing the goal.
+     * @throws {Error} If the agent is not connected when the method is called.
+     */
+    act(goal, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.agent.isConnected()) {
+                throw new Error('Agent is not connected, Please call connect() first');
+            }
+            return this.agent.act(goal, options);
+        });
+    }
 }
 exports.UiControlClient = UiControlClient;

package/dist/cjs/lib/interactive_cli/create-example-project.d.ts

@@ -20,6 +20,7 @@ export declare class CreateExampleProject {
     private static installTestFrameworkPackages;
     private copyESLintConfigFiles;
     private copyGitignore;
+    private addVSCodeSettings;
     private copyTsConfigFile;
     createExampleProject(): Promise<void>;
 }

package/dist/cjs/lib/interactive_cli/create-example-project.js

@@ -176,6 +176,10 @@ class CreateExampleProject {
                                 title: 'Add eslint run command',
                                 task: () => __awaiter(this, void 0, void 0, function* () { return this.addESLintRunCommand(); }),
                             },
+                            {
+                                title: 'Add vscode settings',
+                                task: () => __awaiter(this, void 0, void 0, function* () { return this.addVSCodeSettings(); }),
+                            },
                         ]);
                     }),
                 }];
@@ -235,23 +239,36 @@ class CreateExampleProject {
             ];
         });
     }
+    addVSCodeSettings() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const vscodeSettingsFilePath = path_1.default.join('example_projects_templates', 'configs', 'vscode-settings.json');
+            const vscodeSettingsTargetDirPath = path_1.default.join(this.projectRootDirectoryPath, '.vscode');
+            const vscodeSettingsTargetFilePath = path_1.default.join(vscodeSettingsTargetDirPath, 'settings.json');
+            return [{
+                    enabled: () => !fs_extra_1.default.existsSync(vscodeSettingsTargetFilePath),
+                    task: () => __awaiter(this, void 0, void 0, function* () {
+                        yield fs_extra_1.default.mkdir(vscodeSettingsTargetDirPath, { recursive: true });
+                        yield fs_extra_1.default.copyFile(path_1.default.join((0, path_2.getPathToNodeModulesRoot)(), vscodeSettingsFilePath), vscodeSettingsTargetFilePath);
+                    }),
+                    title: 'Copy VSCode settings',
+                }];
+        });
+    }
     copyTsConfigFile() {
         return __awaiter(this, void 0, void 0, function* () {
             const tsConfigFilePath = path_1.default.join('example_projects_templates', 'typescript', 'tsconfig.json');
             const tsConfigTargetFilePath = path_1.default.join(this.projectRootDirectoryPath, 'tsconfig.json');
-            /* eslint-disable sort-keys */
             return [
                 {
-                    title: 'Copy ts config file',
                     enabled: () => this.cliOptions.typescriptConfig || !fs_extra_1.default.existsSync(tsConfigTargetFilePath),
                     task: () => __awaiter(this, void 0, void 0, function* () {
                         return fs_extra_1.default.copyFile(path_1.default.join((0, path_2.getPathToNodeModulesRoot)(), tsConfigFilePath), tsConfigTargetFilePath);
                     }),
+                    title: 'Copy ts config file',
                 },
             ];
         });
     }
-    /* eslint-enable */
     createExampleProject() {
         return __awaiter(this, void 0, void 0, function* () {
             const tasks = new listr_1.default();

package/dist/cjs/main.d.ts

@@ -4,3 +4,4 @@ export { Instruction, Reporter, ReporterConfig, Snapshot, SnapshotDetailLevel, S
 export { Annotation } from './core/annotation/annotation';
 export { DetectedElement } from './core/model/annotation-result/detected-element';
 export { LogLevels } from './shared';
+export { ToolFailure, ToolError, BaseAgentTool } from './core/models/anthropic';

package/dist/cjs/main.js

@@ -14,7 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.LogLevels = exports.DetectedElement = exports.Annotation = exports.UiController = void 0;
+exports.BaseAgentTool = exports.ToolError = exports.ToolFailure = exports.LogLevels = exports.DetectedElement = exports.Annotation = exports.UiController = void 0;
 var lib_1 = require("./lib");
 Object.defineProperty(exports, "UiController", { enumerable: true, get: function () { return lib_1.UiController; } });
 __exportStar(require("./execution"), exports);
@@ -24,3 +24,7 @@ var detected_element_1 = require("./core/model/annotation-result/detected-elemen
 Object.defineProperty(exports, "DetectedElement", { enumerable: true, get: function () { return detected_element_1.DetectedElement; } });
 var shared_1 = require("./shared");
 Object.defineProperty(exports, "LogLevels", { enumerable: true, get: function () { return shared_1.LogLevels; } });
+var anthropic_1 = require("./core/models/anthropic");
+Object.defineProperty(exports, "ToolFailure", { enumerable: true, get: function () { return anthropic_1.ToolFailure; } });
+Object.defineProperty(exports, "ToolError", { enumerable: true, get: function () { return anthropic_1.ToolError; } });
+Object.defineProperty(exports, "BaseAgentTool", { enumerable: true, get: function () { return anthropic_1.BaseAgentTool; } });

package/dist/cjs/utils/base_64_image/base-64-image.d.ts

@@ -11,5 +11,6 @@ export declare class Base64Image {
     private getSharp;
     getInfo(): Promise<sharp.OutputInfo>;
     resizeToFitInto(dimension: number): Promise<Base64Image>;
-    toString(): string;
+    resizeWithSameAspectRatio(width: number, height: number): Promise<Base64Image>;
+    toString(withPrefix?: boolean): string;
 }

package/dist/cjs/utils/base_64_image/base-64-image.js

@@ -77,8 +77,23 @@ class Base64Image {
             return Base64Image.fromBuffer(buffer);
         });
     }
-    toString() {
-        return `${Base64Image.strPrefix}${this.buffer.toString('base64')}`;
+    resizeWithSameAspectRatio(width, height) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const buffer = yield (yield this.getSharp())
+                .resize({
+                fit: 'contain',
+                height,
+                width,
+            })
+                .toBuffer();
+            return Base64Image.fromBuffer(buffer);
+        });
+    }
+    toString(withPrefix = true) {
+        if (withPrefix) {
+            return `${Base64Image.strPrefix}${this.buffer.toString('base64')}`;
+        }
+        return this.buffer.toString('base64');
     }
 }
 exports.Base64Image = Base64Image;

package/dist/esm/core/models/anthropic/askui-agent.d.ts

@@ -0,0 +1,13 @@
+import { ClaudeAgent } from './claude-agent';
+import { ExecutionRuntime } from '../../../execution/execution-runtime';
+export declare class AskUIAgent extends ClaudeAgent {
+    private osAgentHandler;
+    private executionRuntime;
+    constructor(executionRuntime: ExecutionRuntime);
+    isConnected(): boolean;
+    initializeOsAgentHandler(): Promise<void>;
+    configureAsDesktopAgent(): Promise<void>;
+    configureAsAndroidAgent(): Promise<void>;
+    private static DesktopSystemPrompt;
+    private static AndroidSystemPrompt;
+}

package/dist/esm/core/models/anthropic/askui-agent.js

@@ -0,0 +1,191 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { DesktopKeyPressSequenceTool, DesktopSingleKeyPressTool, MouseClickTool, MouseMoveTool, MouseScrollTool, OsAgentHandler, ScreenShotTool, TypeTool, AgentErrorTool, AndroidSequenceKeyPressTool, AndroidSingleKeyPressTool, ExecuteShellCommandTool, } from './tools/os-agent-tools';
+import { ClaudeAgent } from './claude-agent';
+export class AskUIAgent extends ClaudeAgent {
+    constructor(executionRuntime) {
+        super((params) => executionRuntime.predictActResponse(params));
+        this.osAgentHandler = undefined;
+        this.executionRuntime = executionRuntime;
+    }
+    isConnected() {
+        return this.osAgentHandler !== undefined;
+    }
+    initializeOsAgentHandler() {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.osAgentHandler = yield OsAgentHandler.createInstance(this.executionRuntime);
+        });
+    }
+    configureAsDesktopAgent() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.osAgentHandler) {
+                throw new Error('Agent OS client is not connected');
+            }
+            const tools = [
+                new AgentErrorTool(),
+                new ScreenShotTool(this.osAgentHandler),
+                new MouseMoveTool(this.osAgentHandler),
+                new MouseClickTool(this.osAgentHandler),
+                new MouseScrollTool(this.osAgentHandler),
+                new DesktopKeyPressSequenceTool(this.osAgentHandler),
+                new DesktopSingleKeyPressTool(this.osAgentHandler),
+                new TypeTool(this.osAgentHandler),
+            ];
+            this.setTools(tools);
+            this.setSystemPrompt(AskUIAgent.DesktopSystemPrompt);
+        });
+    }
+    configureAsAndroidAgent() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.osAgentHandler) {
+                throw new Error('Agent OS client is not connected');
+            }
+            const tools = [
+                new AgentErrorTool(),
+                new ScreenShotTool(this.osAgentHandler),
+                new MouseMoveTool(this.osAgentHandler),
+                new MouseClickTool(this.osAgentHandler),
+                new MouseScrollTool(this.osAgentHandler),
+                new AndroidSingleKeyPressTool(this.osAgentHandler),
+                new AndroidSequenceKeyPressTool(this.osAgentHandler),
+                new TypeTool(this.osAgentHandler),
+                new ExecuteShellCommandTool(this.osAgentHandler),
+            ];
+            this.setTools(tools);
+            this.setSystemPrompt(AskUIAgent.AndroidSystemPrompt);
+        });
+    }
+}
+AskUIAgent.DesktopSystemPrompt = `
+<SYSTEM_CAPABILITY>
+You are an autonomous AI assistant operating on a ${process.platform} machine with ${process.arch} architecture. You have full access to the system and internet connectivity.
+Your main goal is to mimic a human user interacting with a desktop computer. So you should try to use the tools in a way that a human would use a mouse and keyboard to interact with a computer.
+
+Key Capabilities:
+* Full system control through mouse and keyboard interactions
+* Screen capture and analysis
+* Web browser automation and navigation
+* File system access and manipulation
+* PDF document handling and text extraction
+* Error handling and recovery mechanisms
+
+Available Tools:
+* Mouse control (move, click, scroll)
+* Keyboard input (single keys, key combinations, typing)
+* Screen capture and analysis
+* Error reporting and recovery
+
+Current Date: ${new Date().toUTCString()} UTC
+</SYSTEM_CAPABILITY>
+
+<OPERATIONAL_GUIDELINES>
+1. Autonomous Operation:
+   * Work independently to achieve user goals
+   * Make informed decisions based on available information
+   * Chain multiple actions efficiently when possible
+   * Verify results after each significant action
+
+2. Web Interaction:
+   * Launch appropriate browser if not already open
+   * Ensure full page visibility through zoom or scrolling
+   * Handle browser-specific behaviors (e.g., Firefox startup wizard)
+   * Extract and process PDF content when encountered
+
+3. Error Handling:
+   * Detect and analyze failure points
+   * Implement appropriate recovery strategies
+   * Report issues with clear diagnostic information
+   * Use the error tool when stuck or unable to proceed
+
+4. Performance Optimization:
+   * Batch related actions when possible
+   * Minimize unnecessary screen captures
+   * Use efficient navigation patterns
+   * Maintain context between actions
+
+5. Safety and Validation:
+   * Verify coordinates are within screen bounds
+   * Validate input parameters before execution
+   * Ensure proper cleanup after operations
+   * Maintain system stability
+</OPERATIONAL_GUIDELINES>
+
+<IMPORTANT_NOTES>
+* When you are stuck or unable to proceed, use the error tool to raise an error.
+* Always verify tool availability before use
+* Use screenshots strategically for state analysis
+* Report issues promptly with clear diagnostic information
+* Maintain awareness of screen boundaries and coordinate validity
+* Adapt to unexpected situations with appropriate fallback strategies
+</IMPORTANT_NOTES>
+`;
+AskUIAgent.AndroidSystemPrompt = `
+<SYSTEM_CAPABILITY>
+You are an autonomous AI assistant operating on an Android device via ADB. The host machine is ${process.platform} with ${process.arch} architecture and internet connectivity.
+Your main goal is to mimic a human user interacting with an Android device. So you should try to use the tools in a way that a human would use a touch screen to interact with an Android device.
+
+Key Capabilities:
+* Full Android device control through ADB
+* Screen capture and analysis
+* Touch input simulation
+* Android-specific key events
+* Error handling and recovery mechanisms
+
+Available Tools:
+* Touch control (click, swipe, scroll)
+* Android key events (single and sequence)
+* Screen capture and analysis
+* Error reporting and recovery
+
+Current Date: ${new Date().toUTCString()} UTC
+</SYSTEM_CAPABILITY>
+
+<OPERATIONAL_GUIDELINES>
+1. Autonomous Operation:
+   * Work independently to achieve user goals
+   * Make informed decisions based on available information
+   * Chain multiple actions efficiently when possible
+   * Verify results after each significant action
+
+2. Screen Interaction:
+   * Analyze screen state before interactions
+   * Use appropriate input methods (touch, keys)
+   * Handle dynamic UI elements effectively
+   * Implement efficient navigation patterns
+
+3. Error Handling:
+   * Detect and analyze failure points
+   * Implement appropriate recovery strategies
+   * Report issues with clear diagnostic information
+   * Use the error tool when stuck or unable to proceed
+
+4. Performance Optimization:
+   * Batch related actions when possible
+   * Minimize unnecessary screen captures
+   * Use efficient navigation patterns
+   * Maintain context between actions
+
+5. Safety and Validation:
+   * Verify coordinates are within screen bounds
+   * Validate input parameters before execution
+   * Ensure proper cleanup after operations
+   * Maintain device stability
+</OPERATIONAL_GUIDELINES>
+
+<IMPORTANT_NOTES>
+* When you are stuck or unable to proceed, use the error tool to raise an error.
+* Always verify tool availability before use
+* Use screenshots strategically for state analysis
+* Ensure all coordinates are integers and within screen bounds
+* Handle permission issues and device state appropriately
+* Report issues promptly with clear diagnostic information
+* Adapt to unexpected situations with appropriate fallback strategies
+</IMPORTANT_NOTES>
+`;

package/dist/esm/core/models/anthropic/claude-agent.d.ts

@@ -0,0 +1,40 @@
+import { Beta } from '@anthropic-ai/sdk/resources';
+import { BaseAgentTool } from './tools/base';
+type PredictActResponseFunction = (params: {
+    max_tokens: number;
+    messages: Beta.BetaMessageParam[];
+    model: string;
+    system?: string;
+    tools?: any[];
+    betas?: string[];
+}) => Promise<Beta.BetaMessage>;
+export declare class ClaudeAgent {
+    private predictActResponseFunction;
+    private maxTokens;
+    private onlyNMostRecentImages;
+    private imageTruncationThreshold;
+    private systemPrompt;
+    private model;
+    private betas;
+    private _toolCollection;
+    private tools;
+    private history;
+    constructor(predictActResponseFunction: PredictActResponseFunction);
+    setTools(tools: BaseAgentTool[]): void;
+    addTool(tool: BaseAgentTool): void;
+    listToolNames(): string[];
+    removeToolByName(toolName: string): void;
+    setSystemPrompt(systemPrompt: string): void;
+    private IsConfigured;
+    private get toolCollection();
+    private setHistory;
+    private getHistory;
+    act(goal: string, options?: {
+        chatId?: string;
+        agentHistory?: Beta.BetaMessageParam[];
+    }): Promise<Beta.BetaMessageParam[]>;
+    private makeApiToolResult;
+    private maybePrependSystemToolResult;
+    private static filterNMostRecentImages;
+}
+export {};