askui 0.25.1 → 0.27.0

package/dist/esm/execution/inference-client.d.ts

@@ -26,6 +26,10 @@ export declare class InferenceClient {
     predictVQAAnswer(prompt: string, image: string, config?: object): Promise<any>;
     predictActResponse(params: {
         max_tokens: number;
+        tool_choice?: {
+            type: 'tool' | 'any' | 'auto';
+            name?: string;
+        };
         messages: BetaMessageParam[];
         model: string;
         system?: string;

package/dist/esm/execution/ui-control-client.d.ts

@@ -5,7 +5,7 @@ import { AnnotationRequest } from '../core/model/annotation-result/annotation-in
 import { DetectedElement } from '../core/model/annotation-result/detected-element';
 import { ClientArgs } from './ui-controller-client-interface';
 import { ModelCompositionBranch } from './model-composition-branch';
-import { AskUIAgent, AgentHistory } from '../core/models/anthropic';
+import { AskUIAgent, AgentHistory, ActOptions } from '../core/models/anthropic';
 export type RelationsForConvenienceMethods = 'nearestTo' | 'leftOf' | 'above' | 'rightOf' | 'below' | 'contains';
 export type TextMatchingOption = 'similar' | 'exact' | 'regex';
 export type ElementExistsQueryType = 'otherElement' | 'switch' | 'element' | 'container' | 'checkbox' | 'element' | 'button' | 'table' | 'text' | 'icon' | 'image' | 'textfield';
@@ -55,6 +55,7 @@ export declare class UiControlClient extends ApiCommands {
     stopVideoRecording(): Promise<void>;
     readVideoRecording(): Promise<string>;
     private shouldAnnotateAfterCommandExecution;
+    private beforeNoneInferenceCallCommandExecution;
     private afterCommandExecution;
     annotate(annotationRequest?: AnnotationRequest): Promise<Annotation>;
     annotateInteractively(): Promise<void>;
@@ -476,51 +477,129 @@ export declare class UiControlClient extends ApiCommands {
      */
     expectAllExist(query: ElementExistsQuery[]): Promise<ExpectAllExistResult>;
     /**
-     * Instructs the agent to achieve a specified goal through autonomous actions.
+     * Holds down a key on the keyboard.
      *
-     * The agent will analyze the screen, determine necessary steps, and perform actions
-     * to accomplish the goal. This may include clicking, typing, scrolling, and other
-     * interface interactions.
+     * **Examples:**
+     * ```typescript
+     * await aui.keyDown('a').exec();
+     * ```
      *
-     * The `options` parameter allows the caller to maintain contextual continuity across
-     * multiple `act` calls, either from the same or different agent interfaces.
+     * @param {PC_AND_MODIFIER_KEY} key - The key to hold down.
+     */
+    keyDown(key: PC_AND_MODIFIER_KEY): Executable;
+    /**
+     * Releases a key up that was previously held down.
      *
      * **Examples:**
+     * ```typescript
+     * await aui.keyUp('a').exec();
+     * ```
+     *
+     * @param {PC_AND_MODIFIER_KEY} key - The key to release up.
+     */
+    keyUp(key: PC_AND_MODIFIER_KEY): Executable;
+    /**
+     * Instructs the agent to autonomously achieve a specified goal through UI interactions.
+     *
+     * This method enables AI-powered automation by allowing the agent to:
+     * - Analyze the current screen state and/or provided images
+     * - Plan and execute a sequence of UI interactions
+     * - Handle complex tasks through natural language instructions
+     * - Maintain context across multiple actions
+     *
+     * The agent can perform various UI interactions including:
+     * - Clicking buttons, links, and other interactive elements
+     * - Typing text into input fields
+     * - Scrolling and navigating through interfaces
+     *
+     * ### Method Signatures
+     * ```typescript
+     * act(goal: string, options?: ActOptions): Promise<AgentHistory>
+     * act(goal: string, imagePathOrBase64: string, options?: ActOptions): Promise<AgentHistory>
+     * ```
+     *
+     * ### Parameters
+     * @param goal - A natural language instruction describing the task to accomplish.
+     *               Be specific and clear about the desired outcome.
+     * @param imagePathOrBase64 - (Optional) Path to an image file or base64-encoded image string.
+     *                           Used to provide additional visual context for the task.
+     * @param options - (Optional) Configuration options for the agent's behavior.
+     * @param options.chatId - A unique identifier to maintain context between related actions.
+     *                        Useful for multi-step tasks that require state preservation.
+     * @param options.agentHistory
+     *                          - (Optional) Previous interaction history to share between
+     *                           different agent instances. Enables cross-platform task coordination.
+     *
+     * ### Returns
+     * @returns Promise<AgentHistory> - A promise that resolves to the updated interaction history,
+     *                                 containing details about the actions taken and their outcomes.
+     *
+     * ### Throws
+     * - If the agent is not properly connected
+     * - If the provided goal cannot be understood or executed
+     * - If required UI elements are not found or accessible
+     * - If the image path is invalid or the base64 string is malformed
+     *
+     * ### Examples
+     *
+     * #### Basic Usage
+     * ```typescript
+     * // Simple task execution
+     * await aui.act("Open Chrome and navigate to google.com");
+     * ```
      *
-     * ```ts
-     * // Use chatId to maintain context across consecutive steps
-     * await aui.act("Search online for the current gold price", {
-     *   chatId: "session-gold-price"
+     * #### Maintaining Context
+     * ```typescript
+     * // Multi-step task with context preservation
+     * await aui.act("Search for current gold prices", {
+     *   chatId: "gold-price-task"
      * });
-     * await aui.act("Create a new text file and type the gold price result into it", {
-     *   chatId: "session-gold-price"
+     *
+     * await aui.act("Create a new text file and save the price", {
+     *   chatId: "gold-price-task"
      * });
+     * ```
      *
-     * // Share history explicitly between separate agents (e.g., desktop and Android)
-     * // By default, the agent operates as a computer agent.
-     * // To control an Android device, you must configure it explicitly:
+     * #### Cross-Platform Coordination
+     * ```typescript
+     * // Share context between desktop and mobile agents
      * await auiAndroid.agent.configureAsAndroidAgent();
+     *
      * const history = await auiDesktop.act("Copy username from desktop app");
-     * await auiAndroid.act("Paste username into the mobile login screen", {
+     * await auiAndroid.act("Paste username into mobile login", {
      *   agentHistory: history
      * });
      * ```
      *
-     * @param {string} goal - A description of what the agent should achieve.
-     * @param {Object} [options] - Optional parameters to maintain or share context.
-     * @param {string} [options.chatId] - A session identifier used to persist memory between
-     *                                    consecutive `act` calls. When multiple actions share the
-     *                                    same `chatId`, the agent retains knowledge of prior steps,
-     *                                    such as extracted data or navigation history.
-     * @param {AgentHistory} [options.agentHistory] - A shared interaction history object that can be
-     *                                           passed between different agent clients (e.g., between
-     *                                           `auiDesktop` and `auiAndroid`) to ensure continuity
-     *                                           of understanding and task flow.
-     * @returns {Promise<AgentHistory>} - Updated action history after executing the goal.
-     * @throws {Error} If the agent is not connected when the method is called.
+     * #### Using Images for Context
+     * ```typescript
+     * // Using image file
+     * await aui.act(
+     *   "Click the 'Submit' button in the provided image",
+     *   'path/to/screenshot.png'
+     * );
+     *
+     * // Using base64 image
+     * const base64Image = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...";
+     * await aui.act(
+     *   "Click the 'Submit' button in the provided image",
+     *   base64Image
+     * );
+     * ```
+     *
+     * ### Best Practices
+     * 1. Be specific in your goal descriptions
+     * 2. Use chatId for related tasks to maintain context
+     * 3. Provide clear visual context when needed
+     * 4. Handle errors appropriately in your implementation
+     * 5. Consider using agentHistory for complex cross-platform workflows
+     */
+    act(goal: string, options?: ActOptions): Promise<AgentHistory>;
+    act(goal: string, imagePathOrBase64String: string, options?: ActOptions): Promise<AgentHistory>;
+    /**
+     * Adds tools to the agent that allow it to interact with AI elements.
+     *
+     * @returns {Promise<void>} - A promise that resolves when the tools are added to the agent.
      */
-    act(goal: string, options?: {
-        chatId?: string;
-        agentHistory?: AgentHistory;
-    }): Promise<AgentHistory>;
+    addAIElementsToolsToAgent(): Promise<void>;
 }

package/dist/esm/execution/ui-control-client.js

@@ -16,6 +16,7 @@ import { UiControlClientDependencyBuilder } from './ui-control-client-dependency
 import { AIElementCollection } from '../core/ai-element/ai-element-collection';
 import { NoRetryStrategy } from './retry-strategies';
 import { AskUIAgent } from '../core/models/anthropic';
+import { AskUIGetAskUIElementTool, AskUIListAIElementTool } from '../core/models/anthropic/tools/askui-api-tools';
 export class UiControlClient extends ApiCommands {
     constructor(workspaceId, executionRuntime, stepReporter, aiElementArgs, agent) {
         super();
@@ -79,6 +80,22 @@ export class UiControlClient extends ApiCommands {
         return (this.stepReporter.config.withDetectedElements === 'onFailure' && error !== undefined)
             || (this.stepReporter.config.withDetectedElements === 'always');
     }
+    beforeNoneInferenceCallCommandExecution(instruction) {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.stepReporter.resetStep(instruction);
+            let annotation;
+            if (this.stepReporter.config.withDetectedElements === 'begin'
+                || this.stepReporter.config.withDetectedElements === 'always') {
+                annotation = yield this.executionRuntime.annotateImage();
+            }
+            const createdAt = new Date();
+            yield this.stepReporter.onStepBegin({
+                createdAt,
+                detectedElements: annotation === null || annotation === void 0 ? void 0 : annotation.detected_elements,
+                screenshot: annotation === null || annotation === void 0 ? void 0 : annotation.image,
+            });
+        });
+    }
     afterCommandExecution(instruction, error) {
         return __awaiter(this, void 0, void 0, function* () {
             var _a;
@@ -148,7 +165,7 @@ export class UiControlClient extends ApiCommands {
             ]);
             logger.debug(instruction);
             try {
-                yield this.stepReporter.resetStep(instruction);
+                this.stepReporter.resetStep(instruction);
                 yield this.executionRuntime.executeInstruction(instruction, modelComposition);
                 yield this.afterCommandExecution(instruction);
                 return yield Promise.resolve();
@@ -337,10 +354,14 @@ export class UiControlClient extends ApiCommands {
     // eslint-disable-next-line class-methods-use-this
     waitFor(delayInMs) {
         return {
-            exec() {
-                logger.debug(`Wait for ${delayInMs} ms`);
-                return new Promise((resolve) => { setTimeout(() => resolve(), delayInMs); });
-            },
+            exec: () => __awaiter(this, void 0, void 0, function* () {
+                const stepTitle = `Wait for ${delayInMs} ms`;
+                const instruction = yield this.buildInstruction(stepTitle, []);
+                yield this.beforeNoneInferenceCallCommandExecution(instruction);
+                yield new Promise((resolve) => { setTimeout(resolve, delayInMs); });
+                yield this.afterCommandExecution(instruction);
+                return Promise.resolve();
+            }),
         };
     }
     /**
@@ -736,55 +757,94 @@ export class UiControlClient extends ApiCommands {
         });
     }
     /**
-     * Instructs the agent to achieve a specified goal through autonomous actions.
+     * Holds down a key on the keyboard.
      *
-     * The agent will analyze the screen, determine necessary steps, and perform actions
-     * to accomplish the goal. This may include clicking, typing, scrolling, and other
-     * interface interactions.
+     * **Examples:**
+     * ```typescript
+     * await aui.keyDown('a').exec();
+     * ```
      *
-     * The `options` parameter allows the caller to maintain contextual continuity across
-     * multiple `act` calls, either from the same or different agent interfaces.
+     * @param {PC_AND_MODIFIER_KEY} key - The key to hold down.
+     */
+    keyDown(key) {
+        return {
+            exec: () => __awaiter(this, void 0, void 0, function* () {
+                const stepTitle = `Hold down key ${key}`;
+                const instruction = yield this.buildInstruction(stepTitle, []);
+                try {
+                    yield this.beforeNoneInferenceCallCommandExecution(instruction);
+                    yield this.agent.getOsAgentHandler().desktopKeyHoldDown(key, []);
+                    yield this.afterCommandExecution(instruction);
+                }
+                catch (error) {
+                    yield this.afterCommandExecution(instruction, error instanceof Error ? error : new Error(String(error)));
+                    return Promise.reject(error);
+                }
+                return Promise.resolve();
+            }),
+        };
+    }
+    /**
+     * Releases a key up that was previously held down.
      *
      * **Examples:**
-     *
-     * ```ts
-     * // Use chatId to maintain context across consecutive steps
-     * await aui.act("Search online for the current gold price", {
-     *   chatId: "session-gold-price"
-     * });
-     * await aui.act("Create a new text file and type the gold price result into it", {
-     *   chatId: "session-gold-price"
-     * });
-     *
-     * // Share history explicitly between separate agents (e.g., desktop and Android)
-     * // By default, the agent operates as a computer agent.
-     * // To control an Android device, you must configure it explicitly:
-     * await auiAndroid.agent.configureAsAndroidAgent();
-     * const history = await auiDesktop.act("Copy username from desktop app");
-     * await auiAndroid.act("Paste username into the mobile login screen", {
-     *   agentHistory: history
-     * });
+     * ```typescript
+     * await aui.keyUp('a').exec();
      * ```
      *
-     * @param {string} goal - A description of what the agent should achieve.
-     * @param {Object} [options] - Optional parameters to maintain or share context.
-     * @param {string} [options.chatId] - A session identifier used to persist memory between
-     *                                    consecutive `act` calls. When multiple actions share the
-     *                                    same `chatId`, the agent retains knowledge of prior steps,
-     *                                    such as extracted data or navigation history.
-     * @param {AgentHistory} [options.agentHistory] - A shared interaction history object that can be
-     *                                           passed between different agent clients (e.g., between
-     *                                           `auiDesktop` and `auiAndroid`) to ensure continuity
-     *                                           of understanding and task flow.
-     * @returns {Promise<AgentHistory>} - Updated action history after executing the goal.
-     * @throws {Error} If the agent is not connected when the method is called.
+     * @param {PC_AND_MODIFIER_KEY} key - The key to release up.
      */
-    act(goal, options) {
+    keyUp(key) {
+        return {
+            exec: () => __awaiter(this, void 0, void 0, function* () {
+                const stepTitle = `Release key ${key}`;
+                const instruction = yield this.buildInstruction(stepTitle, []);
+                try {
+                    yield this.beforeNoneInferenceCallCommandExecution(instruction);
+                    yield this.agent.getOsAgentHandler().desktopKeyRelease(key, []);
+                    yield this.afterCommandExecution(instruction);
+                }
+                catch (error) {
+                    yield this.afterCommandExecution(instruction, error instanceof Error ? error : new Error(String(error)));
+                    return Promise.reject(error);
+                }
+                return Promise.resolve();
+            }),
+        };
+    }
+    act(goal, imageOrOptions, options) {
         return __awaiter(this, void 0, void 0, function* () {
-            if (!this.agent.isConnected()) {
-                throw new Error('Agent is not connected, Please call connect() first');
+            if (typeof imageOrOptions === 'string') {
+                return this.agent.act(goal, imageOrOptions, options);
             }
-            return this.agent.act(goal, options);
+            const fullTitle = `Act: ${goal}`;
+            const stepTitle = fullTitle.length > 50 ? `${fullTitle.substring(0, 47)}...` : fullTitle;
+            const instruction = yield this.buildInstruction(stepTitle, []);
+            try {
+                yield this.beforeNoneInferenceCallCommandExecution(instruction);
+                const result = yield this.agent.act(goal, undefined, imageOrOptions);
+                yield this.afterCommandExecution(instruction);
+                return result;
+            }
+            catch (error) {
+                yield this.afterCommandExecution(instruction, error instanceof Error ? error : new Error(String(error)));
+                return Promise.reject(error);
+            }
+        });
+    }
+    /**
+     * Adds tools to the agent that allow it to interact with AI elements.
+     *
+     * @returns {Promise<void>} - A promise that resolves when the tools are added to the agent.
+     */
+    addAIElementsToolsToAgent() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const aiElementLocator = (aiElementName) => this.get().aiElement(aiElementName).exec();
+            const askUIGetAskUIElementTool = new AskUIGetAskUIElementTool(this.agent.getOsAgentHandler(), aiElementLocator, 'aiElement');
+            this.agent.addTool(askUIGetAskUIElementTool);
+            const listAIElementNamesFunction = () => (AIElementCollection.collectAIElements(this.workspaceId, this.aiElementArgs)).then((aiElementCollection) => aiElementCollection.getNames());
+            const askUIListAIElementTool = new AskUIListAIElementTool(listAIElementNamesFunction);
+            this.agent.addTool(askUIListAIElementTool);
         });
     }
 }

package/dist/esm/lib/interactive_cli/create-example-project.js

@@ -179,7 +179,7 @@ export class CreateExampleProject {
         return __awaiter(this, void 0, void 0, function* () {
             const runCommand = promisify(exec);
             const frameworkDependencies = {
-                jest: 'npm i -D @askui/askui-reporters typescript ts-node @types/jest ts-jest jest @askui/jest-allure-circus eslint @typescript-eslint/parser @typescript-eslint/eslint-plugin eslint-plugin-import @askui/eslint-plugin-askui hpagent',
+                jest: 'npm i -D @askui/askui-reporters typescript ts-node @types/jest@30.0.0 ts-jest@29.4.0 jest@29.7.0 @askui/jest-allure-circus eslint @typescript-eslint/parser @typescript-eslint/eslint-plugin eslint-plugin-import @askui/eslint-plugin-askui hpagent',
             };
             yield runCommand(frameworkDependencies.jest);
         });

package/dist/esm/main.d.ts

@@ -4,4 +4,4 @@ export { Instruction, Reporter, ReporterConfig, Snapshot, SnapshotDetailLevel, S
 export { Annotation } from './core/annotation/annotation';
 export { DetectedElement } from './core/model/annotation-result/detected-element';
 export { LogLevels } from './shared';
-export { ToolFailure, ToolError, BaseAgentTool } from './core/models/anthropic';
+export { ToolFailure, ToolError, BaseAgentTool, BetaTool, ToolResult, } from './core/models/anthropic';

package/dist/esm/main.js

@@ -3,4 +3,4 @@ export * from './execution';
 export { Annotation } from './core/annotation/annotation';
 export { DetectedElement } from './core/model/annotation-result/detected-element';
 export { LogLevels } from './shared';
-export { ToolFailure, ToolError, BaseAgentTool } from './core/models/anthropic';
+export { ToolFailure, ToolError, BaseAgentTool, } from './core/models/anthropic';

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "askui",
-    "version": "0.25.1",
+    "version": "0.27.0",
     "license": "MIT",
     "author": "askui GmbH <info@askui.com> (http://www.askui.com/)",
     "description": "Reliable, automated end-to-end-testing that depends on what is shown on your screen instead of the technology you are running on",
@@ -51,7 +51,7 @@
         "dist/example_projects_templates/"
     ],
     "dependencies": {
-        "@anthropic-ai/sdk": "0.52.0",
+        "@anthropic-ai/sdk": "0.53.0",
         "chalk": "4.1.1",
         "commander": "12.1.0",
         "fkill": "7.2.1",