askui 0.26.0 → 0.27.0

package/dist/cjs/core/ai-element/ai-element-collection.d.ts

@@ -8,5 +8,6 @@ export declare class AIElementCollection {
     static collectAIElements(workspaceId: string | undefined, aiElementArgs: AIElementArgs): Promise<AIElementCollection>;
     getByName(name: string): CustomElementJson[];
     getByNames(names: string[]): CustomElementJson[];
+    getNames(): string[];
     private static CollectAiElementsFromLocation;
 }

package/dist/cjs/core/ai-element/ai-element-collection.js

@@ -73,6 +73,9 @@ class AIElementCollection {
         }
         return names.flatMap((name) => this.getByName(name));
     }
+    getNames() {
+        return [...new Set(this.elements.map((element) => element.name))];
+    }
     static CollectAiElementsFromLocation(aiElementLocation) {
         const files = fs_extra_1.default.readdirSync(aiElementLocation);
         if (files.length === 0) {

package/dist/cjs/core/models/anthropic/askui-agent.d.ts

@@ -1,3 +1,4 @@
+import { OsAgentHandler } from './tools/os-agent-tools';
 import { ClaudeAgent } from './claude-agent';
 import { ExecutionRuntime } from '../../../execution/execution-runtime';
 export declare class AskUIAgent extends ClaudeAgent {
@@ -6,6 +7,7 @@ export declare class AskUIAgent extends ClaudeAgent {
     constructor(executionRuntime: ExecutionRuntime);
     isConnected(): boolean;
     initializeOsAgentHandler(): Promise<void>;
+    getOsAgentHandler(): OsAgentHandler;
     configureAsDesktopAgent(): Promise<void>;
     configureAsAndroidAgent(): Promise<void>;
     private static DesktopSystemPrompt;

package/dist/cjs/core/models/anthropic/askui-agent.js

@@ -26,6 +26,12 @@ class AskUIAgent extends claude_agent_1.ClaudeAgent {
             this.osAgentHandler = yield os_agent_tools_1.OsAgentHandler.createInstance(this.executionRuntime);
         });
     }
+    getOsAgentHandler() {
+        if (!this.osAgentHandler) {
+            throw new Error('Agent OS client is not connected');
+        }
+        return this.osAgentHandler;
+    }
     configureAsDesktopAgent() {
         return __awaiter(this, void 0, void 0, function* () {
             if (!this.osAgentHandler) {
@@ -33,6 +39,7 @@ class AskUIAgent extends claude_agent_1.ClaudeAgent {
             }
             const tools = [
                 new os_agent_tools_1.AgentErrorTool(),
+                new os_agent_tools_1.PrintTool(),
                 new os_agent_tools_1.ScreenShotTool(this.osAgentHandler),
                 new os_agent_tools_1.MouseMoveTool(this.osAgentHandler),
                 new os_agent_tools_1.MouseClickTool(this.osAgentHandler),

package/dist/cjs/core/models/anthropic/claude-agent.js

@@ -26,7 +26,7 @@ class ClaudeAgent {
         this.tools = [];
         this.history = {};
         this.toolChoice = {
-            type: 'any',
+            type: 'auto',
         };
     }
     setToolChoice(toolChoice) {

package/dist/cjs/core/models/anthropic/index.d.ts

@@ -1,3 +1,3 @@
 export { AskUIAgent } from './askui-agent';
-export { ToolFailure, ToolError, BaseAgentTool } from './tools/base';
+export { ToolFailure, ToolError, BaseAgentTool, BetaTool, ToolResult, } from './tools/base';
 export { AgentHistory, ActOptions } from './claude-agent';

package/dist/cjs/core/models/anthropic/tools/askui-api-tools.d.ts

@@ -0,0 +1,19 @@
+import { DetectedElement } from '../../../model/annotation-result/detected-element';
+import { BaseAgentTool, ToolResult, BetaTool } from './base';
+import { OsAgentHandler } from './os-agent-tools';
+export declare class AskUIGetAskUIElementTool extends BaseAgentTool {
+    private osAgentHandler;
+    private locatorFunction;
+    private elementType;
+    constructor(osAgentHandler: OsAgentHandler, locatorFunction: (aiElementName: string) => Promise<DetectedElement[]>, elementType: string);
+    execute(params: {
+        elementName: string;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class AskUIListAIElementTool extends BaseAgentTool {
+    private listFunction;
+    constructor(listFunction: () => Promise<string[]>);
+    execute(): Promise<ToolResult>;
+    toParams(): BetaTool;
+}

package/dist/cjs/core/models/anthropic/tools/askui-api-tools.js

@@ -0,0 +1,81 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AskUIListAIElementTool = exports.AskUIGetAskUIElementTool = void 0;
+const base_1 = require("./base");
+class AskUIGetAskUIElementTool extends base_1.BaseAgentTool {
+    constructor(osAgentHandler, locatorFunction, elementType) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+        this.locatorFunction = locatorFunction;
+        this.elementType = elementType;
+    }
+    execute(params) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const { elementName } = params;
+            const detectedElements = yield this.locatorFunction(elementName);
+            const scaledElementsBoundingBoxes = detectedElements.map((element) => {
+                const xMid = (element.bndbox.xmin + element.bndbox.xmax) / 2;
+                const yMid = (element.bndbox.ymin + element.bndbox.ymax) / 2;
+                const [x, y] = this.osAgentHandler.scaleCoordinates('computer', xMid, yMid);
+                return {
+                    x, y,
+                };
+            });
+            return {
+                output: `Found ${scaledElementsBoundingBoxes.length} elements of type ${this.elementType}. center coordinates: ${JSON.stringify(scaledElementsBoundingBoxes)}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            description: `Locates and retrieves the bounding box coordinates of AskUI ${this.elementType} elements on the screen. ` +
+                `This tool is essential for UI automation as it provides the exact pixel coordinates needed to interact with UI elements. ` +
+                `The coordinates returned  can be used for clicking, hovering, or other mouse interactions. ` +
+                `Use this tool when you need to find and interact with specific ${this.elementType} UI elements by their semantic names.`,
+            input_schema: {
+                properties: {
+                    elementName: {
+                        type: 'string',
+                        description: `The semantic name or identifier of the ${this.elementType} element to locate on the screen. `
+                    },
+                },
+                required: ['elementName'],
+                type: 'object',
+            },
+            name: `get_askui_${this.elementType}_element_tool`,
+        };
+    }
+}
+exports.AskUIGetAskUIElementTool = AskUIGetAskUIElementTool;
+class AskUIListAIElementTool extends base_1.BaseAgentTool {
+    constructor(listFunction) {
+        super();
+        this.listFunction = listFunction;
+    }
+    execute() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const elementNames = yield this.listFunction();
+            return {
+                output: `Found ${elementNames.length} element names that can be used to retrieve bounding boxes. Names: ${JSON.stringify(elementNames)}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            description: 'Retrieves a comprehensive list of all valid AskUI AI element names that can be used for element location and interaction. ' +
+                'The returned names can be used as input for the get_askui_aiElement_element_tool to locate specific ai elements. ',
+            input_schema: { type: 'object', properties: {}, required: [] },
+            name: 'list_ai_element_names_tool',
+        };
+    }
+}
+exports.AskUIListAIElementTool = AskUIListAIElementTool;

package/dist/cjs/core/models/anthropic/tools/base.d.ts

@@ -1,3 +1,5 @@
+import { BetaTool as AnthropicBetaTool } from '@anthropic-ai/sdk/resources/beta/messages/messages';
+export type BetaTool = AnthropicBetaTool;
 export interface ToolResult {
     output?: string;
     error?: string;

package/dist/cjs/core/models/anthropic/tools/os-agent-tools.d.ts

@@ -24,7 +24,7 @@ export declare class OsAgentHandler {
     };
     setTargetResolution(width: number, height: number): void;
     takeScreenshot(): Promise<string>;
-    private scaleCoordinates;
+    scaleCoordinates(source: 'api' | 'computer', x: number, y: number): [number, number];
     requestControl(controlCommand: ControlCommand): Promise<void>;
     mouseMove(x: number, y: number): Promise<void>;
     mouseClick(button: "left" | "right" | "middle", doubleClick: boolean): Promise<void>;
@@ -35,6 +35,9 @@ export declare class OsAgentHandler {
     desktopKeyHoldDown(key: PC_AND_MODIFIER_KEY, modifiers?: MODIFIER_KEY[]): Promise<void>;
     desktopKeyRelease(key: PC_AND_MODIFIER_KEY, modifiers?: MODIFIER_KEY[]): Promise<void>;
     typeText(text: string): Promise<void>;
+    androidKeyPress(key: ANDROID_KEY): Promise<void>;
+    androidKeySequencePress(keys: ANDROID_KEY[]): Promise<void>;
+    executeShellCommand(command: string): Promise<void>;
 }
 export declare class ScreenShotTool extends BaseAgentTool {
     private osAgentHandler;
@@ -165,3 +168,10 @@ export declare class WaitTool extends BaseAgentTool {
     }): Promise<ToolResult>;
     toParams(): BetaTool;
 }
+export declare class PrintTool extends BaseAgentTool {
+    constructor();
+    execute(command: {
+        text: string;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}

package/dist/cjs/core/models/anthropic/tools/os-agent-tools.js

@@ -9,7 +9,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.WaitTool = exports.ExecuteShellCommandTool = exports.AgentErrorTool = exports.AndroidSequenceKeyPressTool = exports.AndroidSingleKeyPressTool = exports.TypeTool = exports.DesktopKeyReleaseTool = exports.DesktopKeyHoldDownTool = exports.DesktopPressAndReleaseKeysTool = exports.MouseReleaseLeftButtonTool = exports.MouseHoldLeftButtonDownTool = exports.MouseDragAndDropTool = exports.MouseScrollTool = exports.MouseClickTool = exports.MouseMoveTool = exports.ScreenShotTool = exports.OsAgentHandler = void 0;
+exports.PrintTool = exports.WaitTool = exports.ExecuteShellCommandTool = exports.AgentErrorTool = exports.AndroidSequenceKeyPressTool = exports.AndroidSingleKeyPressTool = exports.TypeTool = exports.DesktopKeyReleaseTool = exports.DesktopKeyHoldDownTool = exports.DesktopPressAndReleaseKeysTool = exports.MouseReleaseLeftButtonTool = exports.MouseHoldLeftButtonDownTool = exports.MouseDragAndDropTool = exports.MouseScrollTool = exports.MouseClickTool = exports.MouseMoveTool = exports.ScreenShotTool = exports.OsAgentHandler = void 0;
 const dsl_1 = require("../../../../execution/dsl");
 const base_1 = require("./base");
 const ui_control_commands_1 = require("../../../ui-control-commands");
@@ -205,6 +205,24 @@ class OsAgentHandler {
             yield this.requestControl(controlCommand);
         });
     }
+    androidKeyPress(key) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.PRESS_ANDROID_SINGLE_KEY, { x: 0, y: 0 }, key, {})]);
+            yield this.requestControl(controlCommand);
+        });
+    }
+    androidKeySequencePress(keys) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.PRESS_ANDROID_KEY_SEQUENCE, { x: 0, y: 0 }, keys.join(' '), {})]);
+            yield this.requestControl(controlCommand);
+        });
+    }
+    executeShellCommand(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.EXECUTE_COMMAND, { x: 0, y: 0 }, command, {})]);
+            yield this.requestControl(controlCommand);
+        });
+    }
 }
 exports.OsAgentHandler = OsAgentHandler;
 class ScreenShotTool extends base_1.BaseAgentTool {
@@ -556,8 +574,7 @@ class TypeTool extends base_1.BaseAgentTool {
     }
     execute(command) {
         return __awaiter(this, void 0, void 0, function* () {
-            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.TYPE, { x: 0, y: 0 }, command.text, {})]);
-            yield this.osAgentHandler.requestControl(controlCommand);
+            yield this.osAgentHandler.typeText(command.text);
             return {
                 output: `Typed text: ${command.text}`,
             };
@@ -588,8 +605,7 @@ class AndroidSingleKeyPressTool extends base_1.BaseAgentTool {
     }
     execute(command) {
         return __awaiter(this, void 0, void 0, function* () {
-            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.PRESS_KEY_SEQUENCE, { x: 0, y: 0 }, command.key, {})]);
-            yield this.osAgentHandler.requestControl(controlCommand);
+            yield this.osAgentHandler.androidKeyPress(command.key);
             return {
                 output: `Pressed Android key ${command.key}`,
             };
@@ -621,8 +637,7 @@ class AndroidSequenceKeyPressTool extends base_1.BaseAgentTool {
     }
     execute(command) {
         return __awaiter(this, void 0, void 0, function* () {
-            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.PRESS_KEY_SEQUENCE, { x: 0, y: 0 }, command.keys.join(' '), {})]);
-            yield this.osAgentHandler.requestControl(controlCommand);
+            yield this.osAgentHandler.androidKeySequencePress(command.keys);
             return {
                 output: `Pressed Android keys: ${command.keys.join(', ')}`,
             };
@@ -662,7 +677,7 @@ class AgentErrorTool extends base_1.BaseAgentTool {
     toParams() {
         return {
             name: 'agent_error_tool',
-            description: 'Raises an error in the agent',
+            description: 'Intentionally raises an error to signal that the agent cannot proceed with the current task. Use this when the agent encounters an unsolvable problem, gets stuck in a loop, or needs to communicate a critical failure that prevents further automation.',
             input_schema: {
                 type: 'object',
                 properties: {
@@ -684,8 +699,7 @@ class ExecuteShellCommandTool extends base_1.BaseAgentTool {
     }
     execute(command) {
         return __awaiter(this, void 0, void 0, function* () {
-            const controlCommand = new ui_control_commands_1.ControlCommand(ui_control_commands_1.ControlCommandCode.OK, [new ui_control_commands_1.Action(ui_control_commands_1.InputEvent.EXECUTE_COMMAND, { x: 0, y: 0 }, command.command, {})]);
-            yield this.osAgentHandler.requestControl(controlCommand);
+            yield this.osAgentHandler.executeShellCommand(command.command);
             return {
                 output: `Executed shell command: ${command.command}`,
             };
@@ -739,3 +753,33 @@ class WaitTool extends base_1.BaseAgentTool {
     }
 }
 exports.WaitTool = WaitTool;
+class PrintTool extends base_1.BaseAgentTool {
+    constructor() {
+        super();
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            console.log(command.text);
+            return {
+                output: `Printed text: ${command.text}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'print_tool',
+            description: 'Outputs text to the console for debugging, status updates, or user communication. Useful for providing feedback about automation progress, errors, or important information during test execution.',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    text: {
+                        type: 'string',
+                        description: 'The text to output to the console.',
+                    },
+                },
+                required: ['text'],
+            },
+        };
+    }
+}
+exports.PrintTool = PrintTool;

package/dist/cjs/execution/ui-control-client.d.ts

@@ -55,6 +55,7 @@ export declare class UiControlClient extends ApiCommands {
     stopVideoRecording(): Promise<void>;
     readVideoRecording(): Promise<string>;
     private shouldAnnotateAfterCommandExecution;
+    private beforeNoneInferenceCallCommandExecution;
     private afterCommandExecution;
     annotate(annotationRequest?: AnnotationRequest): Promise<Annotation>;
     annotateInteractively(): Promise<void>;
@@ -475,6 +476,28 @@ export declare class UiControlClient extends ApiCommands {
      * @returns {ExpectAllExistResult.elements} - ExpectExistenceElement[].
      */
     expectAllExist(query: ElementExistsQuery[]): Promise<ExpectAllExistResult>;
+    /**
+     * Holds down a key on the keyboard.
+     *
+     * **Examples:**
+     * ```typescript
+     * await aui.keyDown('a').exec();
+     * ```
+     *
+     * @param {PC_AND_MODIFIER_KEY} key - The key to hold down.
+     */
+    keyDown(key: PC_AND_MODIFIER_KEY): Executable;
+    /**
+     * Releases a key up that was previously held down.
+     *
+     * **Examples:**
+     * ```typescript
+     * await aui.keyUp('a').exec();
+     * ```
+     *
+     * @param {PC_AND_MODIFIER_KEY} key - The key to release up.
+     */
+    keyUp(key: PC_AND_MODIFIER_KEY): Executable;
     /**
      * Instructs the agent to autonomously achieve a specified goal through UI interactions.
      *
@@ -573,4 +596,10 @@ export declare class UiControlClient extends ApiCommands {
      */
     act(goal: string, options?: ActOptions): Promise<AgentHistory>;
     act(goal: string, imagePathOrBase64String: string, options?: ActOptions): Promise<AgentHistory>;
+    /**
+     * Adds tools to the agent that allow it to interact with AI elements.
+     *
+     * @returns {Promise<void>} - A promise that resolves when the tools are added to the agent.
+     */
+    addAIElementsToolsToAgent(): Promise<void>;
 }

package/dist/cjs/execution/ui-control-client.js

@@ -19,6 +19,7 @@ const ui_control_client_dependency_builder_1 = require("./ui-control-client-depe
 const ai_element_collection_1 = require("../core/ai-element/ai-element-collection");
 const retry_strategies_1 = require("./retry-strategies");
 const anthropic_1 = require("../core/models/anthropic");
+const askui_api_tools_1 = require("../core/models/anthropic/tools/askui-api-tools");
 class UiControlClient extends dsl_1.ApiCommands {
     constructor(workspaceId, executionRuntime, stepReporter, aiElementArgs, agent) {
         super();
@@ -82,6 +83,22 @@ class UiControlClient extends dsl_1.ApiCommands {
         return (this.stepReporter.config.withDetectedElements === 'onFailure' && error !== undefined)
             || (this.stepReporter.config.withDetectedElements === 'always');
     }
+    beforeNoneInferenceCallCommandExecution(instruction) {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.stepReporter.resetStep(instruction);
+            let annotation;
+            if (this.stepReporter.config.withDetectedElements === 'begin'
+                || this.stepReporter.config.withDetectedElements === 'always') {
+                annotation = yield this.executionRuntime.annotateImage();
+            }
+            const createdAt = new Date();
+            yield this.stepReporter.onStepBegin({
+                createdAt,
+                detectedElements: annotation === null || annotation === void 0 ? void 0 : annotation.detected_elements,
+                screenshot: annotation === null || annotation === void 0 ? void 0 : annotation.image,
+            });
+        });
+    }
     afterCommandExecution(instruction, error) {
         return __awaiter(this, void 0, void 0, function* () {
             var _a;
@@ -151,7 +168,7 @@ class UiControlClient extends dsl_1.ApiCommands {
             ]);
             logger_1.logger.debug(instruction);
             try {
-                yield this.stepReporter.resetStep(instruction);
+                this.stepReporter.resetStep(instruction);
                 yield this.executionRuntime.executeInstruction(instruction, modelComposition);
                 yield this.afterCommandExecution(instruction);
                 return yield Promise.resolve();
@@ -340,10 +357,14 @@ class UiControlClient extends dsl_1.ApiCommands {
     // eslint-disable-next-line class-methods-use-this
     waitFor(delayInMs) {
         return {
-            exec() {
-                logger_1.logger.debug(`Wait for ${delayInMs} ms`);
-                return new Promise((resolve) => { setTimeout(() => resolve(), delayInMs); });
-            },
+            exec: () => __awaiter(this, void 0, void 0, function* () {
+                const stepTitle = `Wait for ${delayInMs} ms`;
+                const instruction = yield this.buildInstruction(stepTitle, []);
+                yield this.beforeNoneInferenceCallCommandExecution(instruction);
+                yield new Promise((resolve) => { setTimeout(resolve, delayInMs); });
+                yield this.afterCommandExecution(instruction);
+                return Promise.resolve();
+            }),
         };
     }
     /**
@@ -738,12 +759,95 @@ class UiControlClient extends dsl_1.ApiCommands {
             };
         });
     }
+    /**
+     * Holds down a key on the keyboard.
+     *
+     * **Examples:**
+     * ```typescript
+     * await aui.keyDown('a').exec();
+     * ```
+     *
+     * @param {PC_AND_MODIFIER_KEY} key - The key to hold down.
+     */
+    keyDown(key) {
+        return {
+            exec: () => __awaiter(this, void 0, void 0, function* () {
+                const stepTitle = `Hold down key ${key}`;
+                const instruction = yield this.buildInstruction(stepTitle, []);
+                try {
+                    yield this.beforeNoneInferenceCallCommandExecution(instruction);
+                    yield this.agent.getOsAgentHandler().desktopKeyHoldDown(key, []);
+                    yield this.afterCommandExecution(instruction);
+                }
+                catch (error) {
+                    yield this.afterCommandExecution(instruction, error instanceof Error ? error : new Error(String(error)));
+                    return Promise.reject(error);
+                }
+                return Promise.resolve();
+            }),
+        };
+    }
+    /**
+     * Releases a key up that was previously held down.
+     *
+     * **Examples:**
+     * ```typescript
+     * await aui.keyUp('a').exec();
+     * ```
+     *
+     * @param {PC_AND_MODIFIER_KEY} key - The key to release up.
+     */
+    keyUp(key) {
+        return {
+            exec: () => __awaiter(this, void 0, void 0, function* () {
+                const stepTitle = `Release key ${key}`;
+                const instruction = yield this.buildInstruction(stepTitle, []);
+                try {
+                    yield this.beforeNoneInferenceCallCommandExecution(instruction);
+                    yield this.agent.getOsAgentHandler().desktopKeyRelease(key, []);
+                    yield this.afterCommandExecution(instruction);
+                }
+                catch (error) {
+                    yield this.afterCommandExecution(instruction, error instanceof Error ? error : new Error(String(error)));
+                    return Promise.reject(error);
+                }
+                return Promise.resolve();
+            }),
+        };
+    }
     act(goal, imageOrOptions, options) {
         return __awaiter(this, void 0, void 0, function* () {
             if (typeof imageOrOptions === 'string') {
                 return this.agent.act(goal, imageOrOptions, options);
             }
-            return this.agent.act(goal, undefined, imageOrOptions);
+            const fullTitle = `Act: ${goal}`;
+            const stepTitle = fullTitle.length > 50 ? `${fullTitle.substring(0, 47)}...` : fullTitle;
+            const instruction = yield this.buildInstruction(stepTitle, []);
+            try {
+                yield this.beforeNoneInferenceCallCommandExecution(instruction);
+                const result = yield this.agent.act(goal, undefined, imageOrOptions);
+                yield this.afterCommandExecution(instruction);
+                return result;
+            }
+            catch (error) {
+                yield this.afterCommandExecution(instruction, error instanceof Error ? error : new Error(String(error)));
+                return Promise.reject(error);
+            }
+        });
+    }
+    /**
+     * Adds tools to the agent that allow it to interact with AI elements.
+     *
+     * @returns {Promise<void>} - A promise that resolves when the tools are added to the agent.
+     */
+    addAIElementsToolsToAgent() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const aiElementLocator = (aiElementName) => this.get().aiElement(aiElementName).exec();
+            const askUIGetAskUIElementTool = new askui_api_tools_1.AskUIGetAskUIElementTool(this.agent.getOsAgentHandler(), aiElementLocator, 'aiElement');
+            this.agent.addTool(askUIGetAskUIElementTool);
+            const listAIElementNamesFunction = () => (ai_element_collection_1.AIElementCollection.collectAIElements(this.workspaceId, this.aiElementArgs)).then((aiElementCollection) => aiElementCollection.getNames());
+            const askUIListAIElementTool = new askui_api_tools_1.AskUIListAIElementTool(listAIElementNamesFunction);
+            this.agent.addTool(askUIListAIElementTool);
         });
     }
 }

package/dist/cjs/lib/interactive_cli/create-example-project.js

@@ -185,7 +185,7 @@ class CreateExampleProject {
         return __awaiter(this, void 0, void 0, function* () {
             const runCommand = (0, util_1.promisify)(child_process_1.exec);
             const frameworkDependencies = {
-                jest: 'npm i -D @askui/askui-reporters typescript ts-node @types/jest ts-jest jest @askui/jest-allure-circus eslint @typescript-eslint/parser @typescript-eslint/eslint-plugin eslint-plugin-import @askui/eslint-plugin-askui hpagent',
+                jest: 'npm i -D @askui/askui-reporters typescript ts-node @types/jest@30.0.0 ts-jest@29.4.0 jest@29.7.0 @askui/jest-allure-circus eslint @typescript-eslint/parser @typescript-eslint/eslint-plugin eslint-plugin-import @askui/eslint-plugin-askui hpagent',
             };
             yield runCommand(frameworkDependencies.jest);
         });

package/dist/cjs/main.d.ts

@@ -4,4 +4,4 @@ export { Instruction, Reporter, ReporterConfig, Snapshot, SnapshotDetailLevel, S
 export { Annotation } from './core/annotation/annotation';
 export { DetectedElement } from './core/model/annotation-result/detected-element';
 export { LogLevels } from './shared';
-export { ToolFailure, ToolError, BaseAgentTool } from './core/models/anthropic';
+export { ToolFailure, ToolError, BaseAgentTool, BetaTool, ToolResult, } from './core/models/anthropic';

package/dist/esm/core/ai-element/ai-element-collection.d.ts

@@ -8,5 +8,6 @@ export declare class AIElementCollection {
     static collectAIElements(workspaceId: string | undefined, aiElementArgs: AIElementArgs): Promise<AIElementCollection>;
     getByName(name: string): CustomElementJson[];
     getByNames(names: string[]): CustomElementJson[];
+    getNames(): string[];
     private static CollectAiElementsFromLocation;
 }

package/dist/esm/core/ai-element/ai-element-collection.js

@@ -67,6 +67,9 @@ export class AIElementCollection {
         }
         return names.flatMap((name) => this.getByName(name));
     }
+    getNames() {
+        return [...new Set(this.elements.map((element) => element.name))];
+    }
     static CollectAiElementsFromLocation(aiElementLocation) {
         const files = fs.readdirSync(aiElementLocation);
         if (files.length === 0) {

package/dist/esm/core/models/anthropic/askui-agent.d.ts

@@ -1,3 +1,4 @@
+import { OsAgentHandler } from './tools/os-agent-tools';
 import { ClaudeAgent } from './claude-agent';
 import { ExecutionRuntime } from '../../../execution/execution-runtime';
 export declare class AskUIAgent extends ClaudeAgent {
@@ -6,6 +7,7 @@ export declare class AskUIAgent extends ClaudeAgent {
     constructor(executionRuntime: ExecutionRuntime);
     isConnected(): boolean;
     initializeOsAgentHandler(): Promise<void>;
+    getOsAgentHandler(): OsAgentHandler;
     configureAsDesktopAgent(): Promise<void>;
     configureAsAndroidAgent(): Promise<void>;
     private static DesktopSystemPrompt;

package/dist/esm/core/models/anthropic/askui-agent.js

@@ -7,7 +7,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-import { DesktopPressAndReleaseKeysTool, MouseClickTool, MouseMoveTool, MouseScrollTool, OsAgentHandler, ScreenShotTool, TypeTool, AgentErrorTool, AndroidSequenceKeyPressTool, AndroidSingleKeyPressTool, ExecuteShellCommandTool, DesktopKeyHoldDownTool, DesktopKeyReleaseTool, MouseReleaseLeftButtonTool, MouseHoldLeftButtonDownTool, MouseDragAndDropTool, WaitTool, } from './tools/os-agent-tools';
+import { DesktopPressAndReleaseKeysTool, MouseClickTool, MouseMoveTool, MouseScrollTool, OsAgentHandler, ScreenShotTool, TypeTool, AgentErrorTool, AndroidSequenceKeyPressTool, AndroidSingleKeyPressTool, ExecuteShellCommandTool, DesktopKeyHoldDownTool, DesktopKeyReleaseTool, MouseReleaseLeftButtonTool, MouseHoldLeftButtonDownTool, MouseDragAndDropTool, WaitTool, PrintTool, } from './tools/os-agent-tools';
 import { ClaudeAgent } from './claude-agent';
 export class AskUIAgent extends ClaudeAgent {
     constructor(executionRuntime) {
@@ -23,6 +23,12 @@ export class AskUIAgent extends ClaudeAgent {
             this.osAgentHandler = yield OsAgentHandler.createInstance(this.executionRuntime);
         });
     }
+    getOsAgentHandler() {
+        if (!this.osAgentHandler) {
+            throw new Error('Agent OS client is not connected');
+        }
+        return this.osAgentHandler;
+    }
     configureAsDesktopAgent() {
         return __awaiter(this, void 0, void 0, function* () {
             if (!this.osAgentHandler) {
@@ -30,6 +36,7 @@ export class AskUIAgent extends ClaudeAgent {
             }
             const tools = [
                 new AgentErrorTool(),
+                new PrintTool(),
                 new ScreenShotTool(this.osAgentHandler),
                 new MouseMoveTool(this.osAgentHandler),
                 new MouseClickTool(this.osAgentHandler),

package/dist/esm/core/models/anthropic/claude-agent.js

@@ -23,7 +23,7 @@ export class ClaudeAgent {
         this.tools = [];
         this.history = {};
         this.toolChoice = {
-            type: 'any',
+            type: 'auto',
         };
     }
     setToolChoice(toolChoice) {

package/dist/esm/core/models/anthropic/index.d.ts

@@ -1,3 +1,3 @@
 export { AskUIAgent } from './askui-agent';
-export { ToolFailure, ToolError, BaseAgentTool } from './tools/base';
+export { ToolFailure, ToolError, BaseAgentTool, BetaTool, ToolResult, } from './tools/base';
 export { AgentHistory, ActOptions } from './claude-agent';

package/dist/esm/core/models/anthropic/index.js

@@ -1,2 +1,2 @@
 export { AskUIAgent } from './askui-agent';
-export { ToolFailure, ToolError, BaseAgentTool } from './tools/base';
+export { ToolFailure, ToolError, BaseAgentTool, } from './tools/base';

package/dist/esm/core/models/anthropic/tools/askui-api-tools.d.ts

@@ -0,0 +1,19 @@
+import { DetectedElement } from '../../../model/annotation-result/detected-element';
+import { BaseAgentTool, ToolResult, BetaTool } from './base';
+import { OsAgentHandler } from './os-agent-tools';
+export declare class AskUIGetAskUIElementTool extends BaseAgentTool {
+    private osAgentHandler;
+    private locatorFunction;
+    private elementType;
+    constructor(osAgentHandler: OsAgentHandler, locatorFunction: (aiElementName: string) => Promise<DetectedElement[]>, elementType: string);
+    execute(params: {
+        elementName: string;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class AskUIListAIElementTool extends BaseAgentTool {
+    private listFunction;
+    constructor(listFunction: () => Promise<string[]>);
+    execute(): Promise<ToolResult>;
+    toParams(): BetaTool;
+}

package/dist/esm/core/models/anthropic/tools/askui-api-tools.js

@@ -0,0 +1,76 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { BaseAgentTool } from './base';
+export class AskUIGetAskUIElementTool extends BaseAgentTool {
+    constructor(osAgentHandler, locatorFunction, elementType) {
+        super();
+        this.osAgentHandler = osAgentHandler;
+        this.locatorFunction = locatorFunction;
+        this.elementType = elementType;
+    }
+    execute(params) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const { elementName } = params;
+            const detectedElements = yield this.locatorFunction(elementName);
+            const scaledElementsBoundingBoxes = detectedElements.map((element) => {
+                const xMid = (element.bndbox.xmin + element.bndbox.xmax) / 2;
+                const yMid = (element.bndbox.ymin + element.bndbox.ymax) / 2;
+                const [x, y] = this.osAgentHandler.scaleCoordinates('computer', xMid, yMid);
+                return {
+                    x, y,
+                };
+            });
+            return {
+                output: `Found ${scaledElementsBoundingBoxes.length} elements of type ${this.elementType}. center coordinates: ${JSON.stringify(scaledElementsBoundingBoxes)}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            description: `Locates and retrieves the bounding box coordinates of AskUI ${this.elementType} elements on the screen. ` +
+                `This tool is essential for UI automation as it provides the exact pixel coordinates needed to interact with UI elements. ` +
+                `The coordinates returned  can be used for clicking, hovering, or other mouse interactions. ` +
+                `Use this tool when you need to find and interact with specific ${this.elementType} UI elements by their semantic names.`,
+            input_schema: {
+                properties: {
+                    elementName: {
+                        type: 'string',
+                        description: `The semantic name or identifier of the ${this.elementType} element to locate on the screen. `
+                    },
+                },
+                required: ['elementName'],
+                type: 'object',
+            },
+            name: `get_askui_${this.elementType}_element_tool`,
+        };
+    }
+}
+export class AskUIListAIElementTool extends BaseAgentTool {
+    constructor(listFunction) {
+        super();
+        this.listFunction = listFunction;
+    }
+    execute() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const elementNames = yield this.listFunction();
+            return {
+                output: `Found ${elementNames.length} element names that can be used to retrieve bounding boxes. Names: ${JSON.stringify(elementNames)}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            description: 'Retrieves a comprehensive list of all valid AskUI AI element names that can be used for element location and interaction. ' +
+                'The returned names can be used as input for the get_askui_aiElement_element_tool to locate specific ai elements. ',
+            input_schema: { type: 'object', properties: {}, required: [] },
+            name: 'list_ai_element_names_tool',
+        };
+    }
+}

package/dist/esm/core/models/anthropic/tools/base.d.ts

@@ -1,3 +1,5 @@
+import { BetaTool as AnthropicBetaTool } from '@anthropic-ai/sdk/resources/beta/messages/messages';
+export type BetaTool = AnthropicBetaTool;
 export interface ToolResult {
     output?: string;
     error?: string;

package/dist/esm/core/models/anthropic/tools/os-agent-tools.d.ts

@@ -24,7 +24,7 @@ export declare class OsAgentHandler {
     };
     setTargetResolution(width: number, height: number): void;
     takeScreenshot(): Promise<string>;
-    private scaleCoordinates;
+    scaleCoordinates(source: 'api' | 'computer', x: number, y: number): [number, number];
     requestControl(controlCommand: ControlCommand): Promise<void>;
     mouseMove(x: number, y: number): Promise<void>;
     mouseClick(button: "left" | "right" | "middle", doubleClick: boolean): Promise<void>;
@@ -35,6 +35,9 @@ export declare class OsAgentHandler {
     desktopKeyHoldDown(key: PC_AND_MODIFIER_KEY, modifiers?: MODIFIER_KEY[]): Promise<void>;
     desktopKeyRelease(key: PC_AND_MODIFIER_KEY, modifiers?: MODIFIER_KEY[]): Promise<void>;
     typeText(text: string): Promise<void>;
+    androidKeyPress(key: ANDROID_KEY): Promise<void>;
+    androidKeySequencePress(keys: ANDROID_KEY[]): Promise<void>;
+    executeShellCommand(command: string): Promise<void>;
 }
 export declare class ScreenShotTool extends BaseAgentTool {
     private osAgentHandler;
@@ -165,3 +168,10 @@ export declare class WaitTool extends BaseAgentTool {
     }): Promise<ToolResult>;
     toParams(): BetaTool;
 }
+export declare class PrintTool extends BaseAgentTool {
+    constructor();
+    execute(command: {
+        text: string;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}

package/dist/esm/core/models/anthropic/tools/os-agent-tools.js

@@ -202,6 +202,24 @@ export class OsAgentHandler {
             yield this.requestControl(controlCommand);
         });
     }
+    androidKeyPress(key) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.PRESS_ANDROID_SINGLE_KEY, { x: 0, y: 0 }, key, {})]);
+            yield this.requestControl(controlCommand);
+        });
+    }
+    androidKeySequencePress(keys) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.PRESS_ANDROID_KEY_SEQUENCE, { x: 0, y: 0 }, keys.join(' '), {})]);
+            yield this.requestControl(controlCommand);
+        });
+    }
+    executeShellCommand(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.EXECUTE_COMMAND, { x: 0, y: 0 }, command, {})]);
+            yield this.requestControl(controlCommand);
+        });
+    }
 }
 export class ScreenShotTool extends BaseAgentTool {
     constructor(osAgentHandler) {
@@ -542,8 +560,7 @@ export class TypeTool extends BaseAgentTool {
     }
     execute(command) {
         return __awaiter(this, void 0, void 0, function* () {
-            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.TYPE, { x: 0, y: 0 }, command.text, {})]);
-            yield this.osAgentHandler.requestControl(controlCommand);
+            yield this.osAgentHandler.typeText(command.text);
             return {
                 output: `Typed text: ${command.text}`,
             };
@@ -573,8 +590,7 @@ export class AndroidSingleKeyPressTool extends BaseAgentTool {
     }
     execute(command) {
         return __awaiter(this, void 0, void 0, function* () {
-            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.PRESS_KEY_SEQUENCE, { x: 0, y: 0 }, command.key, {})]);
-            yield this.osAgentHandler.requestControl(controlCommand);
+            yield this.osAgentHandler.androidKeyPress(command.key);
             return {
                 output: `Pressed Android key ${command.key}`,
             };
@@ -605,8 +621,7 @@ export class AndroidSequenceKeyPressTool extends BaseAgentTool {
     }
     execute(command) {
         return __awaiter(this, void 0, void 0, function* () {
-            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.PRESS_KEY_SEQUENCE, { x: 0, y: 0 }, command.keys.join(' '), {})]);
-            yield this.osAgentHandler.requestControl(controlCommand);
+            yield this.osAgentHandler.androidKeySequencePress(command.keys);
             return {
                 output: `Pressed Android keys: ${command.keys.join(', ')}`,
             };
@@ -645,7 +660,7 @@ export class AgentErrorTool extends BaseAgentTool {
     toParams() {
         return {
             name: 'agent_error_tool',
-            description: 'Raises an error in the agent',
+            description: 'Intentionally raises an error to signal that the agent cannot proceed with the current task. Use this when the agent encounters an unsolvable problem, gets stuck in a loop, or needs to communicate a critical failure that prevents further automation.',
             input_schema: {
                 type: 'object',
                 properties: {
@@ -666,8 +681,7 @@ export class ExecuteShellCommandTool extends BaseAgentTool {
     }
     execute(command) {
         return __awaiter(this, void 0, void 0, function* () {
-            const controlCommand = new ControlCommand(ControlCommandCode.OK, [new Action(InputEvent.EXECUTE_COMMAND, { x: 0, y: 0 }, command.command, {})]);
-            yield this.osAgentHandler.requestControl(controlCommand);
+            yield this.osAgentHandler.executeShellCommand(command.command);
             return {
                 output: `Executed shell command: ${command.command}`,
             };
@@ -719,3 +733,32 @@ export class WaitTool extends BaseAgentTool {
         };
     }
 }
+export class PrintTool extends BaseAgentTool {
+    constructor() {
+        super();
+    }
+    execute(command) {
+        return __awaiter(this, void 0, void 0, function* () {
+            console.log(command.text);
+            return {
+                output: `Printed text: ${command.text}`,
+            };
+        });
+    }
+    toParams() {
+        return {
+            name: 'print_tool',
+            description: 'Outputs text to the console for debugging, status updates, or user communication. Useful for providing feedback about automation progress, errors, or important information during test execution.',
+            input_schema: {
+                type: 'object',
+                properties: {
+                    text: {
+                        type: 'string',
+                        description: 'The text to output to the console.',
+                    },
+                },
+                required: ['text'],
+            },
+        };
+    }
+}

package/dist/esm/execution/ui-control-client.d.ts

@@ -55,6 +55,7 @@ export declare class UiControlClient extends ApiCommands {
     stopVideoRecording(): Promise<void>;
     readVideoRecording(): Promise<string>;
     private shouldAnnotateAfterCommandExecution;
+    private beforeNoneInferenceCallCommandExecution;
     private afterCommandExecution;
     annotate(annotationRequest?: AnnotationRequest): Promise<Annotation>;
     annotateInteractively(): Promise<void>;
@@ -475,6 +476,28 @@ export declare class UiControlClient extends ApiCommands {
      * @returns {ExpectAllExistResult.elements} - ExpectExistenceElement[].
      */
     expectAllExist(query: ElementExistsQuery[]): Promise<ExpectAllExistResult>;
+    /**
+     * Holds down a key on the keyboard.
+     *
+     * **Examples:**
+     * ```typescript
+     * await aui.keyDown('a').exec();
+     * ```
+     *
+     * @param {PC_AND_MODIFIER_KEY} key - The key to hold down.
+     */
+    keyDown(key: PC_AND_MODIFIER_KEY): Executable;
+    /**
+     * Releases a key up that was previously held down.
+     *
+     * **Examples:**
+     * ```typescript
+     * await aui.keyUp('a').exec();
+     * ```
+     *
+     * @param {PC_AND_MODIFIER_KEY} key - The key to release up.
+     */
+    keyUp(key: PC_AND_MODIFIER_KEY): Executable;
     /**
      * Instructs the agent to autonomously achieve a specified goal through UI interactions.
      *
@@ -573,4 +596,10 @@ export declare class UiControlClient extends ApiCommands {
      */
     act(goal: string, options?: ActOptions): Promise<AgentHistory>;
     act(goal: string, imagePathOrBase64String: string, options?: ActOptions): Promise<AgentHistory>;
+    /**
+     * Adds tools to the agent that allow it to interact with AI elements.
+     *
+     * @returns {Promise<void>} - A promise that resolves when the tools are added to the agent.
+     */
+    addAIElementsToolsToAgent(): Promise<void>;
 }

package/dist/esm/execution/ui-control-client.js

@@ -16,6 +16,7 @@ import { UiControlClientDependencyBuilder } from './ui-control-client-dependency
 import { AIElementCollection } from '../core/ai-element/ai-element-collection';
 import { NoRetryStrategy } from './retry-strategies';
 import { AskUIAgent } from '../core/models/anthropic';
+import { AskUIGetAskUIElementTool, AskUIListAIElementTool } from '../core/models/anthropic/tools/askui-api-tools';
 export class UiControlClient extends ApiCommands {
     constructor(workspaceId, executionRuntime, stepReporter, aiElementArgs, agent) {
         super();
@@ -79,6 +80,22 @@ export class UiControlClient extends ApiCommands {
         return (this.stepReporter.config.withDetectedElements === 'onFailure' && error !== undefined)
             || (this.stepReporter.config.withDetectedElements === 'always');
     }
+    beforeNoneInferenceCallCommandExecution(instruction) {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.stepReporter.resetStep(instruction);
+            let annotation;
+            if (this.stepReporter.config.withDetectedElements === 'begin'
+                || this.stepReporter.config.withDetectedElements === 'always') {
+                annotation = yield this.executionRuntime.annotateImage();
+            }
+            const createdAt = new Date();
+            yield this.stepReporter.onStepBegin({
+                createdAt,
+                detectedElements: annotation === null || annotation === void 0 ? void 0 : annotation.detected_elements,
+                screenshot: annotation === null || annotation === void 0 ? void 0 : annotation.image,
+            });
+        });
+    }
     afterCommandExecution(instruction, error) {
         return __awaiter(this, void 0, void 0, function* () {
             var _a;
@@ -148,7 +165,7 @@ export class UiControlClient extends ApiCommands {
             ]);
             logger.debug(instruction);
             try {
-                yield this.stepReporter.resetStep(instruction);
+                this.stepReporter.resetStep(instruction);
                 yield this.executionRuntime.executeInstruction(instruction, modelComposition);
                 yield this.afterCommandExecution(instruction);
                 return yield Promise.resolve();
@@ -337,10 +354,14 @@ export class UiControlClient extends ApiCommands {
     // eslint-disable-next-line class-methods-use-this
     waitFor(delayInMs) {
         return {
-            exec() {
-                logger.debug(`Wait for ${delayInMs} ms`);
-                return new Promise((resolve) => { setTimeout(() => resolve(), delayInMs); });
-            },
+            exec: () => __awaiter(this, void 0, void 0, function* () {
+                const stepTitle = `Wait for ${delayInMs} ms`;
+                const instruction = yield this.buildInstruction(stepTitle, []);
+                yield this.beforeNoneInferenceCallCommandExecution(instruction);
+                yield new Promise((resolve) => { setTimeout(resolve, delayInMs); });
+                yield this.afterCommandExecution(instruction);
+                return Promise.resolve();
+            }),
         };
     }
     /**
@@ -735,12 +756,95 @@ export class UiControlClient extends ApiCommands {
             };
         });
     }
+    /**
+     * Holds down a key on the keyboard.
+     *
+     * **Examples:**
+     * ```typescript
+     * await aui.keyDown('a').exec();
+     * ```
+     *
+     * @param {PC_AND_MODIFIER_KEY} key - The key to hold down.
+     */
+    keyDown(key) {
+        return {
+            exec: () => __awaiter(this, void 0, void 0, function* () {
+                const stepTitle = `Hold down key ${key}`;
+                const instruction = yield this.buildInstruction(stepTitle, []);
+                try {
+                    yield this.beforeNoneInferenceCallCommandExecution(instruction);
+                    yield this.agent.getOsAgentHandler().desktopKeyHoldDown(key, []);
+                    yield this.afterCommandExecution(instruction);
+                }
+                catch (error) {
+                    yield this.afterCommandExecution(instruction, error instanceof Error ? error : new Error(String(error)));
+                    return Promise.reject(error);
+                }
+                return Promise.resolve();
+            }),
+        };
+    }
+    /**
+     * Releases a key up that was previously held down.
+     *
+     * **Examples:**
+     * ```typescript
+     * await aui.keyUp('a').exec();
+     * ```
+     *
+     * @param {PC_AND_MODIFIER_KEY} key - The key to release up.
+     */
+    keyUp(key) {
+        return {
+            exec: () => __awaiter(this, void 0, void 0, function* () {
+                const stepTitle = `Release key ${key}`;
+                const instruction = yield this.buildInstruction(stepTitle, []);
+                try {
+                    yield this.beforeNoneInferenceCallCommandExecution(instruction);
+                    yield this.agent.getOsAgentHandler().desktopKeyRelease(key, []);
+                    yield this.afterCommandExecution(instruction);
+                }
+                catch (error) {
+                    yield this.afterCommandExecution(instruction, error instanceof Error ? error : new Error(String(error)));
+                    return Promise.reject(error);
+                }
+                return Promise.resolve();
+            }),
+        };
+    }
     act(goal, imageOrOptions, options) {
         return __awaiter(this, void 0, void 0, function* () {
             if (typeof imageOrOptions === 'string') {
                 return this.agent.act(goal, imageOrOptions, options);
             }
-            return this.agent.act(goal, undefined, imageOrOptions);
+            const fullTitle = `Act: ${goal}`;
+            const stepTitle = fullTitle.length > 50 ? `${fullTitle.substring(0, 47)}...` : fullTitle;
+            const instruction = yield this.buildInstruction(stepTitle, []);
+            try {
+                yield this.beforeNoneInferenceCallCommandExecution(instruction);
+                const result = yield this.agent.act(goal, undefined, imageOrOptions);
+                yield this.afterCommandExecution(instruction);
+                return result;
+            }
+            catch (error) {
+                yield this.afterCommandExecution(instruction, error instanceof Error ? error : new Error(String(error)));
+                return Promise.reject(error);
+            }
+        });
+    }
+    /**
+     * Adds tools to the agent that allow it to interact with AI elements.
+     *
+     * @returns {Promise<void>} - A promise that resolves when the tools are added to the agent.
+     */
+    addAIElementsToolsToAgent() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const aiElementLocator = (aiElementName) => this.get().aiElement(aiElementName).exec();
+            const askUIGetAskUIElementTool = new AskUIGetAskUIElementTool(this.agent.getOsAgentHandler(), aiElementLocator, 'aiElement');
+            this.agent.addTool(askUIGetAskUIElementTool);
+            const listAIElementNamesFunction = () => (AIElementCollection.collectAIElements(this.workspaceId, this.aiElementArgs)).then((aiElementCollection) => aiElementCollection.getNames());
+            const askUIListAIElementTool = new AskUIListAIElementTool(listAIElementNamesFunction);
+            this.agent.addTool(askUIListAIElementTool);
         });
     }
 }

package/dist/esm/lib/interactive_cli/create-example-project.js

@@ -179,7 +179,7 @@ export class CreateExampleProject {
         return __awaiter(this, void 0, void 0, function* () {
             const runCommand = promisify(exec);
             const frameworkDependencies = {
-                jest: 'npm i -D @askui/askui-reporters typescript ts-node @types/jest ts-jest jest @askui/jest-allure-circus eslint @typescript-eslint/parser @typescript-eslint/eslint-plugin eslint-plugin-import @askui/eslint-plugin-askui hpagent',
+                jest: 'npm i -D @askui/askui-reporters typescript ts-node @types/jest@30.0.0 ts-jest@29.4.0 jest@29.7.0 @askui/jest-allure-circus eslint @typescript-eslint/parser @typescript-eslint/eslint-plugin eslint-plugin-import @askui/eslint-plugin-askui hpagent',
             };
             yield runCommand(frameworkDependencies.jest);
         });

package/dist/esm/main.d.ts

@@ -4,4 +4,4 @@ export { Instruction, Reporter, ReporterConfig, Snapshot, SnapshotDetailLevel, S
 export { Annotation } from './core/annotation/annotation';
 export { DetectedElement } from './core/model/annotation-result/detected-element';
 export { LogLevels } from './shared';
-export { ToolFailure, ToolError, BaseAgentTool } from './core/models/anthropic';
+export { ToolFailure, ToolError, BaseAgentTool, BetaTool, ToolResult, } from './core/models/anthropic';

package/dist/esm/main.js

@@ -3,4 +3,4 @@ export * from './execution';
 export { Annotation } from './core/annotation/annotation';
 export { DetectedElement } from './core/model/annotation-result/detected-element';
 export { LogLevels } from './shared';
-export { ToolFailure, ToolError, BaseAgentTool } from './core/models/anthropic';
+export { ToolFailure, ToolError, BaseAgentTool, } from './core/models/anthropic';

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "askui",
-    "version": "0.26.0",
+    "version": "0.27.0",
     "license": "MIT",
     "author": "askui GmbH <info@askui.com> (http://www.askui.com/)",
     "description": "Reliable, automated end-to-end-testing that depends on what is shown on your screen instead of the technology you are running on",