askui 0.26.0 → 0.28.0

package/dist/esm/execution/ui-control-client.d.ts

@@ -55,6 +55,7 @@ export declare class UiControlClient extends ApiCommands {
     stopVideoRecording(): Promise<void>;
     readVideoRecording(): Promise<string>;
     private shouldAnnotateAfterCommandExecution;
+    private beforeNoneInferenceCallCommandExecution;
     private afterCommandExecution;
     annotate(annotationRequest?: AnnotationRequest): Promise<Annotation>;
     annotateInteractively(): Promise<void>;
@@ -475,6 +476,28 @@ export declare class UiControlClient extends ApiCommands {
      * @returns {ExpectAllExistResult.elements} - ExpectExistenceElement[].
      */
     expectAllExist(query: ElementExistsQuery[]): Promise<ExpectAllExistResult>;
+    /**
+     * Holds down a key on the keyboard.
+     *
+     * **Examples:**
+     * ```typescript
+     * await aui.keyDown('a').exec();
+     * ```
+     *
+     * @param {PC_AND_MODIFIER_KEY} key - The key to hold down.
+     */
+    keyDown(key: PC_AND_MODIFIER_KEY): Executable;
+    /**
+     * Releases a key up that was previously held down.
+     *
+     * **Examples:**
+     * ```typescript
+     * await aui.keyUp('a').exec();
+     * ```
+     *
+     * @param {PC_AND_MODIFIER_KEY} key - The key to release up.
+     */
+    keyUp(key: PC_AND_MODIFIER_KEY): Executable;
     /**
      * Instructs the agent to autonomously achieve a specified goal through UI interactions.
      *
@@ -573,4 +596,57 @@ export declare class UiControlClient extends ApiCommands {
      */
     act(goal: string, options?: ActOptions): Promise<AgentHistory>;
     act(goal: string, imagePathOrBase64String: string, options?: ActOptions): Promise<AgentHistory>;
+    /**
+     * Adds tools to the agent that allow it to interact with AI elements.
+     *
+     * @returns {Promise<void>} - A promise that resolves when the tools are added to the agent.
+     */
+    addAIElementsToolsToAgent(): Promise<void>;
+    /**
+     * Retrieves the starting arguments used when the controller server was initialized.
+     *
+     * Useful for debugging, logging, or verifying the current server configuration.
+     *
+     * @property {string} displayNum - Display number controlled by the controller
+     * @property {boolean} minimize - Whether controller starts minimized
+     * @property {string} runtime - Runtime type ("desktop" or "android")
+     * @property {number} port - Communication port
+     * @property {number} actionWaitTime - Action wait time
+     * @property {string} host - Host address
+     * @property {string} logFile - Log file path
+     * @property {boolean} hideOverlay - Whether overlay is hidden
+     * @property {boolean} debugDraw - Whether debug drawing is enabled
+     * @property {string} deviceId - Android device ID
+     * @property {string} configFile - Configuration file path
+     * @property {string} logLevel - Logging level
+     *
+     * @example
+     * ```typescript
+     * const startingArguments = await aui.getControllerStartingArguments();
+     * console.log(startingArguments);
+     * // Output example:
+     * // {
+     * //   displayNum: 0,
+     * //   minimize: true,
+     * //   runtime: 'desktop',
+     * //   port: 5000,
+     * //   actionWaitTime: 1000,
+     * //   host: '127.0.0.1',
+     * //   logFile: '/tmp/askui/askui-server.log',
+     * //   hideOverlay: false,
+     * //   debugDraw: false,
+     * //   deviceId: 'emulator-5554',
+     * //   configFile: '/tmp/askui/askui-config.json',
+     * //   logLevel: 'info',
+     * // }
+    * ```
+    *
+    * @example Retrieving Android device ID:
+    * ```typescript
+    * const startingArguments = await aui.getControllerStartingArguments();
+    * console.log(startingArguments.deviceId);
+    * // Output example: "emulator-5554"
+    * ```
+    */
+    getControllerStartingArguments(): Promise<Record<'displayNum' | 'minimize' | 'runtime' | 'port' | 'actionWaitTime' | 'host' | 'logFile' | 'hideOverlay' | 'debugDraw' | 'deviceId' | 'configFile' | 'logLevel', string | number | boolean>>;
 }

package/dist/esm/execution/ui-control-client.js

@@ -16,6 +16,7 @@ import { UiControlClientDependencyBuilder } from './ui-control-client-dependency
 import { AIElementCollection } from '../core/ai-element/ai-element-collection';
 import { NoRetryStrategy } from './retry-strategies';
 import { AskUIAgent } from '../core/models/anthropic';
+import { AskUIGetAskUIElementTool, AskUIListAIElementTool } from '../core/models/anthropic/tools/askui-api-tools';
 export class UiControlClient extends ApiCommands {
     constructor(workspaceId, executionRuntime, stepReporter, aiElementArgs, agent) {
         super();
@@ -79,6 +80,22 @@ export class UiControlClient extends ApiCommands {
         return (this.stepReporter.config.withDetectedElements === 'onFailure' && error !== undefined)
             || (this.stepReporter.config.withDetectedElements === 'always');
     }
+    beforeNoneInferenceCallCommandExecution(instruction) {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.stepReporter.resetStep(instruction);
+            let annotation;
+            if (this.stepReporter.config.withDetectedElements === 'begin'
+                || this.stepReporter.config.withDetectedElements === 'always') {
+                annotation = yield this.executionRuntime.annotateImage();
+            }
+            const createdAt = new Date();
+            yield this.stepReporter.onStepBegin({
+                createdAt,
+                detectedElements: annotation === null || annotation === void 0 ? void 0 : annotation.detected_elements,
+                screenshot: annotation === null || annotation === void 0 ? void 0 : annotation.image,
+            });
+        });
+    }
     afterCommandExecution(instruction, error) {
         return __awaiter(this, void 0, void 0, function* () {
             var _a;
@@ -148,7 +165,7 @@ export class UiControlClient extends ApiCommands {
             ]);
             logger.debug(instruction);
             try {
-                yield this.stepReporter.resetStep(instruction);
+                this.stepReporter.resetStep(instruction);
                 yield this.executionRuntime.executeInstruction(instruction, modelComposition);
                 yield this.afterCommandExecution(instruction);
                 return yield Promise.resolve();
@@ -337,10 +354,14 @@ export class UiControlClient extends ApiCommands {
     // eslint-disable-next-line class-methods-use-this
     waitFor(delayInMs) {
         return {
-            exec() {
-                logger.debug(`Wait for ${delayInMs} ms`);
-                return new Promise((resolve) => { setTimeout(() => resolve(), delayInMs); });
-            },
+            exec: () => __awaiter(this, void 0, void 0, function* () {
+                const stepTitle = `Wait for ${delayInMs} ms`;
+                const instruction = yield this.buildInstruction(stepTitle, []);
+                yield this.beforeNoneInferenceCallCommandExecution(instruction);
+                yield new Promise((resolve) => { setTimeout(resolve, delayInMs); });
+                yield this.afterCommandExecution(instruction);
+                return Promise.resolve();
+            }),
         };
     }
     /**
@@ -735,12 +756,146 @@ export class UiControlClient extends ApiCommands {
             };
         });
     }
+    /**
+     * Holds down a key on the keyboard.
+     *
+     * **Examples:**
+     * ```typescript
+     * await aui.keyDown('a').exec();
+     * ```
+     *
+     * @param {PC_AND_MODIFIER_KEY} key - The key to hold down.
+     */
+    keyDown(key) {
+        return {
+            exec: () => __awaiter(this, void 0, void 0, function* () {
+                const stepTitle = `Hold down key ${key}`;
+                const instruction = yield this.buildInstruction(stepTitle, []);
+                try {
+                    yield this.beforeNoneInferenceCallCommandExecution(instruction);
+                    yield this.agent.getOsAgentHandler().desktopKeyHoldDown(key, []);
+                    yield this.afterCommandExecution(instruction);
+                }
+                catch (error) {
+                    yield this.afterCommandExecution(instruction, error instanceof Error ? error : new Error(String(error)));
+                    return Promise.reject(error);
+                }
+                return Promise.resolve();
+            }),
+        };
+    }
+    /**
+     * Releases a key up that was previously held down.
+     *
+     * **Examples:**
+     * ```typescript
+     * await aui.keyUp('a').exec();
+     * ```
+     *
+     * @param {PC_AND_MODIFIER_KEY} key - The key to release up.
+     */
+    keyUp(key) {
+        return {
+            exec: () => __awaiter(this, void 0, void 0, function* () {
+                const stepTitle = `Release key ${key}`;
+                const instruction = yield this.buildInstruction(stepTitle, []);
+                try {
+                    yield this.beforeNoneInferenceCallCommandExecution(instruction);
+                    yield this.agent.getOsAgentHandler().desktopKeyRelease(key, []);
+                    yield this.afterCommandExecution(instruction);
+                }
+                catch (error) {
+                    yield this.afterCommandExecution(instruction, error instanceof Error ? error : new Error(String(error)));
+                    return Promise.reject(error);
+                }
+                return Promise.resolve();
+            }),
+        };
+    }
     act(goal, imageOrOptions, options) {
         return __awaiter(this, void 0, void 0, function* () {
             if (typeof imageOrOptions === 'string') {
                 return this.agent.act(goal, imageOrOptions, options);
             }
-            return this.agent.act(goal, undefined, imageOrOptions);
+            const fullTitle = `Act: ${goal}`;
+            const stepTitle = fullTitle.length > 50 ? `${fullTitle.substring(0, 47)}...` : fullTitle;
+            const instruction = yield this.buildInstruction(stepTitle, []);
+            try {
+                yield this.beforeNoneInferenceCallCommandExecution(instruction);
+                const result = yield this.agent.act(goal, undefined, imageOrOptions);
+                yield this.afterCommandExecution(instruction);
+                return result;
+            }
+            catch (error) {
+                yield this.afterCommandExecution(instruction, error instanceof Error ? error : new Error(String(error)));
+                return Promise.reject(error);
+            }
+        });
+    }
+    /**
+     * Adds tools to the agent that allow it to interact with AI elements.
+     *
+     * @returns {Promise<void>} - A promise that resolves when the tools are added to the agent.
+     */
+    addAIElementsToolsToAgent() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const aiElementLocator = (aiElementName) => this.get().aiElement(aiElementName).exec();
+            const askUIGetAskUIElementTool = new AskUIGetAskUIElementTool(this.agent.getOsAgentHandler(), aiElementLocator, 'aiElement');
+            this.agent.addTool(askUIGetAskUIElementTool);
+            const listAIElementNamesFunction = () => (AIElementCollection.collectAIElements(this.workspaceId, this.aiElementArgs)).then((aiElementCollection) => aiElementCollection.getNames());
+            const askUIListAIElementTool = new AskUIListAIElementTool(listAIElementNamesFunction);
+            this.agent.addTool(askUIListAIElementTool);
+        });
+    }
+    /**
+     * Retrieves the starting arguments used when the controller server was initialized.
+     *
+     * Useful for debugging, logging, or verifying the current server configuration.
+     *
+     * @property {string} displayNum - Display number controlled by the controller
+     * @property {boolean} minimize - Whether controller starts minimized
+     * @property {string} runtime - Runtime type ("desktop" or "android")
+     * @property {number} port - Communication port
+     * @property {number} actionWaitTime - Action wait time
+     * @property {string} host - Host address
+     * @property {string} logFile - Log file path
+     * @property {boolean} hideOverlay - Whether overlay is hidden
+     * @property {boolean} debugDraw - Whether debug drawing is enabled
+     * @property {string} deviceId - Android device ID
+     * @property {string} configFile - Configuration file path
+     * @property {string} logLevel - Logging level
+     *
+     * @example
+     * ```typescript
+     * const startingArguments = await aui.getControllerStartingArguments();
+     * console.log(startingArguments);
+     * // Output example:
+     * // {
+     * //   displayNum: 0,
+     * //   minimize: true,
+     * //   runtime: 'desktop',
+     * //   port: 5000,
+     * //   actionWaitTime: 1000,
+     * //   host: '127.0.0.1',
+     * //   logFile: '/tmp/askui/askui-server.log',
+     * //   hideOverlay: false,
+     * //   debugDraw: false,
+     * //   deviceId: 'emulator-5554',
+     * //   configFile: '/tmp/askui/askui-config.json',
+     * //   logLevel: 'info',
+     * // }
+    * ```
+    *
+    * @example Retrieving Android device ID:
+    * ```typescript
+    * const startingArguments = await aui.getControllerStartingArguments();
+    * console.log(startingArguments.deviceId);
+    * // Output example: "emulator-5554"
+    * ```
+    */
+    getControllerStartingArguments() {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.executionRuntime.getStartingArguments();
         });
     }
 }

package/dist/esm/execution/ui-controller-client.d.ts

@@ -1,6 +1,6 @@
 import WebSocket from 'ws';
 import { DetectedElement } from '../core/model/annotation-result/detected-element';
-import { CaptureScreenshotResponse, ControlResponse, StartRecordingResponse, StopRecordingResponse, ReadRecordingPartResponse, InteractiveAnnotationResponse, GetProcessPidResponse } from '../core/runner-protocol';
+import { CaptureScreenshotResponse, ControlResponse, StartRecordingResponse, StopRecordingResponse, ReadRecordingPartResponse, InteractiveAnnotationResponse, GetProcessPidResponse, GetStartingArgumentsResponse } from '../core/runner-protocol';
 import { ControlCommand } from '../core/ui-control-commands';
 import { UiControllerClientConnectionState } from './ui-controller-client-connection-state';
 export declare class UiControllerClient {
@@ -22,6 +22,7 @@ export declare class UiControllerClient {
     private sendAndReceive;
     private send;
     requestScreenshot(): Promise<CaptureScreenshotResponse>;
+    getStartingArguments(): Promise<GetStartingArgumentsResponse>;
     getServerPid(): Promise<GetProcessPidResponse>;
     startVideoRecording(): Promise<StartRecordingResponse>;
     stopVideoRecording(): Promise<StopRecordingResponse>;

package/dist/esm/execution/ui-controller-client.js

@@ -1,5 +1,5 @@
 import WebSocket from 'ws';
-import { CaptureScreenshotRequest, ControlRequest, StartRecordingRequest, StopRecordingRequest, ReadRecordingRequest, InteractiveAnnotationRequest, GetProcessPidRequest, } from '../core/runner-protocol';
+import { CaptureScreenshotRequest, ControlRequest, StartRecordingRequest, StopRecordingRequest, ReadRecordingRequest, InteractiveAnnotationRequest, GetProcessPidRequest, GetStartingArgumentsRequest, } from '../core/runner-protocol';
 import { logger } from '../lib/logger';
 import { UiControllerClientConnectionState } from './ui-controller-client-connection-state';
 import { ReadRecordingResponseStreamHandler } from './read-recording-response-stream-handler';
@@ -95,6 +95,9 @@ export class UiControllerClient {
     requestScreenshot() {
         return this.sendAndReceive(new CaptureScreenshotRequest());
     }
+    getStartingArguments() {
+        return this.sendAndReceive(new GetStartingArgumentsRequest());
+    }
     getServerPid() {
         return this.sendAndReceive(new GetProcessPidRequest());
     }

package/dist/esm/lib/interactive_cli/create-example-project.js

@@ -179,7 +179,7 @@ export class CreateExampleProject {
         return __awaiter(this, void 0, void 0, function* () {
             const runCommand = promisify(exec);
             const frameworkDependencies = {
-                jest: 'npm i -D @askui/askui-reporters typescript ts-node @types/jest ts-jest jest @askui/jest-allure-circus eslint @typescript-eslint/parser @typescript-eslint/eslint-plugin eslint-plugin-import @askui/eslint-plugin-askui hpagent',
+                jest: 'npm i -D @askui/askui-reporters typescript ts-node @types/jest@30.0.0 ts-jest@29.4.0 jest@29.7.0 @askui/jest-allure-circus eslint @typescript-eslint/parser @typescript-eslint/eslint-plugin eslint-plugin-import @askui/eslint-plugin-askui hpagent',
             };
             yield runCommand(frameworkDependencies.jest);
         });

package/dist/esm/main.d.ts

@@ -4,4 +4,4 @@ export { Instruction, Reporter, ReporterConfig, Snapshot, SnapshotDetailLevel, S
 export { Annotation } from './core/annotation/annotation';
 export { DetectedElement } from './core/model/annotation-result/detected-element';
 export { LogLevels } from './shared';
-export { ToolFailure, ToolError, BaseAgentTool } from './core/models/anthropic';
+export { ToolFailure, ToolError, BaseAgentTool, BetaTool, ToolResult, } from './core/models/anthropic';

package/dist/esm/main.js

@@ -3,4 +3,4 @@ export * from './execution';
 export { Annotation } from './core/annotation/annotation';
 export { DetectedElement } from './core/model/annotation-result/detected-element';
 export { LogLevels } from './shared';
-export { ToolFailure, ToolError, BaseAgentTool } from './core/models/anthropic';
+export { ToolFailure, ToolError, BaseAgentTool, } from './core/models/anthropic';

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "askui",
-    "version": "0.26.0",
+    "version": "0.28.0",
     "license": "MIT",
     "author": "askui GmbH <info@askui.com> (http://www.askui.com/)",
     "description": "Reliable, automated end-to-end-testing that depends on what is shown on your screen instead of the technology you are running on",