askui 0.29.0 → 0.31.0

package/dist/cjs/execution/dsl.js

@@ -4,6 +4,15 @@
 /* eslint-disable max-classes-per-file */
 /* eslint-disable max-len */
 // Autogenerated from typescript.template file
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ApiCommands = exports.Getter = exports.FluentFiltersOrRelationsGetter = exports.FluentFiltersGetter = exports.ExecGetter = exports.FluentCommand = exports.FluentFiltersOrRelationsCondition = exports.FluentFiltersCondition = exports.FluentFiltersOrRelations = exports.FluentFilters = exports.Exec = exports.MODIFIER_KEY_VALUES = exports.ANDROID_KEY_VALUES = exports.PC_KEY_VALUES = exports.Separators = void 0;
 function isStackTraceCodeline(line) {
@@ -17,21 +26,18 @@ function splitStackTrace(stacktrace) {
     return { head: errorStacktraceHead, codelines: errorStacktraceCodeLines };
 }
 function rewriteStackTraceForError(error, newStackTrace) {
-    const errorCopy = new Error(error.message);
-    if (!error.stack) {
-        errorCopy.stack = newStackTrace;
-        return errorCopy;
-    }
-    const errorStacktraceSplit = splitStackTrace(error.stack);
+    var _a;
+    const errorStacktraceSplit = splitStackTrace((_a = error.stack) !== null && _a !== void 0 ? _a : '');
     const newStacktraceSplit = splitStackTrace(newStackTrace);
-    errorCopy.stack = [
+    // eslint-disable-next-line no-param-reassign
+    error.stack = [
         ...errorStacktraceSplit.head,
         ...newStacktraceSplit.codelines,
         ' ',
         ...errorStacktraceSplit.head,
         ...errorStacktraceSplit.codelines,
     ].join('\n');
-    return errorCopy;
+    return error;
 }
 var Separators;
 (function (Separators) {
@@ -55,7 +61,7 @@ class FluentBase {
         });
         return paramsList;
     }
-    fluentCommandStringBuilder(modelComposition, currentInstruction = '', paramsList = new Map()) {
+    fluentCommandStringBuilder(modelComposition = [], skipCache = false, retryStrategy, currentInstruction = '', paramsList = new Map()) {
         const newCurrentInstruction = `${this.textStr} ${currentInstruction}`;
         const newParamsList = FluentBase.addParams(paramsList, this._params);
         if (this instanceof FluentCommand) {
@@ -65,12 +71,12 @@ class FluentBase {
             return fluentCommand.fluentCommandExecutor(newCurrentInstruction.trim(), modelComposition, {
                 customElementsJson: customElements,
                 aiElementNames,
-            });
+            }, skipCache, retryStrategy);
         }
         if (!this.prev) {
             throw new Error('Prev element not defined');
         }
-        return this.prev.fluentCommandStringBuilder(modelComposition, newCurrentInstruction, newParamsList);
+        return this.prev.fluentCommandStringBuilder(modelComposition, skipCache, retryStrategy, newCurrentInstruction, newParamsList);
     }
     getterStringBuilder(currentInstruction = '', paramsList = new Map()) {
         const newCurrentInstruction = `${this.textStr} ${currentInstruction}`;
@@ -93,10 +99,10 @@ class FluentBase {
     get params() { return this._params; }
 }
 class Exec extends FluentBase {
-    exec(modelComposition = []) {
+    exec(execOptions) {
         const originStacktrace = { stack: '' };
         Error.captureStackTrace(originStacktrace, this.exec);
-        return this.fluentCommandStringBuilder(modelComposition).catch((err) => Promise.reject(rewriteStackTraceForError(err, originStacktrace.stack)));
+        return this.fluentCommandStringBuilder(execOptions === null || execOptions === void 0 ? void 0 : execOptions.modelComposition, execOptions === null || execOptions === void 0 ? void 0 : execOptions.skipCache, execOptions === null || execOptions === void 0 ? void 0 : execOptions.retryStrategy).catch((err) => Promise.reject(rewriteStackTraceForError(err, originStacktrace.stack)));
     }
 }
 exports.Exec = Exec;
@@ -1048,10 +1054,10 @@ class FluentFiltersOrRelations extends FluentFilters {
         this._textStr += 'contains';
         return new FluentFilters(this);
     }
-    exec(modelComposition = []) {
+    exec(execOptions) {
         const originStacktrace = { stack: '' };
         Error.captureStackTrace(originStacktrace, this.exec);
-        return this.fluentCommandStringBuilder(modelComposition).catch((err) => Promise.reject(rewriteStackTraceForError(err, originStacktrace.stack)));
+        return this.fluentCommandStringBuilder(execOptions === null || execOptions === void 0 ? void 0 : execOptions.modelComposition, execOptions === null || execOptions === void 0 ? void 0 : execOptions.skipCache, execOptions === null || execOptions === void 0 ? void 0 : execOptions.retryStrategy).catch((err) => Promise.reject(rewriteStackTraceForError(err, originStacktrace.stack)));
     }
 }
 exports.FluentFiltersOrRelations = FluentFiltersOrRelations;
@@ -2063,6 +2069,15 @@ class FluentFiltersOrRelationsCondition extends FluentFiltersCondition {
 }
 exports.FluentFiltersOrRelationsCondition = FluentFiltersOrRelationsCondition;
 class ExecCondition extends Exec {
+    exec(execOptions) {
+        const _super = Object.create(null, {
+            exec: { get: () => super.exec }
+        });
+        return __awaiter(this, void 0, void 0, function* () {
+            const options = Object.assign(Object.assign({}, execOptions), { skipCache: true });
+            return _super.exec.call(this, options);
+        });
+    }
 }
 // Commands
 class FluentCommand extends FluentBase {

package/dist/cjs/execution/execution-runtime.d.ts

@@ -21,7 +21,7 @@ export declare class ExecutionRuntime {
     stopVideoRecording(): Promise<void>;
     readVideoRecording(): Promise<string>;
     requestControl(controlCommand: ControlCommand): Promise<void>;
-    executeInstruction(instruction: Instruction, modelComposition: ModelCompositionBranch[]): Promise<void>;
+    executeInstruction(instruction: Instruction, modelComposition: ModelCompositionBranch[], skipCache?: boolean, retryStrategy?: RetryStrategy): Promise<void>;
     private readonly EXEC_REPETITION_COUNT;
     private executeCommandRepeatedly;
     /**

package/dist/cjs/execution/execution-runtime.js

@@ -28,10 +28,12 @@ class ExecutionRuntime {
     }
     connect() {
         return __awaiter(this, void 0, void 0, function* () {
+            this.inferenceClient.cacheManager.loadFromFile();
             return this.uiControllerClient.connect();
         });
     }
     disconnect() {
+        this.inferenceClient.cacheManager.saveToFile();
         this.uiControllerClient.disconnect();
     }
     startVideoRecording() {
@@ -55,10 +57,10 @@ class ExecutionRuntime {
             yield this.uiControllerClient.requestControl(controlCommand);
         });
     }
-    executeInstruction(instruction, modelComposition) {
-        return __awaiter(this, void 0, void 0, function* () {
+    executeInstruction(instruction_1, modelComposition_1) {
+        return __awaiter(this, arguments, void 0, function* (instruction, modelComposition, skipCache = false, retryStrategy) {
             var _a, _b, _c;
-            const controlCommand = yield this.predictCommandWithRetry(instruction, modelComposition);
+            const controlCommand = yield this.predictCommandWithRetry(instruction, modelComposition, skipCache, retryStrategy);
             if (controlCommand.code === ui_control_commands_1.ControlCommandCode.OK) {
                 return this.requestControl(controlCommand);
             }
@@ -98,19 +100,20 @@ class ExecutionRuntime {
      * --> retry with linear back-off
      */
     /* eslint-disable-next-line consistent-return */
-    predictCommandWithRetry(instruction, modelComposition) {
-        return __awaiter(this, void 0, void 0, function* () {
+    predictCommandWithRetry(instruction_1, modelComposition_1) {
+        return __awaiter(this, arguments, void 0, function* (instruction, modelComposition, skipCache = false, retryStrategy) {
             var _a, _b, _c;
-            let command = yield this.predictCommand(instruction, modelComposition);
+            const strategy = retryStrategy !== null && retryStrategy !== void 0 ? retryStrategy : this.retryStrategy;
+            let command = yield this.predictCommand(instruction, modelComposition, skipCache);
             /* eslint-disable no-await-in-loop */
-            for (let k = 0; k < this.retryStrategy.retryCount; k += 1) {
+            for (let k = 0; k < strategy.retryCount; k += 1) {
                 if (command.code === ui_control_commands_1.ControlCommandCode.OK) {
                     return command;
                 }
-                const msUntilRetry = this.retryStrategy.getDelay(k + 1);
-                logger_1.logger.debug(`Wait ${msUntilRetry} and retry predicting command...`);
+                const msUntilRetry = strategy.getDelay(k + 1);
+                logger_1.logger.debug(`Wait ${msUntilRetry} and retry ${k + 1}/${strategy.retryCount} predicting command...`);
                 yield (0, misc_1.delay)(msUntilRetry);
-                command = yield this.predictCommand(instruction, modelComposition, new control_command_error_1.ControlCommandError((_c = (_b = (_a = command.actions) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.text) !== null && _c !== void 0 ? _c : ''));
+                command = yield this.predictCommand(instruction, modelComposition, skipCache, new control_command_error_1.ControlCommandError((_c = (_b = (_a = command.actions) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.text) !== null && _c !== void 0 ? _c : ''));
             }
             /* eslint-enable no-await-in-loop */
             return command;
@@ -120,11 +123,11 @@ class ExecutionRuntime {
         return this.stepReporter.config.withScreenshots === 'begin'
             || this.stepReporter.config.withScreenshots === 'always';
     }
-    isImageRequired(instruction) {
+    isImageRequired(instruction, customElements) {
         return __awaiter(this, void 0, void 0, function* () {
             if (this.isImageRequiredByConfig())
                 return Promise.resolve(true);
-            return this.inferenceClient.isImageRequired(instruction);
+            return this.inferenceClient.isImageRequired(instruction, customElements);
         });
     }
     isAnnotationRequired() {
@@ -139,8 +142,9 @@ class ExecutionRuntime {
     }
     buildSnapshot(instruction) {
         return __awaiter(this, void 0, void 0, function* () {
+            var _a;
             const createdAt = new Date();
-            const screenshot = (yield this.isImageRequired(instruction))
+            const screenshot = (yield this.isImageRequired(instruction.value, (_a = instruction.customElements) !== null && _a !== void 0 ? _a : []))
                 ? yield this.getScreenshot() : undefined;
             const annotation = this.isAnnotationRequired() ? yield this.annotateImage() : undefined;
             return {
@@ -150,14 +154,14 @@ class ExecutionRuntime {
             };
         });
     }
-    predictCommand(instruction, modelComposition, retryError) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const snapshot = yield this.buildSnapshot(instruction.value);
+    predictCommand(instruction_1, modelComposition_1) {
+        return __awaiter(this, arguments, void 0, function* (instruction, modelComposition, skipCache = false, retryError) {
+            const snapshot = yield this.buildSnapshot(instruction);
             if (retryError !== undefined)
                 this.stepReporter.onStepRetry(snapshot, retryError);
             else
                 this.stepReporter.onStepBegin(snapshot);
-            const controlCommand = yield this.inferenceClient.predictControlCommand(instruction.value, modelComposition, instruction.customElements, snapshot.screenshot);
+            const controlCommand = yield this.inferenceClient.predictControlCommand(instruction.value, modelComposition, instruction.customElements, snapshot.screenshot, skipCache);
             if (instruction.secretText !== undefined) {
                 controlCommand.setTextToBeTyped(instruction.secretText);
             }

package/dist/cjs/execution/inference-client.d.ts

@@ -6,21 +6,23 @@ import { Annotation } from '../core/annotation/annotation';
 import { DetectedElement } from '../core/model/annotation-result/detected-element';
 import { VQAInferenceResponseBody } from '../core/inference-response/inference-response';
 import { ModelCompositionBranch } from './model-composition-branch';
+import { CacheInterface } from '../core/cache';
 export declare class InferenceClient {
     private readonly baseUrl;
     private readonly httpClient;
+    readonly cacheManager: CacheInterface;
     private readonly resize?;
     readonly workspaceId?: string | undefined;
     readonly modelComposition?: ModelCompositionBranch[] | undefined;
     private readonly apiVersion;
     private urls;
-    constructor(baseUrl: string, httpClient: HttpClientGot, resize?: number | undefined, workspaceId?: string | undefined, modelComposition?: ModelCompositionBranch[] | undefined, apiVersion?: string);
-    isImageRequired(instruction: string): Promise<boolean>;
+    constructor(baseUrl: string, httpClient: HttpClientGot, cacheManager: CacheInterface, resize?: number | undefined, workspaceId?: string | undefined, modelComposition?: ModelCompositionBranch[] | undefined, apiVersion?: string);
+    isImageRequired(instruction: string, customElements: CustomElement[]): Promise<boolean>;
     private resizeIfNeeded;
     inference(customElements?: CustomElement[], image?: string, instruction?: string, modelComposition?: ModelCompositionBranch[]): Promise<ControlCommand | Annotation>;
     vqaInference(image: string, prompt: string, config?: object): Promise<VQAInferenceResponseBody>;
     private static logMetaInformation;
-    predictControlCommand(instruction: string, modelComposition: ModelCompositionBranch[], customElements?: CustomElement[], image?: string): Promise<ControlCommand>;
+    predictControlCommand(instruction: string, modelComposition: ModelCompositionBranch[], customElements?: CustomElement[], image?: string, skipCache?: boolean): Promise<ControlCommand>;
     getDetectedElements(instruction: string, image: string, customElements?: CustomElement[]): Promise<DetectedElement[]>;
     predictImageAnnotation(image: string, customElements?: CustomElement[]): Promise<Annotation>;
     predictVQAAnswer(prompt: string, image: string, config?: object): Promise<any>;

package/dist/cjs/execution/inference-client.js

@@ -20,10 +20,12 @@ const transformations_1 = require("../utils/transformations");
 const inference_response_error_1 = require("./inference-response-error");
 const config_error_1 = require("./config-error");
 const logger_1 = require("../lib/logger");
+const control_command_code_1 = require("../core/ui-control-commands/control-command-code");
 class InferenceClient {
-    constructor(baseUrl, httpClient, resize, workspaceId, modelComposition, apiVersion = 'v1') {
+    constructor(baseUrl, httpClient, cacheManager, resize, workspaceId, modelComposition, apiVersion = 'v1') {
         this.baseUrl = baseUrl;
         this.httpClient = httpClient;
+        this.cacheManager = cacheManager;
         this.resize = resize;
         this.workspaceId = workspaceId;
         this.modelComposition = modelComposition;
@@ -44,8 +46,13 @@ class InferenceClient {
         }
         this.resize = this.resize ? Math.ceil(this.resize) : this.resize;
     }
-    isImageRequired(instruction) {
+    isImageRequired(instruction, customElements) {
         return __awaiter(this, void 0, void 0, function* () {
+            const cachedImageRequired = this.cacheManager.isImageRequired(instruction, customElements);
+            if (cachedImageRequired !== undefined) {
+                logger_1.logger.debug(`Cache hit for image required: '${instruction}'.`);
+                return Promise.resolve(cachedImageRequired);
+            }
             const response = yield this.httpClient.post(this.urls.isImageRequired, {
                 instruction,
             });
@@ -96,11 +103,23 @@ class InferenceClient {
         }
     }
     predictControlCommand(instruction_1, modelComposition_1) {
-        return __awaiter(this, arguments, void 0, function* (instruction, modelComposition, customElements = [], image) {
+        return __awaiter(this, arguments, void 0, function* (instruction, modelComposition, customElements = [], image, skipCache = false) {
+            if (!skipCache) {
+                const cachedControlCommand = yield this.cacheManager.getCachedControlCommand(instruction, customElements, image);
+                if (cachedControlCommand !== undefined) {
+                    logger_1.logger.debug(`Cache hit for instruction: '${instruction}'.`);
+                    return Promise.resolve(cachedControlCommand);
+                }
+                logger_1.logger.debug(`Cache miss for instruction: '${instruction}'.`);
+            }
             const inferenceResponse = yield this.inference(customElements, image, instruction, modelComposition);
             if (!(inferenceResponse instanceof ui_control_commands_1.ControlCommand)) {
                 throw new inference_response_error_1.InferenceResponseError('Internal Error. Can not execute command');
             }
+            if (!skipCache && inferenceResponse.code === control_command_code_1.ControlCommandCode.OK) {
+                yield this.cacheManager.addCacheEntryFromControlCommand(instruction, inferenceResponse, customElements, image);
+                logger_1.logger.debug(`Cache added for instruction: '${instruction}'`);
+            }
             return inferenceResponse;
         });
     }

package/dist/cjs/execution/ui-control-client-dependency-builder.js

@@ -23,6 +23,7 @@ const execution_runtime_1 = require("./execution-runtime");
 const reporting_1 = require("../core/reporting");
 const read_credentials_1 = require("./read-credentials");
 const linear_retry_strategy_1 = require("./retry-strategies/linear-retry-strategy");
+const cache_1 = require("../core/cache");
 class UiControlClientDependencyBuilder {
     static buildHttpClient(clientArgs) {
         return __awaiter(this, void 0, void 0, function* () {
@@ -37,7 +38,11 @@ class UiControlClientDependencyBuilder {
         return __awaiter(this, void 0, void 0, function* () {
             var _a;
             const httpClient = yield UiControlClientDependencyBuilder.buildHttpClient(clientArgs);
-            return new inference_client_1.InferenceClient(clientArgs.inferenceServerUrl, httpClient, clientArgs.resize, (_a = clientArgs.credentials) === null || _a === void 0 ? void 0 : _a.workspaceId, clientArgs.modelComposition, clientArgs.inferenceServerApiVersion);
+            let cacheManager = new cache_1.DummyCacheManager();
+            if (clientArgs.cacheConfig) {
+                cacheManager = new cache_1.CacheManager(clientArgs.cacheConfig);
+            }
+            return new inference_client_1.InferenceClient(clientArgs.inferenceServerUrl, httpClient, cacheManager, clientArgs.resize, (_a = clientArgs.credentials) === null || _a === void 0 ? void 0 : _a.workspaceId, clientArgs.modelComposition, clientArgs.inferenceServerApiVersion);
         });
     }
     static buildUiControllerClient(clientArgs) {

package/dist/cjs/execution/ui-control-client.d.ts

@@ -5,6 +5,7 @@ import { AnnotationRequest } from '../core/model/annotation-result/annotation-in
 import { DetectedElement } from '../core/model/annotation-result/detected-element';
 import { ClientArgs } from './ui-controller-client-interface';
 import { ModelCompositionBranch } from './model-composition-branch';
+import { RetryStrategy } from './retry-strategies';
 import { AskUIAgent, AgentHistory, ActOptions } from '../core/models/anthropic';
 export type RelationsForConvenienceMethods = 'nearestTo' | 'leftOf' | 'above' | 'rightOf' | 'below' | 'contains';
 export type TextMatchingOption = 'similar' | 'exact' | 'regex';
@@ -62,7 +63,7 @@ export declare class UiControlClient extends ApiCommands {
     private escapeSeparatorString;
     private buildInstruction;
     private getAIElementsByNames;
-    fluentCommandExecutor(instructionString: string, modelComposition: ModelCompositionBranch[], context?: CommandExecutorContext): Promise<void>;
+    fluentCommandExecutor(instructionString: string, modelComposition: ModelCompositionBranch[], context?: CommandExecutorContext, skipCache?: boolean, retryStrategy?: RetryStrategy): Promise<void>;
     getterExecutor(instruction: string, context?: CommandExecutorContext): Promise<DetectedElement[]>;
     /**
      * Takes a prompt that contains a question you want to be answered
@@ -563,7 +564,6 @@ export declare class UiControlClient extends ApiCommands {
      * #### Cross-Platform Coordination
      * ```typescript
      * // Share context between desktop and mobile agents
-     * await auiAndroid.agent.configureAsAndroidAgent();
      *
      * const history = await auiDesktop.act("Copy username from desktop app");
      * await auiAndroid.act("Paste username into mobile login", {

package/dist/cjs/execution/ui-control-client.js

@@ -18,6 +18,7 @@ const logger_1 = require("../lib/logger");
 const ui_control_client_dependency_builder_1 = require("./ui-control-client-dependency-builder");
 const ai_element_collection_1 = require("../core/ai-element/ai-element-collection");
 const retry_strategies_1 = require("./retry-strategies");
+const control_command_error_1 = require("./control-command-error");
 const anthropic_1 = require("../core/models/anthropic");
 const askui_api_tools_1 = require("../core/models/anthropic/tools/askui-api-tools");
 class UiControlClient extends dsl_1.ApiCommands {
@@ -46,7 +47,7 @@ class UiControlClient extends dsl_1.ApiCommands {
         return __awaiter(this, void 0, void 0, function* () {
             const connectionState = yield this.executionRuntime.connect();
             yield this.agent.initializeOsAgentHandler();
-            yield this.agent.configureAsDesktopAgent();
+            yield this.agent.configureAgent();
             return connectionState;
         });
     }
@@ -160,7 +161,7 @@ class UiControlClient extends dsl_1.ApiCommands {
         });
     }
     fluentCommandExecutor(instructionString_1, modelComposition_1) {
-        return __awaiter(this, arguments, void 0, function* (instructionString, modelComposition, context = { customElementsJson: [], aiElementNames: [] }) {
+        return __awaiter(this, arguments, void 0, function* (instructionString, modelComposition, context = { customElementsJson: [], aiElementNames: [] }, skipCache = false, retryStrategy) {
             const aiElements = yield this.getAIElementsByNames(context.aiElementNames);
             const instruction = yield this.buildInstruction(instructionString, [
                 ...context.customElementsJson,
@@ -169,7 +170,7 @@ class UiControlClient extends dsl_1.ApiCommands {
             logger_1.logger.debug(instruction);
             try {
                 this.stepReporter.resetStep(instruction);
-                yield this.executionRuntime.executeInstruction(instruction, modelComposition);
+                yield this.executionRuntime.executeInstruction(instruction, modelComposition, skipCache, retryStrategy);
                 yield this.afterCommandExecution(instruction);
                 return yield Promise.resolve();
             }
@@ -453,18 +454,34 @@ class UiControlClient extends dsl_1.ApiCommands {
      */
     waitUntil(AskUICommand_1) {
         return __awaiter(this, arguments, void 0, function* (AskUICommand, maxTry = 5, waitTime = 2000) {
+            logger_1.logger.debug(`waitUntil: Starting with maxTry=${maxTry}, waitTime=${waitTime}ms, retryStrategy=${this.executionRuntime.retryStrategy.constructor.name}`);
             const userDefinedStrategy = this.executionRuntime.retryStrategy;
-            try {
-                this.executionRuntime.retryStrategy = new retry_strategies_1.NoRetryStrategy();
-                yield AskUICommand.exec();
-                this.executionRuntime.retryStrategy = userDefinedStrategy;
-            }
-            catch (error) {
-                if (maxTry === 0) {
+            this.executionRuntime.retryStrategy = new retry_strategies_1.NoRetryStrategy();
+            const attempt = (retriesLeft) => __awaiter(this, void 0, void 0, function* () {
+                const attemptNumber = maxTry - retriesLeft;
+                logger_1.logger.debug(`waitUntil: Attempt ${attemptNumber}/${maxTry} (${retriesLeft} retries remaining)`);
+                try {
+                    yield AskUICommand.exec();
+                    logger_1.logger.debug(`waitUntil: Command succeeded on attempt ${attemptNumber}/${maxTry}`);
+                    return;
+                }
+                catch (error) {
+                    if (error instanceof control_command_error_1.ControlCommandError && retriesLeft > 0) {
+                        logger_1.logger.debug(`waitUntil: ControlCommandError on attempt ${attemptNumber}/${maxTry}, waiting ${waitTime}ms before retry.`, error);
+                        yield this.waitFor(waitTime).exec();
+                        yield attempt(retriesLeft - 1);
+                        return;
+                    }
+                    const errorName = error instanceof Error ? error.name : 'Error';
+                    logger_1.logger.debug(`waitUntil: ${errorName} on attempt ${attemptNumber}/${maxTry}, no retries remaining.`, error);
                     throw error;
                 }
-                yield this.waitFor(waitTime).exec();
-                yield this.waitUntil(AskUICommand, maxTry - 1, waitTime);
+            });
+            try {
+                yield attempt(maxTry - 1);
+            }
+            finally {
+                this.executionRuntime.retryStrategy = userDefinedStrategy;
             }
         });
     }

package/dist/cjs/execution/ui-controller-client-interface.d.ts

@@ -5,6 +5,7 @@ import { Reporter } from '../core/reporting';
 import { Context } from './context';
 import { RetryStrategy } from './retry-strategies/retry-strategy';
 import { AIElementArgs } from '../core/ai-element/ai-elements-args';
+import { CacheConfig } from '../core/cache';
 /**
  * Context object to provide additional information about the context of (test) automation.
  *
@@ -63,6 +64,7 @@ export interface ClientArgs {
     readonly inferenceServerApiVersion?: string;
     readonly retryStrategy?: RetryStrategy;
     readonly aiElementArgs?: AIElementArgs;
+    readonly cacheConfig?: CacheConfig;
 }
 export interface ClientArgsWithDefaults extends ClientArgs {
     readonly uiControllerUrl: string;

package/dist/cjs/utils/base_64_image/base-64-image.d.ts

@@ -12,5 +12,7 @@ export declare class Base64Image {
     getInfo(): Promise<sharp.OutputInfo>;
     resizeToFitInto(dimension: number): Promise<Base64Image>;
     resizeWithSameAspectRatio(width: number, height: number): Promise<Base64Image>;
+    cropRegion(x: number, y: number, croppedWidth: number, croppedHeight: number): Promise<Base64Image>;
     toString(withPrefix?: boolean): string;
+    toBuffer(): Buffer;
 }

package/dist/cjs/utils/base_64_image/base-64-image.js

@@ -67,11 +67,12 @@ class Base64Image {
     resizeToFitInto(dimension) {
         return __awaiter(this, void 0, void 0, function* () {
             const { width, height } = yield this.getInfo();
+            const roundedDimension = Math.round(dimension);
             const buffer = yield (yield this.getSharp())
                 .resize({
                 fit: 'contain',
-                height: height > width ? dimension : undefined,
-                width: width >= height ? dimension : undefined,
+                height: height > width ? roundedDimension : undefined,
+                width: width >= height ? roundedDimension : undefined,
             })
                 .toBuffer();
             return Base64Image.fromBuffer(buffer);
@@ -79,11 +80,30 @@ class Base64Image {
     }
     resizeWithSameAspectRatio(width, height) {
         return __awaiter(this, void 0, void 0, function* () {
+            const roundedWidth = Math.round(width);
+            const roundedHeight = Math.round(height);
             const buffer = yield (yield this.getSharp())
                 .resize({
                 fit: 'contain',
-                height,
-                width,
+                height: roundedHeight,
+                width: roundedWidth,
+            })
+                .toBuffer();
+            return Base64Image.fromBuffer(buffer);
+        });
+    }
+    cropRegion(x, y, croppedWidth, croppedHeight) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const roundedX = Math.round(x);
+            const roundedY = Math.round(y);
+            const roundedCroppedWidth = Math.round(croppedWidth);
+            const roundedCroppedHeight = Math.round(croppedHeight);
+            const buffer = yield (yield this.getSharp())
+                .extract({
+                height: roundedCroppedHeight,
+                left: roundedX,
+                top: roundedY,
+                width: roundedCroppedWidth,
             })
                 .toBuffer();
             return Base64Image.fromBuffer(buffer);
@@ -95,6 +115,9 @@ class Base64Image {
         }
         return this.buffer.toString('base64');
     }
+    toBuffer() {
+        return this.buffer;
+    }
 }
 exports.Base64Image = Base64Image;
 Base64Image.strPrefix = 'data:image/png;base64,';

package/dist/esm/core/cache/cache-config.d.ts

@@ -0,0 +1,5 @@
+export type ValidationType = 'PixelPerfect';
+export interface CacheConfig {
+    cacheFilePath?: string;
+    validationType?: ValidationType;
+}

package/dist/esm/core/cache/cache-config.js

@@ -0,0 +1 @@
+export {};

package/dist/esm/core/cache/cache-entry-reference.d.ts

@@ -0,0 +1,7 @@
+import { ImageReference } from './image-reference';
+export declare class CacheEntryReference {
+    image?: ImageReference | undefined;
+    constructor(image?: ImageReference | undefined);
+    static fromJson(json: CacheEntryReference): CacheEntryReference;
+    toJson(): Record<string, unknown>;
+}

package/dist/esm/core/cache/cache-entry-reference.js

@@ -0,0 +1,16 @@
+import { ImageReference } from './image-reference';
+export class CacheEntryReference {
+    constructor(image) {
+        this.image = image;
+    }
+    static fromJson(json) {
+        return new CacheEntryReference(json.image ? ImageReference.fromJson(json.image) : undefined);
+    }
+    toJson() {
+        const jsonObject = {};
+        if (this.image !== undefined) {
+            jsonObject['image'] = this.image.toJson();
+        }
+        return jsonObject;
+    }
+}

package/dist/esm/core/cache/cache-entry.d.ts

@@ -0,0 +1,11 @@
+import { ControlCommand } from '../ui-control-commands';
+import { CacheEntryReference } from './cache-entry-reference';
+export declare class CacheEntry {
+    alwaysValid: boolean;
+    controlCommand: ControlCommand;
+    reference?: CacheEntryReference | undefined;
+    createdAt: Date;
+    constructor(alwaysValid: boolean, controlCommand: ControlCommand, reference?: CacheEntryReference | undefined, createdAt?: Date);
+    static fromJson(json: CacheEntry): CacheEntry | undefined;
+    toJson(): Record<string, unknown>;
+}

package/dist/esm/core/cache/cache-entry.js

@@ -0,0 +1,43 @@
+import { ControlCommand } from '../ui-control-commands';
+import { CacheEntryReference } from './cache-entry-reference';
+import { logger } from '../../lib';
+export class CacheEntry {
+    constructor(alwaysValid, controlCommand, reference, createdAt = new Date()) {
+        this.alwaysValid = alwaysValid;
+        this.controlCommand = controlCommand;
+        this.reference = reference;
+        this.createdAt = createdAt;
+    }
+    static fromJson(json) {
+        try {
+            if (json === undefined) {
+                throw new Error('Cache entry is undefined');
+            }
+            if (json.controlCommand === undefined) {
+                throw new Error('the key "controlCommand" is required');
+            }
+            if (json.createdAt === undefined) {
+                throw new Error('the key "createdAt" is required');
+            }
+            if (json.alwaysValid === undefined) {
+                throw new Error('the key "alwaysValid" is required');
+            }
+            return new CacheEntry(json.alwaysValid, ControlCommand.fromJson(json.controlCommand), json.reference ? CacheEntryReference.fromJson(json.reference) : undefined, new Date(json.createdAt));
+        }
+        catch (error) {
+            logger.error(`Error deserializing cache entry: ${error}`);
+            return undefined;
+        }
+    }
+    toJson() {
+        const jsonObject = {
+            alwaysValid: this.alwaysValid,
+            controlCommand: this.controlCommand.toJson(),
+            createdAt: this.createdAt,
+        };
+        if (this.reference !== undefined) {
+            jsonObject['reference'] = this.reference.toJson();
+        }
+        return jsonObject;
+    }
+}

package/dist/esm/core/cache/cache-interface.d.ts

@@ -0,0 +1,11 @@
+import { CustomElement } from '../model/custom-element';
+import { ControlCommand } from '../ui-control-commands';
+import { CacheEntry } from './cache-entry';
+export interface CacheInterface {
+    loadFromFile(): void;
+    saveToFile(): void;
+    add(key: string, value: CacheEntry): void;
+    addCacheEntryFromControlCommand(instruction: string, controlCommand: ControlCommand, customElements: CustomElement[], image?: string): Promise<void>;
+    isImageRequired(instruction: string, customElements: CustomElement[]): boolean | undefined;
+    getCachedControlCommand(instruction: string, customElements: CustomElement[], image?: string): Promise<ControlCommand | undefined>;
+}

package/dist/esm/core/cache/cache-interface.js

@@ -0,0 +1 @@
+export {};

package/dist/esm/core/cache/cache-manager.d.ts

@@ -0,0 +1,24 @@
+import { ControlCommand } from '../ui-control-commands';
+import { CacheInterface } from './cache-interface';
+import { CacheEntry } from './cache-entry';
+import { CacheConfig, ValidationType } from './cache-config';
+import { CustomElement } from '../model/custom-element';
+export declare class CacheManager implements CacheInterface {
+    private relevantData;
+    validationType: ValidationType;
+    private cacheFile;
+    private readonly referenceWidth;
+    private readonly referenceHeight;
+    private readonly defaultValidationType;
+    constructor(config: CacheConfig);
+    loadFromFile(): void;
+    saveToFile(): void;
+    add(key: string, value: CacheEntry): void;
+    addCacheEntryFromControlCommand(instruction: string, controlCommand: ControlCommand, customElements: CustomElement[], image?: string): Promise<void>;
+    isImageRequired(instruction: string, customElements: CustomElement[]): boolean | undefined;
+    getCachedControlCommand(instruction: string, customElements: CustomElement[], image?: string): Promise<ControlCommand | undefined>;
+    private encodeKey;
+    private getCacheEntriesByKey;
+    private getValidCacheEntry;
+    private validateAccordingToPixelPerfect;
+}