askui 0.20.7 → 0.20.9

package/dist/cjs/utils/http/custom-errors/index.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.httpClientErrorHandler = exports.GeneralHttpClientError = void 0;
+exports.GeneralHttpClientError = void 0;
+exports.httpClientErrorHandler = httpClientErrorHandler;
 const server_http_client_error_1 = require("./server-http-client-error");
 const authentication_http_client_error_1 = require("./authentication-http-client-error");
 const client_http_client_error_1 = require("./client-http-client-error");
@@ -20,4 +21,3 @@ function httpClientErrorHandler(responseCode, errorMessage) {
     }
     return new unkown_http_client_error_1.UnkownHttpClientError(diplayedMessage);
 }
-exports.httpClientErrorHandler = httpClientErrorHandler;

package/dist/cjs/utils/http/http-client-got.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { OptionsOfJSONResponseBody } from 'got';
 import http from 'http';
 import https from 'https';

package/dist/cjs/utils/http/http-client-got.js

@@ -133,8 +133,8 @@ class HttpClientGot {
             return { body, headers };
         });
     }
-    get(url, options = { responseType: 'json' }) {
-        return __awaiter(this, void 0, void 0, function* () {
+    get(url_1) {
+        return __awaiter(this, arguments, void 0, function* (url, options = { responseType: 'json' }) {
             const response = yield this.askuiGot.get(url, this.injectHeadersAndCookies(url, options));
             return response.body;
         });

package/dist/cjs/utils/path.js

@@ -3,9 +3,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getPathToNodeModulesRoot = void 0;
+exports.getPathToNodeModulesRoot = getPathToNodeModulesRoot;
 const path_1 = __importDefault(require("path"));
 function getPathToNodeModulesRoot() {
     return path_1.default.join(__dirname, '..', '..');
 }
-exports.getPathToNodeModulesRoot = getPathToNodeModulesRoot;

package/dist/cjs/utils/proxy/proxy-builder.js

@@ -1,7 +1,11 @@
 "use strict";
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
 }) : (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
     o[k2] = m[k];
@@ -28,7 +32,8 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.envProxyAgents = exports.ProxyImportError = void 0;
+exports.ProxyImportError = void 0;
+exports.envProxyAgents = envProxyAgents;
 const logger_1 = require("../../lib/logger");
 class ProxyImportError extends Error {
 }
@@ -66,4 +71,3 @@ function envProxyAgents() {
         };
     });
 }
-exports.envProxyAgents = envProxyAgents;

package/dist/cjs/utils/transformations.js

@@ -9,7 +9,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.resizeBase64ImageWithSameRatio = void 0;
+exports.resizeBase64ImageWithSameRatio = resizeBase64ImageWithSameRatio;
 const lib_1 = require("../lib");
 const base_64_image_1 = require("./base_64_image/base-64-image");
 const image_resize_errors_1 = require("./image-resize-errors");
@@ -23,8 +23,8 @@ const image_resize_errors_1 = require("./image-resize-errors");
   * @returns {Promise<ResizedImage>} ResiziedImage Interface,
   * the true pixel values = controlui-api response * resizeRatio.
   */
-function resizeBase64ImageWithSameRatio(base64ImageString, maxEdge = 1400) {
-    return __awaiter(this, void 0, void 0, function* () {
+function resizeBase64ImageWithSameRatio(base64ImageString_1) {
+    return __awaiter(this, arguments, void 0, function* (base64ImageString, maxEdge = 1400) {
         lib_1.logger.debug('Image resizing');
         try {
             const image = yield base_64_image_1.Base64Image.fromString(base64ImageString);
@@ -44,4 +44,3 @@ function resizeBase64ImageWithSameRatio(base64ImageString, maxEdge = 1400) {
         }
     });
 }
-exports.resizeBase64ImageWithSameRatio = resizeBase64ImageWithSameRatio;

package/dist/esm/core/inference-response/inference-response.d.ts

@@ -5,6 +5,11 @@ export interface InferenceResponseBody {
     type: ModelType;
     data: ModelType extends 'COMMANDS' ? ControlCommand : Annotation;
 }
+export interface VQAInferenceResponseBody {
+    data: {
+        response: string;
+    };
+}
 export declare class InferenceResponse {
     static fromJson(json: InferenceResponseBody, resizeRatio?: number, image?: string): ControlCommand | Annotation;
     static createModels(type: ModelType, data: ModelType extends 'COMMANDS' ? ControlCommand : Annotation, resizeRatio: number, image?: string): ControlCommand | Annotation;

package/dist/esm/core/inference-response/model-type.d.ts

@@ -1 +1 @@
-export declare type ModelType = 'DETECTED_ELEMENTS' | 'COMMANDS';
+export type ModelType = 'DETECTED_ELEMENTS' | 'COMMANDS';

package/dist/esm/core/model/custom-element.d.ts

@@ -5,16 +5,16 @@ export declare class CustomElement implements CustomElementJson {
     threshold?: number | undefined;
     stopThreshold?: number | undefined;
     rotationDegreePerStep?: number | undefined;
-    imageCompareFormat?: "RGB" | "grayscale" | "edges" | undefined;
-    mask?: {
+    imageCompareFormat?: ("RGB" | "grayscale" | "edges") | undefined;
+    mask?: ({
         x: number;
         y: number;
-    }[] | undefined;
+    })[] | undefined;
     private static schema;
-    constructor(customImage: string, name?: string | undefined, threshold?: number | undefined, stopThreshold?: number | undefined, rotationDegreePerStep?: number | undefined, imageCompareFormat?: "RGB" | "grayscale" | "edges" | undefined, mask?: {
+    constructor(customImage: string, name?: string | undefined, threshold?: number | undefined, stopThreshold?: number | undefined, rotationDegreePerStep?: number | undefined, imageCompareFormat?: ("RGB" | "grayscale" | "edges") | undefined, mask?: ({
         x: number;
         y: number;
-    }[] | undefined);
+    })[] | undefined);
     static fromJsonListWithImagePathOrImage(ceJson?: CustomElementJson[]): Promise<CustomElement[]>;
     static fromJsonWithImagePathOrImage(ceJson: CustomElementJson): Promise<CustomElement>;
     static fromJson(ceJson: CustomElementJson): CustomElement;

package/dist/esm/core/model/custom-element.js

@@ -19,8 +19,8 @@ export class CustomElement {
         this.imageCompareFormat = imageCompareFormat;
         this.mask = mask;
     }
-    static fromJsonListWithImagePathOrImage(ceJson = []) {
-        return __awaiter(this, void 0, void 0, function* () {
+    static fromJsonListWithImagePathOrImage() {
+        return __awaiter(this, arguments, void 0, function* (ceJson = []) {
             return Promise.all(ceJson.map((customElement) => CustomElement.fromJsonWithImagePathOrImage(customElement)));
         });
     }

package/dist/esm/core/reporting/snapshot-detail-level.d.ts

@@ -1,4 +1,4 @@
-export declare type SnapshotDetailLevel = 
+export type SnapshotDetailLevel = 
 /**
  * Details of snapshot, e.g., screenshot or detected elements, may or may not be available
  *  depending on if they are required by the step. There are not guarantees made.

package/dist/esm/core/reporting/step-status-end.d.ts

@@ -1 +1 @@
-export declare type StepStatusEnd = 'passed' | 'failed' | 'erroneous';
+export type StepStatusEnd = 'passed' | 'failed' | 'erroneous';

package/dist/esm/core/reporting/step-status.d.ts

@@ -10,4 +10,4 @@
  *  e.g., the user has no usage left, response of inference backend cannot be processed,
  *  e.g., because lib version is outdated, etc. (currently not supported, everything "failed")
  */
-export declare type StepStatus = 'passed' | 'failed' | 'pending' | 'running' | 'erroneous';
+export type StepStatus = 'passed' | 'failed' | 'pending' | 'running' | 'erroneous';

package/dist/esm/core/ui-control-commands/action.d.ts

@@ -1,5 +1,5 @@
 import { InputEvent } from './input-event';
-export declare type ActionParameters = {
+export type ActionParameters = {
     [key: string]: string | number | boolean | string[] | ActionParameters;
 };
 export declare class Action {

package/dist/esm/execution/dsl.d.ts

@@ -3,12 +3,12 @@ import { DetectedElement } from '../core/model/annotation-result/detected-elemen
 export declare enum Separators {
     STRING = "<|string|>"
 }
-export declare type INTERSECTION_AREA = 'element_center_line' | 'element_edge_area' | 'display_edge_area';
-export declare type PC_KEY = 'backspace' | 'delete' | 'enter' | 'tab' | 'escape' | 'up' | 'down' | 'right' | 'left' | 'home' | 'end' | 'pageup' | 'pagedown' | 'f1' | 'f2' | 'f3' | 'f4' | 'f5' | 'f6' | 'f7' | 'f8' | 'f9' | 'f10' | 'f11' | 'f12' | 'space' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '!' | '"' | '#' | '$' | '%' | '&' | "'" | '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | ':' | ';' | '<' | '=' | '>' | '?' | '@' | '[' | '\\' | ']' | '^' | '_' | '`' | '{' | '|' | '}' | '~ ';
-export declare type ANDROID_KEY = 'home' | 'back' | 'call' | 'endcall' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | 'star' | 'pound' | 'dpad_up' | 'dpad_down' | 'dpad_left' | 'dpad_right' | 'dpad_center' | 'volume_up' | 'volume_down' | 'power' | 'camera' | 'clear' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | 'comma' | 'period' | 'alt_left' | 'alt_right' | 'shift_left' | 'shift_right' | 'tab' | 'space' | 'sym' | 'explorer' | 'envelope' | 'enter' | 'del' | 'grave' | 'minus' | 'equals' | 'left_bracket' | 'right_bracket' | 'backslash' | 'semicolon' | 'apostrophe' | 'slash' | 'at' | 'num' | 'headsethook' | 'focus' | 'plus' | 'menu' | 'notification' | 'search' | 'media_play_pause' | 'media_stop' | 'media_next' | 'media_previous' | 'media_rewind' | 'media_fast_forward' | 'mute' | 'page_up' | 'page_down' | 'switch_charset' | 'escape' | 'forward_del' | 'ctrl_left' | 'ctrl_right' | 'caps_lock' | 'scroll_lock' | 'function' | 'break' | 'move_home' | 'move_end' | 'insert' | 'forward' | 'media_play' | 'media_pause' | 'media_close' | 'media_eject' | 'media_record' | 'f1' | 'f2' | 'f3' | 'f4' | 'f5' | 'f6' | 'f7' | 'f8' | 'f9' | 'f10' | 'f11' | 'f12' | 'num_lock' | 'numpad_0' | 'numpad_1' | 'numpad_2' | 'numpad_3' | 'numpad_4' | 'numpad_5' | 'numpad_6' | 'numpad_7' | 'numpad_8' | 'numpad_9' | 'numpad_divide' | 'numpad_multiply' | 'numpad_subtract' | 'numpad_add' | 'numpad_dot' | 'numpad_comma' | 'numpad_enter' | 'numpad_equals' | 'numpad_left_paren' | 'numpad_right_paren' | 'volume_mute' | 'info' | 'channel_up' | 'channel_down' | 'zoom_in' | 'zoom_out' | 'window' | 'guide' | 'bookmark' | 'captions' | 'settings' | 'app_switch' | 'language_switch' | 'contacts' | 'calendar' | 'music' | 'calculator' | 'assist' | 'brightness_down' | 'brightness_up' | 'media_audio_track' | 'sleep' | 'wakeup' | 'pairing' | 'media_top_menu' | 'last_channel' | 'tv_data_service' | 'voice_assist' | 'help' | 'navigate_previous' | 'navigate_next' | 'navigate_in' | 'navigate_out' | 'dpad_up_left' | 'dpad_down_left' | 'dpad_up_right' | 'dpad_down_right' | 'media_skip_forward' | 'media_skip_backward' | 'media_step_forward' | 'media_step_backward' | 'soft_sleep' | 'cut' | 'copy' | 'paste' | 'all_apps' | 'refresh';
-export declare type MODIFIER_KEY = 'command' | 'alt' | 'control' | 'shift' | 'right_shift';
-export declare type COLOR = 'black' | 'white' | 'red' | 'green' | 'yellow green' | 'orange' | 'yellow' | 'purple' | 'pink' | 'gray' | 'lime green' | 'royal blue';
-export declare type PC_AND_MODIFIER_KEY = 'command' | 'alt' | 'control' | 'shift' | 'right_shift' | 'backspace' | 'delete' | 'enter' | 'tab' | 'escape' | 'up' | 'down' | 'right' | 'left' | 'home' | 'end' | 'pageup' | 'pagedown' | 'f1' | 'f2' | 'f3' | 'f4' | 'f5' | 'f6' | 'f7' | 'f8' | 'f9' | 'f10' | 'f11' | 'f12' | 'space' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '!' | '"' | '#' | '$' | '%' | '&' | "'" | '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | ':' | ';' | '<' | '=' | '>' | '?' | '@' | '[' | '\\' | ']' | '^' | '_' | '`' | '{' | '|' | '}' | '~ ';
+export type INTERSECTION_AREA = 'element_center_line' | 'element_edge_area' | 'display_edge_area';
+export type PC_KEY = 'backspace' | 'delete' | 'enter' | 'tab' | 'escape' | 'up' | 'down' | 'right' | 'left' | 'home' | 'end' | 'pageup' | 'pagedown' | 'f1' | 'f2' | 'f3' | 'f4' | 'f5' | 'f6' | 'f7' | 'f8' | 'f9' | 'f10' | 'f11' | 'f12' | 'space' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '!' | '"' | '#' | '$' | '%' | '&' | "'" | '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | ':' | ';' | '<' | '=' | '>' | '?' | '@' | '[' | '\\' | ']' | '^' | '_' | '`' | '{' | '|' | '}' | '~ ';
+export type ANDROID_KEY = 'home' | 'back' | 'call' | 'endcall' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | 'star' | 'pound' | 'dpad_up' | 'dpad_down' | 'dpad_left' | 'dpad_right' | 'dpad_center' | 'volume_up' | 'volume_down' | 'power' | 'camera' | 'clear' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | 'comma' | 'period' | 'alt_left' | 'alt_right' | 'shift_left' | 'shift_right' | 'tab' | 'space' | 'sym' | 'explorer' | 'envelope' | 'enter' | 'del' | 'grave' | 'minus' | 'equals' | 'left_bracket' | 'right_bracket' | 'backslash' | 'semicolon' | 'apostrophe' | 'slash' | 'at' | 'num' | 'headsethook' | 'focus' | 'plus' | 'menu' | 'notification' | 'search' | 'media_play_pause' | 'media_stop' | 'media_next' | 'media_previous' | 'media_rewind' | 'media_fast_forward' | 'mute' | 'page_up' | 'page_down' | 'switch_charset' | 'escape' | 'forward_del' | 'ctrl_left' | 'ctrl_right' | 'caps_lock' | 'scroll_lock' | 'function' | 'break' | 'move_home' | 'move_end' | 'insert' | 'forward' | 'media_play' | 'media_pause' | 'media_close' | 'media_eject' | 'media_record' | 'f1' | 'f2' | 'f3' | 'f4' | 'f5' | 'f6' | 'f7' | 'f8' | 'f9' | 'f10' | 'f11' | 'f12' | 'num_lock' | 'numpad_0' | 'numpad_1' | 'numpad_2' | 'numpad_3' | 'numpad_4' | 'numpad_5' | 'numpad_6' | 'numpad_7' | 'numpad_8' | 'numpad_9' | 'numpad_divide' | 'numpad_multiply' | 'numpad_subtract' | 'numpad_add' | 'numpad_dot' | 'numpad_comma' | 'numpad_enter' | 'numpad_equals' | 'numpad_left_paren' | 'numpad_right_paren' | 'volume_mute' | 'info' | 'channel_up' | 'channel_down' | 'zoom_in' | 'zoom_out' | 'window' | 'guide' | 'bookmark' | 'captions' | 'settings' | 'app_switch' | 'language_switch' | 'contacts' | 'calendar' | 'music' | 'calculator' | 'assist' | 'brightness_down' | 'brightness_up' | 'media_audio_track' | 'sleep' | 'wakeup' | 'pairing' | 'media_top_menu' | 'last_channel' | 'tv_data_service' | 'voice_assist' | 'help' | 'navigate_previous' | 'navigate_next' | 'navigate_in' | 'navigate_out' | 'dpad_up_left' | 'dpad_down_left' | 'dpad_up_right' | 'dpad_down_right' | 'media_skip_forward' | 'media_skip_backward' | 'media_step_forward' | 'media_step_backward' | 'soft_sleep' | 'cut' | 'copy' | 'paste' | 'all_apps' | 'refresh';
+export type MODIFIER_KEY = 'command' | 'alt' | 'control' | 'shift' | 'right_shift';
+export type COLOR = 'black' | 'white' | 'red' | 'green' | 'yellow green' | 'orange' | 'yellow' | 'purple' | 'pink' | 'gray' | 'lime green' | 'royal blue';
+export type PC_AND_MODIFIER_KEY = 'command' | 'alt' | 'control' | 'shift' | 'right_shift' | 'backspace' | 'delete' | 'enter' | 'tab' | 'escape' | 'up' | 'down' | 'right' | 'left' | 'home' | 'end' | 'pageup' | 'pagedown' | 'f1' | 'f2' | 'f3' | 'f4' | 'f5' | 'f6' | 'f7' | 'f8' | 'f9' | 'f10' | 'f11' | 'f12' | 'space' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '!' | '"' | '#' | '$' | '%' | '&' | "'" | '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | ':' | ';' | '<' | '=' | '>' | '?' | '@' | '[' | '\\' | ']' | '^' | '_' | '`' | '{' | '|' | '}' | '~ ';
 export interface CommandExecutorContext {
     customElementsJson: CustomElementJson[];
     aiElementNames: string[];

package/dist/esm/execution/execution-runtime.d.ts

@@ -5,11 +5,13 @@ import { CustomElementJson } from '../core/model/custom-element-json';
 import { DetectedElement } from '../core/model/annotation-result/detected-element';
 import { UiControllerClientConnectionState } from './ui-controller-client-connection-state';
 import { Instruction, StepReporter } from '../core/reporting';
+import { RetryStrategy } from './retry-strategies/retry-strategy';
 export declare class ExecutionRuntime {
     private uiControllerClient;
     private inferenceClient;
     private stepReporter;
-    constructor(uiControllerClient: UiControllerClient, inferenceClient: InferenceClient, stepReporter: StepReporter);
+    private retryStrategy;
+    constructor(uiControllerClient: UiControllerClient, inferenceClient: InferenceClient, stepReporter: StepReporter, retryStrategy: RetryStrategy);
     connect(): Promise<UiControllerClientConnectionState>;
     disconnect(): void;
     startVideoRecording(): Promise<void>;
@@ -19,7 +21,6 @@ export declare class ExecutionRuntime {
     executeInstruction(instruction: Instruction): Promise<void>;
     private readonly EXEC_REPETITION_COUNT;
     private executeCommandRepeatedly;
-    private readonly PREDICT_COMMAND_RETRY_COUNT;
     /**
      * Command prediction may fail, e.g., due to application still loading
      * --> retry with linear back-off
@@ -35,4 +36,5 @@ export declare class ExecutionRuntime {
     takeScreenshotIfImageisNotProvided(imagePath?: string): Promise<string>;
     getDetectedElements(instruction: string, customElementJson?: CustomElementJson[]): Promise<DetectedElement[]>;
     annotateImage(imagePath?: string, customElementJson?: CustomElementJson[], elements?: DetectedElement[]): Promise<Annotation>;
+    predictVQA(prompt: string, config?: object): Promise<any>;
 }

package/dist/esm/execution/execution-runtime.js

@@ -16,12 +16,12 @@ import { Annotation } from '../core/annotation/annotation';
 import { logger } from '../lib/logger';
 import { Base64Image } from '../utils/base_64_image/base-64-image';
 export class ExecutionRuntime {
-    constructor(uiControllerClient, inferenceClient, stepReporter) {
+    constructor(uiControllerClient, inferenceClient, stepReporter, retryStrategy) {
         this.uiControllerClient = uiControllerClient;
         this.inferenceClient = inferenceClient;
         this.stepReporter = stepReporter;
+        this.retryStrategy = retryStrategy;
         this.EXEC_REPETITION_COUNT = 25;
-        this.PREDICT_COMMAND_RETRY_COUNT = 2;
     }
     connect() {
         return __awaiter(this, void 0, void 0, function* () {
@@ -53,8 +53,8 @@ export class ExecutionRuntime {
         });
     }
     executeInstruction(instruction) {
-        var _a, _b, _c;
         return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b, _c;
             const controlCommand = yield this.predictCommandWithRetry(instruction);
             if (controlCommand.code === ControlCommandCode.OK) {
                 return this.requestControl(controlCommand);
@@ -67,8 +67,8 @@ export class ExecutionRuntime {
         });
     }
     executeCommandRepeatedly(instruction) {
-        var _a, _b, _c;
         return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b, _c;
             /* eslint-disable no-await-in-loop */
             for (let repeatCount = this.EXEC_REPETITION_COUNT; repeatCount >= 0; repeatCount -= 1) {
                 if (repeatCount === 0) {
@@ -87,6 +87,7 @@ export class ExecutionRuntime {
                     throw new ControlCommandError((_c = (_b = (_a = controlCommand.actions) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.text) !== null && _c !== void 0 ? _c : '');
                 }
             }
+            /* eslint-enable no-await-in-loop */
         });
     }
     /**
@@ -95,15 +96,15 @@ export class ExecutionRuntime {
      */
     /* eslint-disable-next-line consistent-return */
     predictCommandWithRetry(instruction) {
-        var _a, _b, _c;
         return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b, _c;
             let command = yield this.predictCommand(instruction);
             /* eslint-disable no-await-in-loop */
-            for (let k = 0; k < this.PREDICT_COMMAND_RETRY_COUNT; k += 1) {
+            for (let k = 0; k < this.retryStrategy.retryCount; k += 1) {
                 if (command.code === ControlCommandCode.OK) {
                     return command;
                 }
-                const msUntilRetry = k * 1000;
+                const msUntilRetry = this.retryStrategy.getDelay(k + 1);
                 logger.debug(`Wait ${msUntilRetry} and retry predicting command...`);
                 yield delay(msUntilRetry);
                 command = yield this.predictCommand(instruction, new ControlCommandError((_c = (_b = (_a = command.actions) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.text) !== null && _c !== void 0 ? _c : ''));
@@ -205,4 +206,10 @@ export class ExecutionRuntime {
             return this.inferenceClient.predictImageAnnotation(base64Image, customElements);
         });
     }
+    predictVQA(prompt, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const base64Image = yield this.takeScreenshotIfImageisNotProvided();
+            return this.inferenceClient.predictVQAAnswer(prompt, base64Image, config);
+        });
+    }
 }

package/dist/esm/execution/index.d.ts

@@ -1 +1,2 @@
 export * from './ui-control-client';
+export * from './retry-strategies';

package/dist/esm/execution/index.js

@@ -1 +1,2 @@
 export * from './ui-control-client';
+export * from './retry-strategies';

package/dist/esm/execution/inference-client.d.ts

@@ -3,6 +3,7 @@ import { ControlCommand } from '../core/ui-control-commands';
 import { CustomElement } from '../core/model/custom-element';
 import { Annotation } from '../core/annotation/annotation';
 import { DetectedElement } from '../core/model/annotation-result/detected-element';
+import { VQAInferenceResponseBody } from '../core/inference-response/inference-response';
 import { ModelCompositionBranch } from './model-composition-branch';
 export declare class InferenceClient {
     private readonly baseUrl;
@@ -16,8 +17,10 @@ export declare class InferenceClient {
     isImageRequired(instruction: string): Promise<boolean>;
     private resizeIfNeeded;
     inference(customElements?: CustomElement[], image?: string, instruction?: string): Promise<ControlCommand | Annotation>;
+    vqaInference(image: string, prompt: string, config?: object): Promise<VQAInferenceResponseBody>;
     private static logMetaInformation;
     predictControlCommand(instruction: string, customElements?: CustomElement[], image?: string): Promise<ControlCommand>;
     getDetectedElements(instruction: string, image: string, customElements?: CustomElement[]): Promise<DetectedElement[]>;
     predictImageAnnotation(image: string, customElements?: CustomElement[]): Promise<Annotation>;
+    predictVQAAnswer(prompt: string, image: string, config?: object): Promise<any>;
 }

package/dist/esm/execution/inference-client.js

@@ -29,6 +29,7 @@ export class InferenceClient {
         this.urls = {
             inference: urljoin(url, 'inference'),
             isImageRequired: urljoin(url, 'instruction', 'is-image-required'),
+            vqaInference: urljoin(url, 'vqa', 'inference'),
         };
         this.httpClient.urlsToRetry = Object.values(this.urls);
         if (this.resize !== undefined && this.resize <= 0) {
@@ -53,8 +54,8 @@ export class InferenceClient {
             return resizeBase64ImageWithSameRatio(image, this.resize);
         });
     }
-    inference(customElements = [], image, instruction) {
-        return __awaiter(this, void 0, void 0, function* () {
+    inference() {
+        return __awaiter(this, arguments, void 0, function* (customElements = [], image, instruction) {
             const resizedImage = yield this.resizeIfNeeded(customElements, image);
             const response = yield this.httpClient.post(this.urls.inference, this.urls.inference.includes('v4-experimental') ? {
                 image: resizedImage.base64Image,
@@ -71,13 +72,24 @@ export class InferenceClient {
             return InferenceResponse.fromJson(response.body, resizedImage.resizeRatio, image);
         });
     }
+    vqaInference(image, prompt, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const response = yield this.httpClient.post(this.urls.vqaInference, {
+                config,
+                image,
+                prompt,
+            });
+            InferenceClient.logMetaInformation(response);
+            return response.body;
+        });
+    }
     static logMetaInformation(response) {
         if (response.headers['askui-usage-warnings'] !== undefined) {
             logger.warn(response.headers['askui-usage-warnings']);
         }
     }
-    predictControlCommand(instruction, customElements = [], image) {
-        return __awaiter(this, void 0, void 0, function* () {
+    predictControlCommand(instruction_1) {
+        return __awaiter(this, arguments, void 0, function* (instruction, customElements = [], image) {
             const inferenceResponse = yield this.inference(customElements, image, instruction);
             if (!(inferenceResponse instanceof ControlCommand)) {
                 throw new InferenceResponseError('Internal Error. Can not execute command');
@@ -85,8 +97,8 @@ export class InferenceClient {
             return inferenceResponse;
         });
     }
-    getDetectedElements(instruction, image, customElements = []) {
-        return __awaiter(this, void 0, void 0, function* () {
+    getDetectedElements(instruction_1, image_1) {
+        return __awaiter(this, arguments, void 0, function* (instruction, image, customElements = []) {
             const inferenceResponse = yield this.inference(customElements, image, instruction);
             if (!(inferenceResponse instanceof Annotation)) {
                 throw new InferenceResponseError('Internal Error. Unable to get the detected elements');
@@ -94,8 +106,8 @@ export class InferenceClient {
             return inferenceResponse.detected_elements;
         });
     }
-    predictImageAnnotation(image, customElements = []) {
-        return __awaiter(this, void 0, void 0, function* () {
+    predictImageAnnotation(image_1) {
+        return __awaiter(this, arguments, void 0, function* (image, customElements = []) {
             const inferenceResponse = yield this.inference(customElements, image);
             if (!(inferenceResponse instanceof Annotation)) {
                 throw new InferenceResponseError('Internal Error. Can not execute annotation');
@@ -103,4 +115,17 @@ export class InferenceClient {
             return inferenceResponse;
         });
     }
+    predictVQAAnswer(prompt, image, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const inferenceResponse = yield this.vqaInference(image, prompt, config);
+            const { response } = inferenceResponse.data;
+            try {
+                return JSON.parse(response);
+            }
+            catch (error) {
+                logger.warn(`Response is no valid JSON: ${response}`);
+            }
+            return response;
+        });
+    }
 }

package/dist/esm/execution/retry-strategies/exponential-retry-strategy.d.ts

@@ -0,0 +1,19 @@
+import { RetryStrategy } from './retry-strategy';
+/**
+ * ExponentialRetryStrategy implements a retry strategy that uses an exponential backoff algorithm.
+ */
+export declare class ExponentialRetryStrategy implements RetryStrategy {
+    baseDelayMs: number;
+    retryCount: number;
+    /**
+     * @param baseDelayMs - The initial delay before the first retry (default is 1000ms)
+     * @param retryCount - The maximum number of retries (default is 3)
+     */
+    constructor(baseDelayMs?: number, retryCount?: number);
+    /**
+     * Calculates the delay for a given retry attempt using exponential backoff.
+     * @param attempt - The current retry attempt number (0-based)
+     * @returns The delay in milliseconds for the current attempt
+     */
+    getDelay(attempt: number): number;
+}

package/dist/esm/execution/retry-strategies/exponential-retry-strategy.js

@@ -0,0 +1,21 @@
+/**
+ * ExponentialRetryStrategy implements a retry strategy that uses an exponential backoff algorithm.
+ */
+export class ExponentialRetryStrategy {
+    /**
+     * @param baseDelayMs - The initial delay before the first retry (default is 1000ms)
+     * @param retryCount - The maximum number of retries (default is 3)
+     */
+    constructor(baseDelayMs = 1000, retryCount = 3) {
+        this.baseDelayMs = baseDelayMs;
+        this.retryCount = retryCount;
+    }
+    /**
+     * Calculates the delay for a given retry attempt using exponential backoff.
+     * @param attempt - The current retry attempt number (0-based)
+     * @returns The delay in milliseconds for the current attempt
+     */
+    getDelay(attempt) {
+        return this.baseDelayMs * (Math.pow(2, attempt));
+    }
+}

package/dist/esm/execution/retry-strategies/fixed-retry-strategy.d.ts

@@ -0,0 +1,19 @@
+import { RetryStrategy } from './retry-strategy';
+/**
+ * FixedRetryStrategy implements a retry strategy that uses a constant delay for each retry attempt.
+ */
+export declare class FixedRetryStrategy implements RetryStrategy {
+    baseDelayMs: number;
+    retryCount: number;
+    /**
+     * @param baseDelayMs - The constant delay before each retry (default is 1000ms)
+     * @param retryCount - The maximum number of retries (default is 3)
+     */
+    constructor(baseDelayMs?: number, retryCount?: number);
+    /**
+     * Returns the fixed delay for each retry attempt.
+     * @param _attempt - The current retry attempt number (not used in this strategy)
+     * @returns The fixed delay in milliseconds for the current attempt
+     */
+    getDelay(_attempt: number): number;
+}

package/dist/esm/execution/retry-strategies/fixed-retry-strategy.js

@@ -0,0 +1,21 @@
+/**
+ * FixedRetryStrategy implements a retry strategy that uses a constant delay for each retry attempt.
+ */
+export class FixedRetryStrategy {
+    /**
+     * @param baseDelayMs - The constant delay before each retry (default is 1000ms)
+     * @param retryCount - The maximum number of retries (default is 3)
+     */
+    constructor(baseDelayMs = 1000, retryCount = 3) {
+        this.baseDelayMs = baseDelayMs;
+        this.retryCount = retryCount;
+    }
+    /**
+     * Returns the fixed delay for each retry attempt.
+     * @param _attempt - The current retry attempt number (not used in this strategy)
+     * @returns The fixed delay in milliseconds for the current attempt
+     */
+    getDelay(_attempt) {
+        return this.baseDelayMs;
+    }
+}

package/dist/esm/execution/retry-strategies/index.d.ts

@@ -0,0 +1,4 @@
+export * from './exponential-retry-strategy';
+export * from './linear-retry-strategy';
+export * from './fixed-retry-strategy';
+export * from './retry-strategy';

package/dist/esm/execution/retry-strategies/index.js

@@ -0,0 +1,4 @@
+export * from './exponential-retry-strategy';
+export * from './linear-retry-strategy';
+export * from './fixed-retry-strategy';
+export * from './retry-strategy';

package/dist/esm/execution/retry-strategies/linear-retry-strategy.d.ts

@@ -0,0 +1,19 @@
+import { RetryStrategy } from './retry-strategy';
+/**
+ * LinearRetryStrategy implements a retry strategy that uses a linear backoff algorithm.
+ */
+export declare class LinearRetryStrategy implements RetryStrategy {
+    baseDelayMs: number;
+    retryCount: number;
+    /**
+     * @param baseDelayMs - The initial delay before the first retry (default is 1000ms)
+     * @param retryCount - The maximum number of retries (default is 3)
+     */
+    constructor(baseDelayMs?: number, retryCount?: number);
+    /**
+     * Calculates the delay for a given retry attempt using linear backoff.
+     * @param attempt - The current retry attempt number (1-based)
+     * @returns The delay in milliseconds for the current attempt
+     */
+    getDelay(attempt: number): number;
+}

package/dist/esm/execution/retry-strategies/linear-retry-strategy.js

@@ -0,0 +1,21 @@
+/**
+ * LinearRetryStrategy implements a retry strategy that uses a linear backoff algorithm.
+ */
+export class LinearRetryStrategy {
+    /**
+     * @param baseDelayMs - The initial delay before the first retry (default is 1000ms)
+     * @param retryCount - The maximum number of retries (default is 3)
+     */
+    constructor(baseDelayMs = 1000, retryCount = 3) {
+        this.baseDelayMs = baseDelayMs;
+        this.retryCount = retryCount;
+    }
+    /**
+     * Calculates the delay for a given retry attempt using linear backoff.
+     * @param attempt - The current retry attempt number (1-based)
+     * @returns The delay in milliseconds for the current attempt
+     */
+    getDelay(attempt) {
+        return this.baseDelayMs * attempt;
+    }
+}

package/dist/esm/execution/retry-strategies/retry-strategy.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Interface representing a retry strategy for operations.
+ */
+export interface RetryStrategy {
+    /**
+     * The base delay in milliseconds before the first retry.
+     */
+    baseDelayMs: number;
+    /**
+     * The maximum number of retry attempts.
+     */
+    retryCount: number;
+    /**
+     * Function to calculate the delay before the next retry attempt.
+     * @param attempt - The current retry attempt number (0-based).
+     * @returns The delay in milliseconds for the next retry.
+     */
+    getDelay(attempt: number): number;
+}

package/dist/esm/execution/retry-strategies/retry-strategy.js

@@ -0,0 +1 @@
+export {};