askui 0.20.7 → 0.20.9

package/dist/cjs/core/inference-response/inference-response.d.ts

@@ -5,6 +5,11 @@ export interface InferenceResponseBody {
     type: ModelType;
     data: ModelType extends 'COMMANDS' ? ControlCommand : Annotation;
 }
+export interface VQAInferenceResponseBody {
+    data: {
+        response: string;
+    };
+}
 export declare class InferenceResponse {
     static fromJson(json: InferenceResponseBody, resizeRatio?: number, image?: string): ControlCommand | Annotation;
     static createModels(type: ModelType, data: ModelType extends 'COMMANDS' ? ControlCommand : Annotation, resizeRatio: number, image?: string): ControlCommand | Annotation;

package/dist/cjs/core/inference-response/model-type.d.ts

@@ -1 +1 @@
-export declare type ModelType = 'DETECTED_ELEMENTS' | 'COMMANDS';
+export type ModelType = 'DETECTED_ELEMENTS' | 'COMMANDS';

package/dist/cjs/core/model/custom-element.d.ts

@@ -5,16 +5,16 @@ export declare class CustomElement implements CustomElementJson {
     threshold?: number | undefined;
     stopThreshold?: number | undefined;
     rotationDegreePerStep?: number | undefined;
-    imageCompareFormat?: "RGB" | "grayscale" | "edges" | undefined;
-    mask?: {
+    imageCompareFormat?: ("RGB" | "grayscale" | "edges") | undefined;
+    mask?: ({
         x: number;
         y: number;
-    }[] | undefined;
+    })[] | undefined;
     private static schema;
-    constructor(customImage: string, name?: string | undefined, threshold?: number | undefined, stopThreshold?: number | undefined, rotationDegreePerStep?: number | undefined, imageCompareFormat?: "RGB" | "grayscale" | "edges" | undefined, mask?: {
+    constructor(customImage: string, name?: string | undefined, threshold?: number | undefined, stopThreshold?: number | undefined, rotationDegreePerStep?: number | undefined, imageCompareFormat?: ("RGB" | "grayscale" | "edges") | undefined, mask?: ({
         x: number;
         y: number;
-    }[] | undefined);
+    })[] | undefined);
     static fromJsonListWithImagePathOrImage(ceJson?: CustomElementJson[]): Promise<CustomElement[]>;
     static fromJsonWithImagePathOrImage(ceJson: CustomElementJson): Promise<CustomElement>;
     static fromJson(ceJson: CustomElementJson): CustomElement;

package/dist/cjs/core/model/custom-element.js

@@ -22,8 +22,8 @@ class CustomElement {
         this.imageCompareFormat = imageCompareFormat;
         this.mask = mask;
     }
-    static fromJsonListWithImagePathOrImage(ceJson = []) {
-        return __awaiter(this, void 0, void 0, function* () {
+    static fromJsonListWithImagePathOrImage() {
+        return __awaiter(this, arguments, void 0, function* (ceJson = []) {
             return Promise.all(ceJson.map((customElement) => CustomElement.fromJsonWithImagePathOrImage(customElement)));
         });
     }

package/dist/cjs/core/reporting/index.js

@@ -1,7 +1,11 @@
 "use strict";
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
 }) : (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
     o[k2] = m[k];

package/dist/cjs/core/reporting/snapshot-detail-level.d.ts

@@ -1,4 +1,4 @@
-export declare type SnapshotDetailLevel = 
+export type SnapshotDetailLevel = 
 /**
  * Details of snapshot, e.g., screenshot or detected elements, may or may not be available
  *  depending on if they are required by the step. There are not guarantees made.

package/dist/cjs/core/reporting/snapshot-detail-level.js

@@ -1,10 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.maxSnapshotDetailLevel = void 0;
+exports.maxSnapshotDetailLevel = maxSnapshotDetailLevel;
 const levelsOrdered = ['required', 'onFailure', 'begin', 'always'];
 const levelValues = levelsOrdered.reduce((acc, level, index) => (Object.assign(Object.assign({}, acc), { [level]: index })), {});
 function maxSnapshotDetailLevel(...levels) {
     var _a;
     return (_a = levelsOrdered[Math.max(...levels.map((level) => levelValues[level]))]) !== null && _a !== void 0 ? _a : 'required';
 }
-exports.maxSnapshotDetailLevel = maxSnapshotDetailLevel;

package/dist/cjs/core/reporting/step-status-end.d.ts

@@ -1 +1 @@
-export declare type StepStatusEnd = 'passed' | 'failed' | 'erroneous';
+export type StepStatusEnd = 'passed' | 'failed' | 'erroneous';

package/dist/cjs/core/reporting/step-status.d.ts

@@ -10,4 +10,4 @@
  *  e.g., the user has no usage left, response of inference backend cannot be processed,
  *  e.g., because lib version is outdated, etc. (currently not supported, everything "failed")
  */
-export declare type StepStatus = 'passed' | 'failed' | 'pending' | 'running' | 'erroneous';
+export type StepStatus = 'passed' | 'failed' | 'pending' | 'running' | 'erroneous';

package/dist/cjs/core/runner-protocol/index.js

@@ -1,7 +1,11 @@
 "use strict";
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
 }) : (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
     o[k2] = m[k];

package/dist/cjs/core/ui-control-commands/action.d.ts

@@ -1,5 +1,5 @@
 import { InputEvent } from './input-event';
-export declare type ActionParameters = {
+export type ActionParameters = {
     [key: string]: string | number | boolean | string[] | ActionParameters;
 };
 export declare class Action {

package/dist/cjs/core/ui-control-commands/control-command-code.js

@@ -6,4 +6,4 @@ var ControlCommandCode;
     ControlCommandCode["ERROR"] = "ERROR";
     ControlCommandCode["WARNING"] = "WARNING";
     ControlCommandCode["OK"] = "OK";
-})(ControlCommandCode = exports.ControlCommandCode || (exports.ControlCommandCode = {}));
+})(ControlCommandCode || (exports.ControlCommandCode = ControlCommandCode = {}));

package/dist/cjs/core/ui-control-commands/input-event.js

@@ -21,4 +21,4 @@ var InputEvent;
     InputEvent["MOUSE_DOWN"] = "MOUSE_DOWN";
     InputEvent["MOUSE_UP"] = "MOUSE_UP";
     InputEvent["EXECUTE_COMMAND"] = "EXECUTE_COMMAND";
-})(InputEvent = exports.InputEvent || (exports.InputEvent = {}));
+})(InputEvent || (exports.InputEvent = InputEvent = {}));

package/dist/cjs/execution/dsl.d.ts

@@ -3,12 +3,12 @@ import { DetectedElement } from '../core/model/annotation-result/detected-elemen
 export declare enum Separators {
     STRING = "<|string|>"
 }
-export declare type INTERSECTION_AREA = 'element_center_line' | 'element_edge_area' | 'display_edge_area';
-export declare type PC_KEY = 'backspace' | 'delete' | 'enter' | 'tab' | 'escape' | 'up' | 'down' | 'right' | 'left' | 'home' | 'end' | 'pageup' | 'pagedown' | 'f1' | 'f2' | 'f3' | 'f4' | 'f5' | 'f6' | 'f7' | 'f8' | 'f9' | 'f10' | 'f11' | 'f12' | 'space' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '!' | '"' | '#' | '$' | '%' | '&' | "'" | '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | ':' | ';' | '<' | '=' | '>' | '?' | '@' | '[' | '\\' | ']' | '^' | '_' | '`' | '{' | '|' | '}' | '~ ';
-export declare type ANDROID_KEY = 'home' | 'back' | 'call' | 'endcall' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | 'star' | 'pound' | 'dpad_up' | 'dpad_down' | 'dpad_left' | 'dpad_right' | 'dpad_center' | 'volume_up' | 'volume_down' | 'power' | 'camera' | 'clear' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | 'comma' | 'period' | 'alt_left' | 'alt_right' | 'shift_left' | 'shift_right' | 'tab' | 'space' | 'sym' | 'explorer' | 'envelope' | 'enter' | 'del' | 'grave' | 'minus' | 'equals' | 'left_bracket' | 'right_bracket' | 'backslash' | 'semicolon' | 'apostrophe' | 'slash' | 'at' | 'num' | 'headsethook' | 'focus' | 'plus' | 'menu' | 'notification' | 'search' | 'media_play_pause' | 'media_stop' | 'media_next' | 'media_previous' | 'media_rewind' | 'media_fast_forward' | 'mute' | 'page_up' | 'page_down' | 'switch_charset' | 'escape' | 'forward_del' | 'ctrl_left' | 'ctrl_right' | 'caps_lock' | 'scroll_lock' | 'function' | 'break' | 'move_home' | 'move_end' | 'insert' | 'forward' | 'media_play' | 'media_pause' | 'media_close' | 'media_eject' | 'media_record' | 'f1' | 'f2' | 'f3' | 'f4' | 'f5' | 'f6' | 'f7' | 'f8' | 'f9' | 'f10' | 'f11' | 'f12' | 'num_lock' | 'numpad_0' | 'numpad_1' | 'numpad_2' | 'numpad_3' | 'numpad_4' | 'numpad_5' | 'numpad_6' | 'numpad_7' | 'numpad_8' | 'numpad_9' | 'numpad_divide' | 'numpad_multiply' | 'numpad_subtract' | 'numpad_add' | 'numpad_dot' | 'numpad_comma' | 'numpad_enter' | 'numpad_equals' | 'numpad_left_paren' | 'numpad_right_paren' | 'volume_mute' | 'info' | 'channel_up' | 'channel_down' | 'zoom_in' | 'zoom_out' | 'window' | 'guide' | 'bookmark' | 'captions' | 'settings' | 'app_switch' | 'language_switch' | 'contacts' | 'calendar' | 'music' | 'calculator' | 'assist' | 'brightness_down' | 'brightness_up' | 'media_audio_track' | 'sleep' | 'wakeup' | 'pairing' | 'media_top_menu' | 'last_channel' | 'tv_data_service' | 'voice_assist' | 'help' | 'navigate_previous' | 'navigate_next' | 'navigate_in' | 'navigate_out' | 'dpad_up_left' | 'dpad_down_left' | 'dpad_up_right' | 'dpad_down_right' | 'media_skip_forward' | 'media_skip_backward' | 'media_step_forward' | 'media_step_backward' | 'soft_sleep' | 'cut' | 'copy' | 'paste' | 'all_apps' | 'refresh';
-export declare type MODIFIER_KEY = 'command' | 'alt' | 'control' | 'shift' | 'right_shift';
-export declare type COLOR = 'black' | 'white' | 'red' | 'green' | 'yellow green' | 'orange' | 'yellow' | 'purple' | 'pink' | 'gray' | 'lime green' | 'royal blue';
-export declare type PC_AND_MODIFIER_KEY = 'command' | 'alt' | 'control' | 'shift' | 'right_shift' | 'backspace' | 'delete' | 'enter' | 'tab' | 'escape' | 'up' | 'down' | 'right' | 'left' | 'home' | 'end' | 'pageup' | 'pagedown' | 'f1' | 'f2' | 'f3' | 'f4' | 'f5' | 'f6' | 'f7' | 'f8' | 'f9' | 'f10' | 'f11' | 'f12' | 'space' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '!' | '"' | '#' | '$' | '%' | '&' | "'" | '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | ':' | ';' | '<' | '=' | '>' | '?' | '@' | '[' | '\\' | ']' | '^' | '_' | '`' | '{' | '|' | '}' | '~ ';
+export type INTERSECTION_AREA = 'element_center_line' | 'element_edge_area' | 'display_edge_area';
+export type PC_KEY = 'backspace' | 'delete' | 'enter' | 'tab' | 'escape' | 'up' | 'down' | 'right' | 'left' | 'home' | 'end' | 'pageup' | 'pagedown' | 'f1' | 'f2' | 'f3' | 'f4' | 'f5' | 'f6' | 'f7' | 'f8' | 'f9' | 'f10' | 'f11' | 'f12' | 'space' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '!' | '"' | '#' | '$' | '%' | '&' | "'" | '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | ':' | ';' | '<' | '=' | '>' | '?' | '@' | '[' | '\\' | ']' | '^' | '_' | '`' | '{' | '|' | '}' | '~ ';
+export type ANDROID_KEY = 'home' | 'back' | 'call' | 'endcall' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | 'star' | 'pound' | 'dpad_up' | 'dpad_down' | 'dpad_left' | 'dpad_right' | 'dpad_center' | 'volume_up' | 'volume_down' | 'power' | 'camera' | 'clear' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | 'comma' | 'period' | 'alt_left' | 'alt_right' | 'shift_left' | 'shift_right' | 'tab' | 'space' | 'sym' | 'explorer' | 'envelope' | 'enter' | 'del' | 'grave' | 'minus' | 'equals' | 'left_bracket' | 'right_bracket' | 'backslash' | 'semicolon' | 'apostrophe' | 'slash' | 'at' | 'num' | 'headsethook' | 'focus' | 'plus' | 'menu' | 'notification' | 'search' | 'media_play_pause' | 'media_stop' | 'media_next' | 'media_previous' | 'media_rewind' | 'media_fast_forward' | 'mute' | 'page_up' | 'page_down' | 'switch_charset' | 'escape' | 'forward_del' | 'ctrl_left' | 'ctrl_right' | 'caps_lock' | 'scroll_lock' | 'function' | 'break' | 'move_home' | 'move_end' | 'insert' | 'forward' | 'media_play' | 'media_pause' | 'media_close' | 'media_eject' | 'media_record' | 'f1' | 'f2' | 'f3' | 'f4' | 'f5' | 'f6' | 'f7' | 'f8' | 'f9' | 'f10' | 'f11' | 'f12' | 'num_lock' | 'numpad_0' | 'numpad_1' | 'numpad_2' | 'numpad_3' | 'numpad_4' | 'numpad_5' | 'numpad_6' | 'numpad_7' | 'numpad_8' | 'numpad_9' | 'numpad_divide' | 'numpad_multiply' | 'numpad_subtract' | 'numpad_add' | 'numpad_dot' | 'numpad_comma' | 'numpad_enter' | 'numpad_equals' | 'numpad_left_paren' | 'numpad_right_paren' | 'volume_mute' | 'info' | 'channel_up' | 'channel_down' | 'zoom_in' | 'zoom_out' | 'window' | 'guide' | 'bookmark' | 'captions' | 'settings' | 'app_switch' | 'language_switch' | 'contacts' | 'calendar' | 'music' | 'calculator' | 'assist' | 'brightness_down' | 'brightness_up' | 'media_audio_track' | 'sleep' | 'wakeup' | 'pairing' | 'media_top_menu' | 'last_channel' | 'tv_data_service' | 'voice_assist' | 'help' | 'navigate_previous' | 'navigate_next' | 'navigate_in' | 'navigate_out' | 'dpad_up_left' | 'dpad_down_left' | 'dpad_up_right' | 'dpad_down_right' | 'media_skip_forward' | 'media_skip_backward' | 'media_step_forward' | 'media_step_backward' | 'soft_sleep' | 'cut' | 'copy' | 'paste' | 'all_apps' | 'refresh';
+export type MODIFIER_KEY = 'command' | 'alt' | 'control' | 'shift' | 'right_shift';
+export type COLOR = 'black' | 'white' | 'red' | 'green' | 'yellow green' | 'orange' | 'yellow' | 'purple' | 'pink' | 'gray' | 'lime green' | 'royal blue';
+export type PC_AND_MODIFIER_KEY = 'command' | 'alt' | 'control' | 'shift' | 'right_shift' | 'backspace' | 'delete' | 'enter' | 'tab' | 'escape' | 'up' | 'down' | 'right' | 'left' | 'home' | 'end' | 'pageup' | 'pagedown' | 'f1' | 'f2' | 'f3' | 'f4' | 'f5' | 'f6' | 'f7' | 'f8' | 'f9' | 'f10' | 'f11' | 'f12' | 'space' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '!' | '"' | '#' | '$' | '%' | '&' | "'" | '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | ':' | ';' | '<' | '=' | '>' | '?' | '@' | '[' | '\\' | ']' | '^' | '_' | '`' | '{' | '|' | '}' | '~ ';
 export interface CommandExecutorContext {
     customElementsJson: CustomElementJson[];
     aiElementNames: string[];

package/dist/cjs/execution/dsl.js

@@ -36,7 +36,7 @@ function rewriteStackTraceForError(error, newStackTrace) {
 var Separators;
 (function (Separators) {
     Separators["STRING"] = "<|string|>";
-})(Separators = exports.Separators || (exports.Separators = {}));
+})(Separators || (exports.Separators = Separators = {}));
 class FluentBase {
     constructor(prev) {
         this.prev = prev;

package/dist/cjs/execution/execution-runtime.d.ts

@@ -5,11 +5,13 @@ import { CustomElementJson } from '../core/model/custom-element-json';
 import { DetectedElement } from '../core/model/annotation-result/detected-element';
 import { UiControllerClientConnectionState } from './ui-controller-client-connection-state';
 import { Instruction, StepReporter } from '../core/reporting';
+import { RetryStrategy } from './retry-strategies/retry-strategy';
 export declare class ExecutionRuntime {
     private uiControllerClient;
     private inferenceClient;
     private stepReporter;
-    constructor(uiControllerClient: UiControllerClient, inferenceClient: InferenceClient, stepReporter: StepReporter);
+    private retryStrategy;
+    constructor(uiControllerClient: UiControllerClient, inferenceClient: InferenceClient, stepReporter: StepReporter, retryStrategy: RetryStrategy);
     connect(): Promise<UiControllerClientConnectionState>;
     disconnect(): void;
     startVideoRecording(): Promise<void>;
@@ -19,7 +21,6 @@ export declare class ExecutionRuntime {
     executeInstruction(instruction: Instruction): Promise<void>;
     private readonly EXEC_REPETITION_COUNT;
     private executeCommandRepeatedly;
-    private readonly PREDICT_COMMAND_RETRY_COUNT;
     /**
      * Command prediction may fail, e.g., due to application still loading
      * --> retry with linear back-off
@@ -35,4 +36,5 @@ export declare class ExecutionRuntime {
     takeScreenshotIfImageisNotProvided(imagePath?: string): Promise<string>;
     getDetectedElements(instruction: string, customElementJson?: CustomElementJson[]): Promise<DetectedElement[]>;
     annotateImage(imagePath?: string, customElementJson?: CustomElementJson[], elements?: DetectedElement[]): Promise<Annotation>;
+    predictVQA(prompt: string, config?: object): Promise<any>;
 }

package/dist/cjs/execution/execution-runtime.js

@@ -19,12 +19,12 @@ const annotation_1 = require("../core/annotation/annotation");
 const logger_1 = require("../lib/logger");
 const base_64_image_1 = require("../utils/base_64_image/base-64-image");
 class ExecutionRuntime {
-    constructor(uiControllerClient, inferenceClient, stepReporter) {
+    constructor(uiControllerClient, inferenceClient, stepReporter, retryStrategy) {
         this.uiControllerClient = uiControllerClient;
         this.inferenceClient = inferenceClient;
         this.stepReporter = stepReporter;
+        this.retryStrategy = retryStrategy;
         this.EXEC_REPETITION_COUNT = 25;
-        this.PREDICT_COMMAND_RETRY_COUNT = 2;
     }
     connect() {
         return __awaiter(this, void 0, void 0, function* () {
@@ -56,8 +56,8 @@ class ExecutionRuntime {
         });
     }
     executeInstruction(instruction) {
-        var _a, _b, _c;
         return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b, _c;
             const controlCommand = yield this.predictCommandWithRetry(instruction);
             if (controlCommand.code === ui_control_commands_1.ControlCommandCode.OK) {
                 return this.requestControl(controlCommand);
@@ -70,8 +70,8 @@ class ExecutionRuntime {
         });
     }
     executeCommandRepeatedly(instruction) {
-        var _a, _b, _c;
         return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b, _c;
             /* eslint-disable no-await-in-loop */
             for (let repeatCount = this.EXEC_REPETITION_COUNT; repeatCount >= 0; repeatCount -= 1) {
                 if (repeatCount === 0) {
@@ -90,6 +90,7 @@ class ExecutionRuntime {
                     throw new control_command_error_1.ControlCommandError((_c = (_b = (_a = controlCommand.actions) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.text) !== null && _c !== void 0 ? _c : '');
                 }
             }
+            /* eslint-enable no-await-in-loop */
         });
     }
     /**
@@ -98,15 +99,15 @@ class ExecutionRuntime {
      */
     /* eslint-disable-next-line consistent-return */
     predictCommandWithRetry(instruction) {
-        var _a, _b, _c;
         return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b, _c;
             let command = yield this.predictCommand(instruction);
             /* eslint-disable no-await-in-loop */
-            for (let k = 0; k < this.PREDICT_COMMAND_RETRY_COUNT; k += 1) {
+            for (let k = 0; k < this.retryStrategy.retryCount; k += 1) {
                 if (command.code === ui_control_commands_1.ControlCommandCode.OK) {
                     return command;
                 }
-                const msUntilRetry = k * 1000;
+                const msUntilRetry = this.retryStrategy.getDelay(k + 1);
                 logger_1.logger.debug(`Wait ${msUntilRetry} and retry predicting command...`);
                 yield (0, misc_1.delay)(msUntilRetry);
                 command = yield this.predictCommand(instruction, new control_command_error_1.ControlCommandError((_c = (_b = (_a = command.actions) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.text) !== null && _c !== void 0 ? _c : ''));
@@ -208,5 +209,11 @@ class ExecutionRuntime {
             return this.inferenceClient.predictImageAnnotation(base64Image, customElements);
         });
     }
+    predictVQA(prompt, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const base64Image = yield this.takeScreenshotIfImageisNotProvided();
+            return this.inferenceClient.predictVQAAnswer(prompt, base64Image, config);
+        });
+    }
 }
 exports.ExecutionRuntime = ExecutionRuntime;

package/dist/cjs/execution/index.d.ts

@@ -1 +1,2 @@
 export * from './ui-control-client';
+export * from './retry-strategies';

package/dist/cjs/execution/index.js

@@ -1,7 +1,11 @@
 "use strict";
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
 }) : (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
     o[k2] = m[k];
@@ -11,3 +15,4 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./ui-control-client"), exports);
+__exportStar(require("./retry-strategies"), exports);

package/dist/cjs/execution/inference-client.d.ts

@@ -3,6 +3,7 @@ import { ControlCommand } from '../core/ui-control-commands';
 import { CustomElement } from '../core/model/custom-element';
 import { Annotation } from '../core/annotation/annotation';
 import { DetectedElement } from '../core/model/annotation-result/detected-element';
+import { VQAInferenceResponseBody } from '../core/inference-response/inference-response';
 import { ModelCompositionBranch } from './model-composition-branch';
 export declare class InferenceClient {
     private readonly baseUrl;
@@ -16,8 +17,10 @@ export declare class InferenceClient {
     isImageRequired(instruction: string): Promise<boolean>;
     private resizeIfNeeded;
     inference(customElements?: CustomElement[], image?: string, instruction?: string): Promise<ControlCommand | Annotation>;
+    vqaInference(image: string, prompt: string, config?: object): Promise<VQAInferenceResponseBody>;
     private static logMetaInformation;
     predictControlCommand(instruction: string, customElements?: CustomElement[], image?: string): Promise<ControlCommand>;
     getDetectedElements(instruction: string, image: string, customElements?: CustomElement[]): Promise<DetectedElement[]>;
     predictImageAnnotation(image: string, customElements?: CustomElement[]): Promise<Annotation>;
+    predictVQAAnswer(prompt: string, image: string, config?: object): Promise<any>;
 }

package/dist/cjs/execution/inference-client.js

@@ -35,6 +35,7 @@ class InferenceClient {
         this.urls = {
             inference: (0, url_join_1.default)(url, 'inference'),
             isImageRequired: (0, url_join_1.default)(url, 'instruction', 'is-image-required'),
+            vqaInference: (0, url_join_1.default)(url, 'vqa', 'inference'),
         };
         this.httpClient.urlsToRetry = Object.values(this.urls);
         if (this.resize !== undefined && this.resize <= 0) {
@@ -59,8 +60,8 @@ class InferenceClient {
             return (0, transformations_1.resizeBase64ImageWithSameRatio)(image, this.resize);
         });
     }
-    inference(customElements = [], image, instruction) {
-        return __awaiter(this, void 0, void 0, function* () {
+    inference() {
+        return __awaiter(this, arguments, void 0, function* (customElements = [], image, instruction) {
             const resizedImage = yield this.resizeIfNeeded(customElements, image);
             const response = yield this.httpClient.post(this.urls.inference, this.urls.inference.includes('v4-experimental') ? {
                 image: resizedImage.base64Image,
@@ -77,13 +78,24 @@ class InferenceClient {
             return ui_control_commands_1.InferenceResponse.fromJson(response.body, resizedImage.resizeRatio, image);
         });
     }
+    vqaInference(image, prompt, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const response = yield this.httpClient.post(this.urls.vqaInference, {
+                config,
+                image,
+                prompt,
+            });
+            InferenceClient.logMetaInformation(response);
+            return response.body;
+        });
+    }
     static logMetaInformation(response) {
         if (response.headers['askui-usage-warnings'] !== undefined) {
             logger_1.logger.warn(response.headers['askui-usage-warnings']);
         }
     }
-    predictControlCommand(instruction, customElements = [], image) {
-        return __awaiter(this, void 0, void 0, function* () {
+    predictControlCommand(instruction_1) {
+        return __awaiter(this, arguments, void 0, function* (instruction, customElements = [], image) {
             const inferenceResponse = yield this.inference(customElements, image, instruction);
             if (!(inferenceResponse instanceof ui_control_commands_1.ControlCommand)) {
                 throw new inference_response_error_1.InferenceResponseError('Internal Error. Can not execute command');
@@ -91,8 +103,8 @@ class InferenceClient {
             return inferenceResponse;
         });
     }
-    getDetectedElements(instruction, image, customElements = []) {
-        return __awaiter(this, void 0, void 0, function* () {
+    getDetectedElements(instruction_1, image_1) {
+        return __awaiter(this, arguments, void 0, function* (instruction, image, customElements = []) {
             const inferenceResponse = yield this.inference(customElements, image, instruction);
             if (!(inferenceResponse instanceof annotation_1.Annotation)) {
                 throw new inference_response_error_1.InferenceResponseError('Internal Error. Unable to get the detected elements');
@@ -100,8 +112,8 @@ class InferenceClient {
             return inferenceResponse.detected_elements;
         });
     }
-    predictImageAnnotation(image, customElements = []) {
-        return __awaiter(this, void 0, void 0, function* () {
+    predictImageAnnotation(image_1) {
+        return __awaiter(this, arguments, void 0, function* (image, customElements = []) {
             const inferenceResponse = yield this.inference(customElements, image);
             if (!(inferenceResponse instanceof annotation_1.Annotation)) {
                 throw new inference_response_error_1.InferenceResponseError('Internal Error. Can not execute annotation');
@@ -109,5 +121,18 @@ class InferenceClient {
             return inferenceResponse;
         });
     }
+    predictVQAAnswer(prompt, image, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const inferenceResponse = yield this.vqaInference(image, prompt, config);
+            const { response } = inferenceResponse.data;
+            try {
+                return JSON.parse(response);
+            }
+            catch (error) {
+                logger_1.logger.warn(`Response is no valid JSON: ${response}`);
+            }
+            return response;
+        });
+    }
 }
 exports.InferenceClient = InferenceClient;

package/dist/cjs/execution/misc.js

@@ -1,7 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.delay = void 0;
+exports.delay = delay;
 function delay(ms) {
     return new Promise((resolve) => { setTimeout(resolve, ms); });
 }
-exports.delay = delay;

package/dist/cjs/execution/read-credentials.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.readCredentials = void 0;
+exports.readCredentials = readCredentials;
 const lib_1 = require("../lib");
 const read_environment_credentials_1 = require("./read-environment-credentials");
 function readCredentials(clientArgs) {
@@ -21,4 +21,3 @@ function readCredentials(clientArgs) {
     }
     return credentials;
 }
-exports.readCredentials = readCredentials;

package/dist/cjs/execution/read-environment-credentials.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.envCredentials = void 0;
+exports.envCredentials = envCredentials;
 const lib_1 = require("../lib");
 function envCredentials() {
     const envToken = process.env['ASKUI_TOKEN'];
@@ -14,4 +14,3 @@ function envCredentials() {
     }
     return undefined;
 }
-exports.envCredentials = envCredentials;

package/dist/cjs/execution/retry-strategies/exponential-retry-strategy.d.ts

@@ -0,0 +1,19 @@
+import { RetryStrategy } from './retry-strategy';
+/**
+ * ExponentialRetryStrategy implements a retry strategy that uses an exponential backoff algorithm.
+ */
+export declare class ExponentialRetryStrategy implements RetryStrategy {
+    baseDelayMs: number;
+    retryCount: number;
+    /**
+     * @param baseDelayMs - The initial delay before the first retry (default is 1000ms)
+     * @param retryCount - The maximum number of retries (default is 3)
+     */
+    constructor(baseDelayMs?: number, retryCount?: number);
+    /**
+     * Calculates the delay for a given retry attempt using exponential backoff.
+     * @param attempt - The current retry attempt number (0-based)
+     * @returns The delay in milliseconds for the current attempt
+     */
+    getDelay(attempt: number): number;
+}

package/dist/cjs/execution/retry-strategies/exponential-retry-strategy.js

@@ -0,0 +1,25 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ExponentialRetryStrategy = void 0;
+/**
+ * ExponentialRetryStrategy implements a retry strategy that uses an exponential backoff algorithm.
+ */
+class ExponentialRetryStrategy {
+    /**
+     * @param baseDelayMs - The initial delay before the first retry (default is 1000ms)
+     * @param retryCount - The maximum number of retries (default is 3)
+     */
+    constructor(baseDelayMs = 1000, retryCount = 3) {
+        this.baseDelayMs = baseDelayMs;
+        this.retryCount = retryCount;
+    }
+    /**
+     * Calculates the delay for a given retry attempt using exponential backoff.
+     * @param attempt - The current retry attempt number (0-based)
+     * @returns The delay in milliseconds for the current attempt
+     */
+    getDelay(attempt) {
+        return this.baseDelayMs * (Math.pow(2, attempt));
+    }
+}
+exports.ExponentialRetryStrategy = ExponentialRetryStrategy;

package/dist/cjs/execution/retry-strategies/fixed-retry-strategy.d.ts

@@ -0,0 +1,19 @@
+import { RetryStrategy } from './retry-strategy';
+/**
+ * FixedRetryStrategy implements a retry strategy that uses a constant delay for each retry attempt.
+ */
+export declare class FixedRetryStrategy implements RetryStrategy {
+    baseDelayMs: number;
+    retryCount: number;
+    /**
+     * @param baseDelayMs - The constant delay before each retry (default is 1000ms)
+     * @param retryCount - The maximum number of retries (default is 3)
+     */
+    constructor(baseDelayMs?: number, retryCount?: number);
+    /**
+     * Returns the fixed delay for each retry attempt.
+     * @param _attempt - The current retry attempt number (not used in this strategy)
+     * @returns The fixed delay in milliseconds for the current attempt
+     */
+    getDelay(_attempt: number): number;
+}

package/dist/cjs/execution/retry-strategies/fixed-retry-strategy.js

@@ -0,0 +1,25 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FixedRetryStrategy = void 0;
+/**
+ * FixedRetryStrategy implements a retry strategy that uses a constant delay for each retry attempt.
+ */
+class FixedRetryStrategy {
+    /**
+     * @param baseDelayMs - The constant delay before each retry (default is 1000ms)
+     * @param retryCount - The maximum number of retries (default is 3)
+     */
+    constructor(baseDelayMs = 1000, retryCount = 3) {
+        this.baseDelayMs = baseDelayMs;
+        this.retryCount = retryCount;
+    }
+    /**
+     * Returns the fixed delay for each retry attempt.
+     * @param _attempt - The current retry attempt number (not used in this strategy)
+     * @returns The fixed delay in milliseconds for the current attempt
+     */
+    getDelay(_attempt) {
+        return this.baseDelayMs;
+    }
+}
+exports.FixedRetryStrategy = FixedRetryStrategy;

package/dist/cjs/execution/retry-strategies/index.d.ts

@@ -0,0 +1,4 @@
+export * from './exponential-retry-strategy';
+export * from './linear-retry-strategy';
+export * from './fixed-retry-strategy';
+export * from './retry-strategy';

package/dist/cjs/execution/retry-strategies/index.js

@@ -0,0 +1,20 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./exponential-retry-strategy"), exports);
+__exportStar(require("./linear-retry-strategy"), exports);
+__exportStar(require("./fixed-retry-strategy"), exports);
+__exportStar(require("./retry-strategy"), exports);

package/dist/cjs/execution/retry-strategies/linear-retry-strategy.d.ts

@@ -0,0 +1,19 @@
+import { RetryStrategy } from './retry-strategy';
+/**
+ * LinearRetryStrategy implements a retry strategy that uses a linear backoff algorithm.
+ */
+export declare class LinearRetryStrategy implements RetryStrategy {
+    baseDelayMs: number;
+    retryCount: number;
+    /**
+     * @param baseDelayMs - The initial delay before the first retry (default is 1000ms)
+     * @param retryCount - The maximum number of retries (default is 3)
+     */
+    constructor(baseDelayMs?: number, retryCount?: number);
+    /**
+     * Calculates the delay for a given retry attempt using linear backoff.
+     * @param attempt - The current retry attempt number (1-based)
+     * @returns The delay in milliseconds for the current attempt
+     */
+    getDelay(attempt: number): number;
+}