askui 0.20.7 → 0.20.8

package/dist/cjs/core/inference-response/inference-response.d.ts

@@ -5,6 +5,11 @@ export interface InferenceResponseBody {
     type: ModelType;
     data: ModelType extends 'COMMANDS' ? ControlCommand : Annotation;
 }
+export interface VQAInferenceResponseBody {
+    data: {
+        response: string;
+    };
+}
 export declare class InferenceResponse {
     static fromJson(json: InferenceResponseBody, resizeRatio?: number, image?: string): ControlCommand | Annotation;
     static createModels(type: ModelType, data: ModelType extends 'COMMANDS' ? ControlCommand : Annotation, resizeRatio: number, image?: string): ControlCommand | Annotation;

package/dist/cjs/execution/execution-runtime.d.ts

@@ -35,4 +35,5 @@ export declare class ExecutionRuntime {
     takeScreenshotIfImageisNotProvided(imagePath?: string): Promise<string>;
     getDetectedElements(instruction: string, customElementJson?: CustomElementJson[]): Promise<DetectedElement[]>;
     annotateImage(imagePath?: string, customElementJson?: CustomElementJson[], elements?: DetectedElement[]): Promise<Annotation>;
+    predictVQA(prompt: string, config?: object): Promise<any>;
 }

package/dist/cjs/execution/execution-runtime.js

@@ -208,5 +208,11 @@ class ExecutionRuntime {
             return this.inferenceClient.predictImageAnnotation(base64Image, customElements);
         });
     }
+    predictVQA(prompt, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const base64Image = yield this.takeScreenshotIfImageisNotProvided();
+            return this.inferenceClient.predictVQAAnswer(prompt, base64Image, config);
+        });
+    }
 }
 exports.ExecutionRuntime = ExecutionRuntime;

package/dist/cjs/execution/inference-client.d.ts

@@ -3,6 +3,7 @@ import { ControlCommand } from '../core/ui-control-commands';
 import { CustomElement } from '../core/model/custom-element';
 import { Annotation } from '../core/annotation/annotation';
 import { DetectedElement } from '../core/model/annotation-result/detected-element';
+import { VQAInferenceResponseBody } from '../core/inference-response/inference-response';
 import { ModelCompositionBranch } from './model-composition-branch';
 export declare class InferenceClient {
     private readonly baseUrl;
@@ -16,8 +17,10 @@ export declare class InferenceClient {
     isImageRequired(instruction: string): Promise<boolean>;
     private resizeIfNeeded;
     inference(customElements?: CustomElement[], image?: string, instruction?: string): Promise<ControlCommand | Annotation>;
+    vqaInference(image: string, prompt: string, config?: object): Promise<VQAInferenceResponseBody>;
     private static logMetaInformation;
     predictControlCommand(instruction: string, customElements?: CustomElement[], image?: string): Promise<ControlCommand>;
     getDetectedElements(instruction: string, image: string, customElements?: CustomElement[]): Promise<DetectedElement[]>;
     predictImageAnnotation(image: string, customElements?: CustomElement[]): Promise<Annotation>;
+    predictVQAAnswer(prompt: string, image: string, config?: object): Promise<any>;
 }

package/dist/cjs/execution/inference-client.js

@@ -35,6 +35,7 @@ class InferenceClient {
         this.urls = {
             inference: (0, url_join_1.default)(url, 'inference'),
             isImageRequired: (0, url_join_1.default)(url, 'instruction', 'is-image-required'),
+            vqaInference: (0, url_join_1.default)(url, 'vqa', 'inference'),
         };
         this.httpClient.urlsToRetry = Object.values(this.urls);
         if (this.resize !== undefined && this.resize <= 0) {
@@ -77,6 +78,17 @@ class InferenceClient {
             return ui_control_commands_1.InferenceResponse.fromJson(response.body, resizedImage.resizeRatio, image);
         });
     }
+    vqaInference(image, prompt, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const response = yield this.httpClient.post(this.urls.vqaInference, {
+                config,
+                image,
+                prompt,
+            });
+            InferenceClient.logMetaInformation(response);
+            return response.body;
+        });
+    }
     static logMetaInformation(response) {
         if (response.headers['askui-usage-warnings'] !== undefined) {
             logger_1.logger.warn(response.headers['askui-usage-warnings']);
@@ -109,5 +121,18 @@ class InferenceClient {
             return inferenceResponse;
         });
     }
+    predictVQAAnswer(prompt, image, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const inferenceResponse = yield this.vqaInference(image, prompt, config);
+            const { response } = inferenceResponse.data;
+            try {
+                return JSON.parse(response);
+            }
+            catch (error) {
+                logger_1.logger.warn(`Response is no valid JSON: ${response}`);
+            }
+            return response;
+        });
+    }
 }
 exports.InferenceClient = InferenceClient;

package/dist/cjs/execution/ui-control-client.d.ts

@@ -59,6 +59,95 @@ export declare class UiControlClient extends ApiCommands {
     private getAIElementsByNames;
     fluentCommandExecutor(instructionString: string, context?: CommandExecutorContext): Promise<void>;
     getterExecutor(instruction: string, context?: CommandExecutorContext): Promise<DetectedElement[]>;
+    /**
+     * Takes a prompt that contains a question you want to be answered
+     * or the data you want to have extracted from your screen.
+     *
+     * The optional 'config' can be used to specifiy the JSON schema the
+     * returned object shall have (https://json-schema.org).
+     *
+     * See the following examples on how to use it:
+     *
+     * let isWidgetsNew =
+     *   await aui.ask(
+     *     "Does the sidebar element 'Widgets' have a 'NEW' tag?",
+     *     {
+     *       json_schema: {
+     *         "type": "boolean"
+     *       }
+     *     });
+     *
+     * Output of console.log(isWidgetsNew): true
+     *
+     * let newClients =
+     *   await aui.ask(
+     *     "How many new clients?",
+     *     {
+     *       json_schema: {
+     *         "type": "number"
+     *       }
+     *     });
+     *
+     * Output of console.log(newClients): 9123
+     *
+     * let userNames =
+     *   await aui.ask(
+     *     "Return a list with the users names.",
+     *     {
+     *       json_schema: {
+     *         "type": "array",
+     *         "items": {
+     *           "type": "string"
+     *         }
+     *       }
+     *     });
+     *
+     * Output of console.log(userNames):
+     * [
+     *   'Yiorgos Avraamu',
+     *   'Avram Tsarios',
+     *   'Quintin Ed',
+     *   'Enéas Kwadwo',
+     *   'Agapetus Tadeáš'
+     * ]
+     *
+     * let users =
+     *   await aui.ask(
+     *     "Extract the users from the table.",
+     *     {
+     *       json_schema: {
+     *         "type": "array",
+     *         "items": {
+     *           "type": "object",
+     *           "properties": {
+     *             "name": {
+     *               "type": "string"
+     *             },
+     *             "usage": {
+     *               "type": "number"
+     *             }
+     *           },
+     *           "additionalProperties": false,
+     *           "required": ["name", "usage"]
+     *         },
+     *       },
+     *     });
+     *
+     * Output of console.log(users):
+     * [
+     *   { name: 'Yiorgos Avraamu', usage: 50 },
+     *   { name: 'Avram Tarasios', usage: 10 },
+     *   { name: 'Quintin Ed', usage: 74 },
+     *   { name: 'Eneás Kwadwo', usage: 98 },
+     *   { name: 'Agapetus Tadeáš', usage: 22 }
+     * ]
+     *
+     * @param {string} prompt - The question you want to be answered or
+     *                          the data you want to have extracted.
+     * @param {Object} config - object that specifies the return json: {json_schema: {...}}.
+     * @returns {any} - The answer as JSON specified in the config object.
+     */
+    ask(prompt: string, config?: object): Promise<any>;
     private secretText;
     private getAndResetSecretText;
     /**

package/dist/cjs/execution/ui-control-client.js

@@ -166,6 +166,100 @@ class UiControlClient extends dsl_1.ApiCommands {
             ]);
         });
     }
+    /**
+     * Takes a prompt that contains a question you want to be answered
+     * or the data you want to have extracted from your screen.
+     *
+     * The optional 'config' can be used to specifiy the JSON schema the
+     * returned object shall have (https://json-schema.org).
+     *
+     * See the following examples on how to use it:
+     *
+     * let isWidgetsNew =
+     *   await aui.ask(
+     *     "Does the sidebar element 'Widgets' have a 'NEW' tag?",
+     *     {
+     *       json_schema: {
+     *         "type": "boolean"
+     *       }
+     *     });
+     *
+     * Output of console.log(isWidgetsNew): true
+     *
+     * let newClients =
+     *   await aui.ask(
+     *     "How many new clients?",
+     *     {
+     *       json_schema: {
+     *         "type": "number"
+     *       }
+     *     });
+     *
+     * Output of console.log(newClients): 9123
+     *
+     * let userNames =
+     *   await aui.ask(
+     *     "Return a list with the users names.",
+     *     {
+     *       json_schema: {
+     *         "type": "array",
+     *         "items": {
+     *           "type": "string"
+     *         }
+     *       }
+     *     });
+     *
+     * Output of console.log(userNames):
+     * [
+     *   'Yiorgos Avraamu',
+     *   'Avram Tsarios',
+     *   'Quintin Ed',
+     *   'Enéas Kwadwo',
+     *   'Agapetus Tadeáš'
+     * ]
+     *
+     * let users =
+     *   await aui.ask(
+     *     "Extract the users from the table.",
+     *     {
+     *       json_schema: {
+     *         "type": "array",
+     *         "items": {
+     *           "type": "object",
+     *           "properties": {
+     *             "name": {
+     *               "type": "string"
+     *             },
+     *             "usage": {
+     *               "type": "number"
+     *             }
+     *           },
+     *           "additionalProperties": false,
+     *           "required": ["name", "usage"]
+     *         },
+     *       },
+     *     });
+     *
+     * Output of console.log(users):
+     * [
+     *   { name: 'Yiorgos Avraamu', usage: 50 },
+     *   { name: 'Avram Tarasios', usage: 10 },
+     *   { name: 'Quintin Ed', usage: 74 },
+     *   { name: 'Eneás Kwadwo', usage: 98 },
+     *   { name: 'Agapetus Tadeáš', usage: 22 }
+     * ]
+     *
+     * @param {string} prompt - The question you want to be answered or
+     *                          the data you want to have extracted.
+     * @param {Object} config - object that specifies the return json: {json_schema: {...}}.
+     * @returns {any} - The answer as JSON specified in the config object.
+     */
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    ask(prompt, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.executionRuntime.predictVQA(prompt, config);
+        });
+    }
     getAndResetSecretText() {
         const { secretText } = this;
         this.secretText = undefined;

package/dist/esm/core/inference-response/inference-response.d.ts

@@ -5,6 +5,11 @@ export interface InferenceResponseBody {
     type: ModelType;
     data: ModelType extends 'COMMANDS' ? ControlCommand : Annotation;
 }
+export interface VQAInferenceResponseBody {
+    data: {
+        response: string;
+    };
+}
 export declare class InferenceResponse {
     static fromJson(json: InferenceResponseBody, resizeRatio?: number, image?: string): ControlCommand | Annotation;
     static createModels(type: ModelType, data: ModelType extends 'COMMANDS' ? ControlCommand : Annotation, resizeRatio: number, image?: string): ControlCommand | Annotation;

package/dist/esm/execution/execution-runtime.d.ts

@@ -35,4 +35,5 @@ export declare class ExecutionRuntime {
     takeScreenshotIfImageisNotProvided(imagePath?: string): Promise<string>;
     getDetectedElements(instruction: string, customElementJson?: CustomElementJson[]): Promise<DetectedElement[]>;
     annotateImage(imagePath?: string, customElementJson?: CustomElementJson[], elements?: DetectedElement[]): Promise<Annotation>;
+    predictVQA(prompt: string, config?: object): Promise<any>;
 }

package/dist/esm/execution/execution-runtime.js

@@ -205,4 +205,10 @@ export class ExecutionRuntime {
             return this.inferenceClient.predictImageAnnotation(base64Image, customElements);
         });
     }
+    predictVQA(prompt, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const base64Image = yield this.takeScreenshotIfImageisNotProvided();
+            return this.inferenceClient.predictVQAAnswer(prompt, base64Image, config);
+        });
+    }
 }

package/dist/esm/execution/inference-client.d.ts

@@ -3,6 +3,7 @@ import { ControlCommand } from '../core/ui-control-commands';
 import { CustomElement } from '../core/model/custom-element';
 import { Annotation } from '../core/annotation/annotation';
 import { DetectedElement } from '../core/model/annotation-result/detected-element';
+import { VQAInferenceResponseBody } from '../core/inference-response/inference-response';
 import { ModelCompositionBranch } from './model-composition-branch';
 export declare class InferenceClient {
     private readonly baseUrl;
@@ -16,8 +17,10 @@ export declare class InferenceClient {
     isImageRequired(instruction: string): Promise<boolean>;
     private resizeIfNeeded;
     inference(customElements?: CustomElement[], image?: string, instruction?: string): Promise<ControlCommand | Annotation>;
+    vqaInference(image: string, prompt: string, config?: object): Promise<VQAInferenceResponseBody>;
     private static logMetaInformation;
     predictControlCommand(instruction: string, customElements?: CustomElement[], image?: string): Promise<ControlCommand>;
     getDetectedElements(instruction: string, image: string, customElements?: CustomElement[]): Promise<DetectedElement[]>;
     predictImageAnnotation(image: string, customElements?: CustomElement[]): Promise<Annotation>;
+    predictVQAAnswer(prompt: string, image: string, config?: object): Promise<any>;
 }

package/dist/esm/execution/inference-client.js

@@ -29,6 +29,7 @@ export class InferenceClient {
         this.urls = {
             inference: urljoin(url, 'inference'),
             isImageRequired: urljoin(url, 'instruction', 'is-image-required'),
+            vqaInference: urljoin(url, 'vqa', 'inference'),
         };
         this.httpClient.urlsToRetry = Object.values(this.urls);
         if (this.resize !== undefined && this.resize <= 0) {
@@ -71,6 +72,17 @@ export class InferenceClient {
             return InferenceResponse.fromJson(response.body, resizedImage.resizeRatio, image);
         });
     }
+    vqaInference(image, prompt, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const response = yield this.httpClient.post(this.urls.vqaInference, {
+                config,
+                image,
+                prompt,
+            });
+            InferenceClient.logMetaInformation(response);
+            return response.body;
+        });
+    }
     static logMetaInformation(response) {
         if (response.headers['askui-usage-warnings'] !== undefined) {
             logger.warn(response.headers['askui-usage-warnings']);
@@ -103,4 +115,17 @@ export class InferenceClient {
             return inferenceResponse;
         });
     }
+    predictVQAAnswer(prompt, image, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const inferenceResponse = yield this.vqaInference(image, prompt, config);
+            const { response } = inferenceResponse.data;
+            try {
+                return JSON.parse(response);
+            }
+            catch (error) {
+                logger.warn(`Response is no valid JSON: ${response}`);
+            }
+            return response;
+        });
+    }
 }

package/dist/esm/execution/ui-control-client.d.ts

@@ -59,6 +59,95 @@ export declare class UiControlClient extends ApiCommands {
     private getAIElementsByNames;
     fluentCommandExecutor(instructionString: string, context?: CommandExecutorContext): Promise<void>;
     getterExecutor(instruction: string, context?: CommandExecutorContext): Promise<DetectedElement[]>;
+    /**
+     * Takes a prompt that contains a question you want to be answered
+     * or the data you want to have extracted from your screen.
+     *
+     * The optional 'config' can be used to specifiy the JSON schema the
+     * returned object shall have (https://json-schema.org).
+     *
+     * See the following examples on how to use it:
+     *
+     * let isWidgetsNew =
+     *   await aui.ask(
+     *     "Does the sidebar element 'Widgets' have a 'NEW' tag?",
+     *     {
+     *       json_schema: {
+     *         "type": "boolean"
+     *       }
+     *     });
+     *
+     * Output of console.log(isWidgetsNew): true
+     *
+     * let newClients =
+     *   await aui.ask(
+     *     "How many new clients?",
+     *     {
+     *       json_schema: {
+     *         "type": "number"
+     *       }
+     *     });
+     *
+     * Output of console.log(newClients): 9123
+     *
+     * let userNames =
+     *   await aui.ask(
+     *     "Return a list with the users names.",
+     *     {
+     *       json_schema: {
+     *         "type": "array",
+     *         "items": {
+     *           "type": "string"
+     *         }
+     *       }
+     *     });
+     *
+     * Output of console.log(userNames):
+     * [
+     *   'Yiorgos Avraamu',
+     *   'Avram Tsarios',
+     *   'Quintin Ed',
+     *   'Enéas Kwadwo',
+     *   'Agapetus Tadeáš'
+     * ]
+     *
+     * let users =
+     *   await aui.ask(
+     *     "Extract the users from the table.",
+     *     {
+     *       json_schema: {
+     *         "type": "array",
+     *         "items": {
+     *           "type": "object",
+     *           "properties": {
+     *             "name": {
+     *               "type": "string"
+     *             },
+     *             "usage": {
+     *               "type": "number"
+     *             }
+     *           },
+     *           "additionalProperties": false,
+     *           "required": ["name", "usage"]
+     *         },
+     *       },
+     *     });
+     *
+     * Output of console.log(users):
+     * [
+     *   { name: 'Yiorgos Avraamu', usage: 50 },
+     *   { name: 'Avram Tarasios', usage: 10 },
+     *   { name: 'Quintin Ed', usage: 74 },
+     *   { name: 'Eneás Kwadwo', usage: 98 },
+     *   { name: 'Agapetus Tadeáš', usage: 22 }
+     * ]
+     *
+     * @param {string} prompt - The question you want to be answered or
+     *                          the data you want to have extracted.
+     * @param {Object} config - object that specifies the return json: {json_schema: {...}}.
+     * @returns {any} - The answer as JSON specified in the config object.
+     */
+    ask(prompt: string, config?: object): Promise<any>;
     private secretText;
     private getAndResetSecretText;
     /**

package/dist/esm/execution/ui-control-client.js

@@ -163,6 +163,100 @@ export class UiControlClient extends ApiCommands {
             ]);
         });
     }
+    /**
+     * Takes a prompt that contains a question you want to be answered
+     * or the data you want to have extracted from your screen.
+     *
+     * The optional 'config' can be used to specifiy the JSON schema the
+     * returned object shall have (https://json-schema.org).
+     *
+     * See the following examples on how to use it:
+     *
+     * let isWidgetsNew =
+     *   await aui.ask(
+     *     "Does the sidebar element 'Widgets' have a 'NEW' tag?",
+     *     {
+     *       json_schema: {
+     *         "type": "boolean"
+     *       }
+     *     });
+     *
+     * Output of console.log(isWidgetsNew): true
+     *
+     * let newClients =
+     *   await aui.ask(
+     *     "How many new clients?",
+     *     {
+     *       json_schema: {
+     *         "type": "number"
+     *       }
+     *     });
+     *
+     * Output of console.log(newClients): 9123
+     *
+     * let userNames =
+     *   await aui.ask(
+     *     "Return a list with the users names.",
+     *     {
+     *       json_schema: {
+     *         "type": "array",
+     *         "items": {
+     *           "type": "string"
+     *         }
+     *       }
+     *     });
+     *
+     * Output of console.log(userNames):
+     * [
+     *   'Yiorgos Avraamu',
+     *   'Avram Tsarios',
+     *   'Quintin Ed',
+     *   'Enéas Kwadwo',
+     *   'Agapetus Tadeáš'
+     * ]
+     *
+     * let users =
+     *   await aui.ask(
+     *     "Extract the users from the table.",
+     *     {
+     *       json_schema: {
+     *         "type": "array",
+     *         "items": {
+     *           "type": "object",
+     *           "properties": {
+     *             "name": {
+     *               "type": "string"
+     *             },
+     *             "usage": {
+     *               "type": "number"
+     *             }
+     *           },
+     *           "additionalProperties": false,
+     *           "required": ["name", "usage"]
+     *         },
+     *       },
+     *     });
+     *
+     * Output of console.log(users):
+     * [
+     *   { name: 'Yiorgos Avraamu', usage: 50 },
+     *   { name: 'Avram Tarasios', usage: 10 },
+     *   { name: 'Quintin Ed', usage: 74 },
+     *   { name: 'Eneás Kwadwo', usage: 98 },
+     *   { name: 'Agapetus Tadeáš', usage: 22 }
+     * ]
+     *
+     * @param {string} prompt - The question you want to be answered or
+     *                          the data you want to have extracted.
+     * @param {Object} config - object that specifies the return json: {json_schema: {...}}.
+     * @returns {any} - The answer as JSON specified in the config object.
+     */
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    ask(prompt, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.executionRuntime.predictVQA(prompt, config);
+        });
+    }
     getAndResetSecretText() {
         const { secretText } = this;
         this.secretText = undefined;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "askui",
-    "version": "0.20.7",
+    "version": "0.20.8",
     "license": "MIT",
     "author": "askui GmbH <info@askui.com> (http://www.askui.com/)",
     "description": "Reliable, automated end-to-end-testing that depends on what is shown on your screen instead of the technology you are running on",