askui 0.20.6 → 0.20.8

package/dist/cjs/core/inference-response/inference-response.d.ts

@@ -5,6 +5,11 @@ export interface InferenceResponseBody {
     type: ModelType;
     data: ModelType extends 'COMMANDS' ? ControlCommand : Annotation;
 }
+export interface VQAInferenceResponseBody {
+    data: {
+        response: string;
+    };
+}
 export declare class InferenceResponse {
     static fromJson(json: InferenceResponseBody, resizeRatio?: number, image?: string): ControlCommand | Annotation;
     static createModels(type: ModelType, data: ModelType extends 'COMMANDS' ? ControlCommand : Annotation, resizeRatio: number, image?: string): ControlCommand | Annotation;

package/dist/cjs/execution/execution-runtime.d.ts

@@ -35,4 +35,5 @@ export declare class ExecutionRuntime {
     takeScreenshotIfImageisNotProvided(imagePath?: string): Promise<string>;
     getDetectedElements(instruction: string, customElementJson?: CustomElementJson[]): Promise<DetectedElement[]>;
     annotateImage(imagePath?: string, customElementJson?: CustomElementJson[], elements?: DetectedElement[]): Promise<Annotation>;
+    predictVQA(prompt: string, config?: object): Promise<any>;
 }

package/dist/cjs/execution/execution-runtime.js

@@ -208,5 +208,11 @@ class ExecutionRuntime {
             return this.inferenceClient.predictImageAnnotation(base64Image, customElements);
         });
     }
+    predictVQA(prompt, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const base64Image = yield this.takeScreenshotIfImageisNotProvided();
+            return this.inferenceClient.predictVQAAnswer(prompt, base64Image, config);
+        });
+    }
 }
 exports.ExecutionRuntime = ExecutionRuntime;

package/dist/cjs/execution/inference-client.d.ts

@@ -3,6 +3,7 @@ import { ControlCommand } from '../core/ui-control-commands';
 import { CustomElement } from '../core/model/custom-element';
 import { Annotation } from '../core/annotation/annotation';
 import { DetectedElement } from '../core/model/annotation-result/detected-element';
+import { VQAInferenceResponseBody } from '../core/inference-response/inference-response';
 import { ModelCompositionBranch } from './model-composition-branch';
 export declare class InferenceClient {
     private readonly baseUrl;
@@ -16,8 +17,10 @@ export declare class InferenceClient {
     isImageRequired(instruction: string): Promise<boolean>;
     private resizeIfNeeded;
     inference(customElements?: CustomElement[], image?: string, instruction?: string): Promise<ControlCommand | Annotation>;
+    vqaInference(image: string, prompt: string, config?: object): Promise<VQAInferenceResponseBody>;
     private static logMetaInformation;
     predictControlCommand(instruction: string, customElements?: CustomElement[], image?: string): Promise<ControlCommand>;
     getDetectedElements(instruction: string, image: string, customElements?: CustomElement[]): Promise<DetectedElement[]>;
     predictImageAnnotation(image: string, customElements?: CustomElement[]): Promise<Annotation>;
+    predictVQAAnswer(prompt: string, image: string, config?: object): Promise<any>;
 }

package/dist/cjs/execution/inference-client.js

@@ -35,6 +35,7 @@ class InferenceClient {
         this.urls = {
             inference: (0, url_join_1.default)(url, 'inference'),
             isImageRequired: (0, url_join_1.default)(url, 'instruction', 'is-image-required'),
+            vqaInference: (0, url_join_1.default)(url, 'vqa', 'inference'),
         };
         this.httpClient.urlsToRetry = Object.values(this.urls);
         if (this.resize !== undefined && this.resize <= 0) {
@@ -77,6 +78,17 @@ class InferenceClient {
             return ui_control_commands_1.InferenceResponse.fromJson(response.body, resizedImage.resizeRatio, image);
         });
     }
+    vqaInference(image, prompt, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const response = yield this.httpClient.post(this.urls.vqaInference, {
+                config,
+                image,
+                prompt,
+            });
+            InferenceClient.logMetaInformation(response);
+            return response.body;
+        });
+    }
     static logMetaInformation(response) {
         if (response.headers['askui-usage-warnings'] !== undefined) {
             logger_1.logger.warn(response.headers['askui-usage-warnings']);
@@ -109,5 +121,18 @@ class InferenceClient {
             return inferenceResponse;
         });
     }
+    predictVQAAnswer(prompt, image, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const inferenceResponse = yield this.vqaInference(image, prompt, config);
+            const { response } = inferenceResponse.data;
+            try {
+                return JSON.parse(response);
+            }
+            catch (error) {
+                logger_1.logger.warn(`Response is no valid JSON: ${response}`);
+            }
+            return response;
+        });
+    }
 }
 exports.InferenceClient = InferenceClient;

package/dist/cjs/execution/ui-control-client.d.ts

@@ -59,6 +59,95 @@ export declare class UiControlClient extends ApiCommands {
     private getAIElementsByNames;
     fluentCommandExecutor(instructionString: string, context?: CommandExecutorContext): Promise<void>;
     getterExecutor(instruction: string, context?: CommandExecutorContext): Promise<DetectedElement[]>;
+    /**
+     * Takes a prompt that contains a question you want to be answered
+     * or the data you want to have extracted from your screen.
+     *
+     * The optional 'config' can be used to specifiy the JSON schema the
+     * returned object shall have (https://json-schema.org).
+     *
+     * See the following examples on how to use it:
+     *
+     * let isWidgetsNew =
+     *   await aui.ask(
+     *     "Does the sidebar element 'Widgets' have a 'NEW' tag?",
+     *     {
+     *       json_schema: {
+     *         "type": "boolean"
+     *       }
+     *     });
+     *
+     * Output of console.log(isWidgetsNew): true
+     *
+     * let newClients =
+     *   await aui.ask(
+     *     "How many new clients?",
+     *     {
+     *       json_schema: {
+     *         "type": "number"
+     *       }
+     *     });
+     *
+     * Output of console.log(newClients): 9123
+     *
+     * let userNames =
+     *   await aui.ask(
+     *     "Return a list with the users names.",
+     *     {
+     *       json_schema: {
+     *         "type": "array",
+     *         "items": {
+     *           "type": "string"
+     *         }
+     *       }
+     *     });
+     *
+     * Output of console.log(userNames):
+     * [
+     *   'Yiorgos Avraamu',
+     *   'Avram Tsarios',
+     *   'Quintin Ed',
+     *   'Enéas Kwadwo',
+     *   'Agapetus Tadeáš'
+     * ]
+     *
+     * let users =
+     *   await aui.ask(
+     *     "Extract the users from the table.",
+     *     {
+     *       json_schema: {
+     *         "type": "array",
+     *         "items": {
+     *           "type": "object",
+     *           "properties": {
+     *             "name": {
+     *               "type": "string"
+     *             },
+     *             "usage": {
+     *               "type": "number"
+     *             }
+     *           },
+     *           "additionalProperties": false,
+     *           "required": ["name", "usage"]
+     *         },
+     *       },
+     *     });
+     *
+     * Output of console.log(users):
+     * [
+     *   { name: 'Yiorgos Avraamu', usage: 50 },
+     *   { name: 'Avram Tarasios', usage: 10 },
+     *   { name: 'Quintin Ed', usage: 74 },
+     *   { name: 'Eneás Kwadwo', usage: 98 },
+     *   { name: 'Agapetus Tadeáš', usage: 22 }
+     * ]
+     *
+     * @param {string} prompt - The question you want to be answered or
+     *                          the data you want to have extracted.
+     * @param {Object} config - object that specifies the return json: {json_schema: {...}}.
+     * @returns {any} - The answer as JSON specified in the config object.
+     */
+    ask(prompt: string, config?: object): Promise<any>;
     private secretText;
     private getAndResetSecretText;
     /**

package/dist/cjs/execution/ui-control-client.js

@@ -166,6 +166,100 @@ class UiControlClient extends dsl_1.ApiCommands {
             ]);
         });
     }
+    /**
+     * Takes a prompt that contains a question you want to be answered
+     * or the data you want to have extracted from your screen.
+     *
+     * The optional 'config' can be used to specifiy the JSON schema the
+     * returned object shall have (https://json-schema.org).
+     *
+     * See the following examples on how to use it:
+     *
+     * let isWidgetsNew =
+     *   await aui.ask(
+     *     "Does the sidebar element 'Widgets' have a 'NEW' tag?",
+     *     {
+     *       json_schema: {
+     *         "type": "boolean"
+     *       }
+     *     });
+     *
+     * Output of console.log(isWidgetsNew): true
+     *
+     * let newClients =
+     *   await aui.ask(
+     *     "How many new clients?",
+     *     {
+     *       json_schema: {
+     *         "type": "number"
+     *       }
+     *     });
+     *
+     * Output of console.log(newClients): 9123
+     *
+     * let userNames =
+     *   await aui.ask(
+     *     "Return a list with the users names.",
+     *     {
+     *       json_schema: {
+     *         "type": "array",
+     *         "items": {
+     *           "type": "string"
+     *         }
+     *       }
+     *     });
+     *
+     * Output of console.log(userNames):
+     * [
+     *   'Yiorgos Avraamu',
+     *   'Avram Tsarios',
+     *   'Quintin Ed',
+     *   'Enéas Kwadwo',
+     *   'Agapetus Tadeáš'
+     * ]
+     *
+     * let users =
+     *   await aui.ask(
+     *     "Extract the users from the table.",
+     *     {
+     *       json_schema: {
+     *         "type": "array",
+     *         "items": {
+     *           "type": "object",
+     *           "properties": {
+     *             "name": {
+     *               "type": "string"
+     *             },
+     *             "usage": {
+     *               "type": "number"
+     *             }
+     *           },
+     *           "additionalProperties": false,
+     *           "required": ["name", "usage"]
+     *         },
+     *       },
+     *     });
+     *
+     * Output of console.log(users):
+     * [
+     *   { name: 'Yiorgos Avraamu', usage: 50 },
+     *   { name: 'Avram Tarasios', usage: 10 },
+     *   { name: 'Quintin Ed', usage: 74 },
+     *   { name: 'Eneás Kwadwo', usage: 98 },
+     *   { name: 'Agapetus Tadeáš', usage: 22 }
+     * ]
+     *
+     * @param {string} prompt - The question you want to be answered or
+     *                          the data you want to have extracted.
+     * @param {Object} config - object that specifies the return json: {json_schema: {...}}.
+     * @returns {any} - The answer as JSON specified in the config object.
+     */
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    ask(prompt, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.executionRuntime.predictVQA(prompt, config);
+        });
+    }
     getAndResetSecretText() {
         const { secretText } = this;
         this.secretText = undefined;

package/dist/cjs/lib/interactive_cli/cli.js

@@ -17,7 +17,6 @@ const inquirer_1 = __importDefault(require("inquirer"));
 const commander_1 = require("commander");
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const create_example_project_1 = require("./create-example-project");
-const nonEmpty = (subject) => (input) => input.trim().length > 0 || `${subject} is required.`;
 /* eslint-disable sort-keys */
 const questions = [
     {
@@ -26,21 +25,6 @@ const questions = [
         message: 'Which framework do you prefer?:',
         choices: ['jest'],
     },
-    {
-        type: 'input',
-        name: 'workspaceId',
-        message: 'Your workspace id:',
-        validate: nonEmpty('workspace id'),
-        when: (answers) => !answers.skipCredentials,
-    },
-    {
-        type: 'password',
-        name: 'accessToken',
-        message: 'Your access token:',
-        mask: '*',
-        validate: nonEmpty('access token'),
-        when: (answers) => !answers.skipCredentials,
-    },
     {
         type: 'confirm',
         name: 'typescriptConfig',
@@ -57,22 +41,6 @@ const options = [
         description: 'the test framework to use.',
         option: '-f, --test-framework <value>',
     },
-    {
-        choices: [],
-        default: false,
-        description: 'skip the credentials setup.',
-        option: '-sc, --skip-credentials',
-    },
-    {
-        choices: [],
-        description: 'a workspace id.',
-        option: '-w, --workspace-id <value>',
-    },
-    {
-        choices: [],
-        description: 'an access token for the workspace with the id.',
-        option: '-a, --access-token <value>',
-    },
     {
         choices: [],
         default: 'askui_example',

package/dist/cjs/lib/interactive_cli/create-example-project.d.ts

@@ -18,7 +18,6 @@ export declare class CreateExampleProject {
     private createAskUIHelperFromTemplate;
     private setupTestFrameWork;
     private static installTestFrameworkPackages;
-    private addUserCredentials;
     private copyESLintConfigFiles;
     private copyGitignore;
     private copyTsConfigFile;

package/dist/cjs/lib/interactive_cli/create-example-project.js

@@ -30,7 +30,7 @@ class CreateExampleProject {
         this.projectRootDirectoryPath = process.cwd();
         this.automationsDirectoryName = this.getAutomationsDirectoryName();
         this.automationsDirectoryPath = path_1.default.join(this.projectRootDirectoryPath, this.automationsDirectoryName);
-        this.askUIControllerUrl = 'https://docs.askui.com/docs/general/Components/AskUI-Controller';
+        this.askUIControllerUrl = 'https://docs.askui.com/docs/suite/Components/AskUI-Development-Environment#askui-startcontroller-command';
         this.helperTemplateConfig = {};
     }
     getAutomationsDirectoryName() {
@@ -131,10 +131,7 @@ class CreateExampleProject {
                                 title: 'Create askui-helper.ts ',
                                 task: () => __awaiter(this, void 0, void 0, function* () {
                                     const askuiHelperTemplateFilePath = path_1.default.join((0, path_2.getPathToNodeModulesRoot)(), 'example_projects_templates', 'templates');
-                                    let templateFileName = 'askui-helper.nj';
-                                    if (this.cliOptions.operatingSystem === 'windows') {
-                                        templateFileName = 'askui-helper-windows.nj';
-                                    }
+                                    const templateFileName = 'askui-helper.nj';
                                     nunjucks_1.default.configure(askuiHelperTemplateFilePath, { autoescape: false });
                                     const result = nunjucks_1.default.render(templateFileName, this.helperTemplateConfig);
                                     const filePath = path_1.default.join(this.automationsDirectoryPath, 'helpers', 'askui-helper.ts');
@@ -193,24 +190,6 @@ class CreateExampleProject {
             yield runCommand(frameworkDependencies.jest);
         });
     }
-    addUserCredentials() {
-        return __awaiter(this, void 0, void 0, function* () {
-            return [
-                {
-                    /* eslint-disable sort-keys */
-                    title: 'Add user credentials',
-                    enabled: () => !this.cliOptions.skipCredentials,
-                    task: () => __awaiter(this, void 0, void 0, function* () {
-                        this.helperTemplateConfig['credentials'] = ` credentials: {
-        workspaceId: '${this.cliOptions.workspaceId}',
-        token: '${this.cliOptions.accessToken}',
-      },`;
-                    }),
-                },
-                /* eslint-enable */
-            ];
-        });
-    }
     copyESLintConfigFiles() {
         return __awaiter(this, void 0, void 0, function* () {
             const esLintRcFilePath = path_1.default.join('example_projects_templates', 'typescript', '.eslintrc.json-template');
@@ -282,20 +261,14 @@ class CreateExampleProject {
                 ...(yield this.copyESLintConfigFiles()),
                 ...(yield this.copyGitignore()),
                 ...(yield this.copyTsConfigFile()),
-                ...(yield this.addUserCredentials()),
                 ...(yield this.createAskUIHelperFromTemplate()),
             ]);
             yield tasks.run();
             /* eslint-disable no-console */
             console.log(chalk_1.default.greenBright('\nCongratulations!'));
             console.log(`askui example was created under ${chalk_1.default.gray(this.automationsDirectoryPath)}`);
-            if (this.cliOptions.operatingSystem === 'windows') {
-                console.log(chalk_1.default.redBright(`\nPlease make sure the AskUI Controller is running: ${this.askUIControllerUrl}\n`));
-                console.log(`You can start your automation with this command ${chalk_1.default.green('AskUI-RunProject')}`);
-            }
-            else {
-                console.log(`You can start your automation with this command ${chalk_1.default.green('npm run askui')}`);
-            }
+            console.log(chalk_1.default.redBright(`\nPlease make sure the AskUI Controller is running: ${this.askUIControllerUrl}\n`));
+            console.log(`You can start your automation with this command ${chalk_1.default.green('AskUI-RunProject')}`);
             /* eslint-enable no-console */
         });
     }

package/dist/cjs/lib/ui-controller.d.ts

@@ -1,4 +1,8 @@
 import { UiControllerArgs } from './ui-controller-args';
+/**
+ * @deprecated Will be removed soon because the AskUI Controller
+ *             gets started as an operating system service.
+ */
 export declare class UiController {
     private args?;
     private server;

package/dist/cjs/lib/ui-controller.js

@@ -14,6 +14,10 @@ const ui_controller_darwin_1 = require("./ui-controller-darwin");
 const ui_controller_linux_1 = require("./ui-controller-linux");
 const ui_controller_win32_1 = require("./ui-controller-win32");
 const download_binaries_1 = require("./download-binaries");
+/**
+ * @deprecated Will be removed soon because the AskUI Controller
+ *             gets started as an operating system service.
+ */
 class UiController {
     constructor(args) {
         this.args = args;

package/dist/esm/core/inference-response/inference-response.d.ts

@@ -5,6 +5,11 @@ export interface InferenceResponseBody {
     type: ModelType;
     data: ModelType extends 'COMMANDS' ? ControlCommand : Annotation;
 }
+export interface VQAInferenceResponseBody {
+    data: {
+        response: string;
+    };
+}
 export declare class InferenceResponse {
     static fromJson(json: InferenceResponseBody, resizeRatio?: number, image?: string): ControlCommand | Annotation;
     static createModels(type: ModelType, data: ModelType extends 'COMMANDS' ? ControlCommand : Annotation, resizeRatio: number, image?: string): ControlCommand | Annotation;

package/dist/esm/execution/execution-runtime.d.ts

@@ -35,4 +35,5 @@ export declare class ExecutionRuntime {
     takeScreenshotIfImageisNotProvided(imagePath?: string): Promise<string>;
     getDetectedElements(instruction: string, customElementJson?: CustomElementJson[]): Promise<DetectedElement[]>;
     annotateImage(imagePath?: string, customElementJson?: CustomElementJson[], elements?: DetectedElement[]): Promise<Annotation>;
+    predictVQA(prompt: string, config?: object): Promise<any>;
 }

package/dist/esm/execution/execution-runtime.js

@@ -205,4 +205,10 @@ export class ExecutionRuntime {
             return this.inferenceClient.predictImageAnnotation(base64Image, customElements);
         });
     }
+    predictVQA(prompt, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const base64Image = yield this.takeScreenshotIfImageisNotProvided();
+            return this.inferenceClient.predictVQAAnswer(prompt, base64Image, config);
+        });
+    }
 }

package/dist/esm/execution/inference-client.d.ts

@@ -3,6 +3,7 @@ import { ControlCommand } from '../core/ui-control-commands';
 import { CustomElement } from '../core/model/custom-element';
 import { Annotation } from '../core/annotation/annotation';
 import { DetectedElement } from '../core/model/annotation-result/detected-element';
+import { VQAInferenceResponseBody } from '../core/inference-response/inference-response';
 import { ModelCompositionBranch } from './model-composition-branch';
 export declare class InferenceClient {
     private readonly baseUrl;
@@ -16,8 +17,10 @@ export declare class InferenceClient {
     isImageRequired(instruction: string): Promise<boolean>;
     private resizeIfNeeded;
     inference(customElements?: CustomElement[], image?: string, instruction?: string): Promise<ControlCommand | Annotation>;
+    vqaInference(image: string, prompt: string, config?: object): Promise<VQAInferenceResponseBody>;
     private static logMetaInformation;
     predictControlCommand(instruction: string, customElements?: CustomElement[], image?: string): Promise<ControlCommand>;
     getDetectedElements(instruction: string, image: string, customElements?: CustomElement[]): Promise<DetectedElement[]>;
     predictImageAnnotation(image: string, customElements?: CustomElement[]): Promise<Annotation>;
+    predictVQAAnswer(prompt: string, image: string, config?: object): Promise<any>;
 }

package/dist/esm/execution/inference-client.js

@@ -29,6 +29,7 @@ export class InferenceClient {
         this.urls = {
             inference: urljoin(url, 'inference'),
             isImageRequired: urljoin(url, 'instruction', 'is-image-required'),
+            vqaInference: urljoin(url, 'vqa', 'inference'),
         };
         this.httpClient.urlsToRetry = Object.values(this.urls);
         if (this.resize !== undefined && this.resize <= 0) {
@@ -71,6 +72,17 @@ export class InferenceClient {
             return InferenceResponse.fromJson(response.body, resizedImage.resizeRatio, image);
         });
     }
+    vqaInference(image, prompt, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const response = yield this.httpClient.post(this.urls.vqaInference, {
+                config,
+                image,
+                prompt,
+            });
+            InferenceClient.logMetaInformation(response);
+            return response.body;
+        });
+    }
     static logMetaInformation(response) {
         if (response.headers['askui-usage-warnings'] !== undefined) {
             logger.warn(response.headers['askui-usage-warnings']);
@@ -103,4 +115,17 @@ export class InferenceClient {
             return inferenceResponse;
         });
     }
+    predictVQAAnswer(prompt, image, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const inferenceResponse = yield this.vqaInference(image, prompt, config);
+            const { response } = inferenceResponse.data;
+            try {
+                return JSON.parse(response);
+            }
+            catch (error) {
+                logger.warn(`Response is no valid JSON: ${response}`);
+            }
+            return response;
+        });
+    }
 }

package/dist/esm/execution/ui-control-client.d.ts

@@ -59,6 +59,95 @@ export declare class UiControlClient extends ApiCommands {
     private getAIElementsByNames;
     fluentCommandExecutor(instructionString: string, context?: CommandExecutorContext): Promise<void>;
     getterExecutor(instruction: string, context?: CommandExecutorContext): Promise<DetectedElement[]>;
+    /**
+     * Takes a prompt that contains a question you want to be answered
+     * or the data you want to have extracted from your screen.
+     *
+     * The optional 'config' can be used to specifiy the JSON schema the
+     * returned object shall have (https://json-schema.org).
+     *
+     * See the following examples on how to use it:
+     *
+     * let isWidgetsNew =
+     *   await aui.ask(
+     *     "Does the sidebar element 'Widgets' have a 'NEW' tag?",
+     *     {
+     *       json_schema: {
+     *         "type": "boolean"
+     *       }
+     *     });
+     *
+     * Output of console.log(isWidgetsNew): true
+     *
+     * let newClients =
+     *   await aui.ask(
+     *     "How many new clients?",
+     *     {
+     *       json_schema: {
+     *         "type": "number"
+     *       }
+     *     });
+     *
+     * Output of console.log(newClients): 9123
+     *
+     * let userNames =
+     *   await aui.ask(
+     *     "Return a list with the users names.",
+     *     {
+     *       json_schema: {
+     *         "type": "array",
+     *         "items": {
+     *           "type": "string"
+     *         }
+     *       }
+     *     });
+     *
+     * Output of console.log(userNames):
+     * [
+     *   'Yiorgos Avraamu',
+     *   'Avram Tsarios',
+     *   'Quintin Ed',
+     *   'Enéas Kwadwo',
+     *   'Agapetus Tadeáš'
+     * ]
+     *
+     * let users =
+     *   await aui.ask(
+     *     "Extract the users from the table.",
+     *     {
+     *       json_schema: {
+     *         "type": "array",
+     *         "items": {
+     *           "type": "object",
+     *           "properties": {
+     *             "name": {
+     *               "type": "string"
+     *             },
+     *             "usage": {
+     *               "type": "number"
+     *             }
+     *           },
+     *           "additionalProperties": false,
+     *           "required": ["name", "usage"]
+     *         },
+     *       },
+     *     });
+     *
+     * Output of console.log(users):
+     * [
+     *   { name: 'Yiorgos Avraamu', usage: 50 },
+     *   { name: 'Avram Tarasios', usage: 10 },
+     *   { name: 'Quintin Ed', usage: 74 },
+     *   { name: 'Eneás Kwadwo', usage: 98 },
+     *   { name: 'Agapetus Tadeáš', usage: 22 }
+     * ]
+     *
+     * @param {string} prompt - The question you want to be answered or
+     *                          the data you want to have extracted.
+     * @param {Object} config - object that specifies the return json: {json_schema: {...}}.
+     * @returns {any} - The answer as JSON specified in the config object.
+     */
+    ask(prompt: string, config?: object): Promise<any>;
     private secretText;
     private getAndResetSecretText;
     /**

package/dist/esm/execution/ui-control-client.js

@@ -163,6 +163,100 @@ export class UiControlClient extends ApiCommands {
             ]);
         });
     }
+    /**
+     * Takes a prompt that contains a question you want to be answered
+     * or the data you want to have extracted from your screen.
+     *
+     * The optional 'config' can be used to specifiy the JSON schema the
+     * returned object shall have (https://json-schema.org).
+     *
+     * See the following examples on how to use it:
+     *
+     * let isWidgetsNew =
+     *   await aui.ask(
+     *     "Does the sidebar element 'Widgets' have a 'NEW' tag?",
+     *     {
+     *       json_schema: {
+     *         "type": "boolean"
+     *       }
+     *     });
+     *
+     * Output of console.log(isWidgetsNew): true
+     *
+     * let newClients =
+     *   await aui.ask(
+     *     "How many new clients?",
+     *     {
+     *       json_schema: {
+     *         "type": "number"
+     *       }
+     *     });
+     *
+     * Output of console.log(newClients): 9123
+     *
+     * let userNames =
+     *   await aui.ask(
+     *     "Return a list with the users names.",
+     *     {
+     *       json_schema: {
+     *         "type": "array",
+     *         "items": {
+     *           "type": "string"
+     *         }
+     *       }
+     *     });
+     *
+     * Output of console.log(userNames):
+     * [
+     *   'Yiorgos Avraamu',
+     *   'Avram Tsarios',
+     *   'Quintin Ed',
+     *   'Enéas Kwadwo',
+     *   'Agapetus Tadeáš'
+     * ]
+     *
+     * let users =
+     *   await aui.ask(
+     *     "Extract the users from the table.",
+     *     {
+     *       json_schema: {
+     *         "type": "array",
+     *         "items": {
+     *           "type": "object",
+     *           "properties": {
+     *             "name": {
+     *               "type": "string"
+     *             },
+     *             "usage": {
+     *               "type": "number"
+     *             }
+     *           },
+     *           "additionalProperties": false,
+     *           "required": ["name", "usage"]
+     *         },
+     *       },
+     *     });
+     *
+     * Output of console.log(users):
+     * [
+     *   { name: 'Yiorgos Avraamu', usage: 50 },
+     *   { name: 'Avram Tarasios', usage: 10 },
+     *   { name: 'Quintin Ed', usage: 74 },
+     *   { name: 'Eneás Kwadwo', usage: 98 },
+     *   { name: 'Agapetus Tadeáš', usage: 22 }
+     * ]
+     *
+     * @param {string} prompt - The question you want to be answered or
+     *                          the data you want to have extracted.
+     * @param {Object} config - object that specifies the return json: {json_schema: {...}}.
+     * @returns {any} - The answer as JSON specified in the config object.
+     */
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    ask(prompt, config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this.executionRuntime.predictVQA(prompt, config);
+        });
+    }
     getAndResetSecretText() {
         const { secretText } = this;
         this.secretText = undefined;

package/dist/esm/lib/interactive_cli/cli.js

@@ -11,7 +11,6 @@ import inquirer from 'inquirer';
 import { Command, Option } from 'commander';
 import fs from 'fs-extra';
 import { CreateExampleProject } from './create-example-project';
-const nonEmpty = (subject) => (input) => input.trim().length > 0 || `${subject} is required.`;
 /* eslint-disable sort-keys */
 const questions = [
     {
@@ -20,21 +19,6 @@ const questions = [
         message: 'Which framework do you prefer?:',
         choices: ['jest'],
     },
-    {
-        type: 'input',
-        name: 'workspaceId',
-        message: 'Your workspace id:',
-        validate: nonEmpty('workspace id'),
-        when: (answers) => !answers.skipCredentials,
-    },
-    {
-        type: 'password',
-        name: 'accessToken',
-        message: 'Your access token:',
-        mask: '*',
-        validate: nonEmpty('access token'),
-        when: (answers) => !answers.skipCredentials,
-    },
     {
         type: 'confirm',
         name: 'typescriptConfig',
@@ -51,22 +35,6 @@ const options = [
         description: 'the test framework to use.',
         option: '-f, --test-framework <value>',
     },
-    {
-        choices: [],
-        default: false,
-        description: 'skip the credentials setup.',
-        option: '-sc, --skip-credentials',
-    },
-    {
-        choices: [],
-        description: 'a workspace id.',
-        option: '-w, --workspace-id <value>',
-    },
-    {
-        choices: [],
-        description: 'an access token for the workspace with the id.',
-        option: '-a, --access-token <value>',
-    },
     {
         choices: [],
         default: 'askui_example',

package/dist/esm/lib/interactive_cli/create-example-project.d.ts

@@ -18,7 +18,6 @@ export declare class CreateExampleProject {
     private createAskUIHelperFromTemplate;
     private setupTestFrameWork;
     private static installTestFrameworkPackages;
-    private addUserCredentials;
     private copyESLintConfigFiles;
     private copyGitignore;
     private copyTsConfigFile;

package/dist/esm/lib/interactive_cli/create-example-project.js

@@ -24,7 +24,7 @@ export class CreateExampleProject {
         this.projectRootDirectoryPath = process.cwd();
         this.automationsDirectoryName = this.getAutomationsDirectoryName();
         this.automationsDirectoryPath = path.join(this.projectRootDirectoryPath, this.automationsDirectoryName);
-        this.askUIControllerUrl = 'https://docs.askui.com/docs/general/Components/AskUI-Controller';
+        this.askUIControllerUrl = 'https://docs.askui.com/docs/suite/Components/AskUI-Development-Environment#askui-startcontroller-command';
         this.helperTemplateConfig = {};
     }
     getAutomationsDirectoryName() {
@@ -125,10 +125,7 @@ export class CreateExampleProject {
                                 title: 'Create askui-helper.ts ',
                                 task: () => __awaiter(this, void 0, void 0, function* () {
                                     const askuiHelperTemplateFilePath = path.join(getPathToNodeModulesRoot(), 'example_projects_templates', 'templates');
-                                    let templateFileName = 'askui-helper.nj';
-                                    if (this.cliOptions.operatingSystem === 'windows') {
-                                        templateFileName = 'askui-helper-windows.nj';
-                                    }
+                                    const templateFileName = 'askui-helper.nj';
                                     nunjucks.configure(askuiHelperTemplateFilePath, { autoescape: false });
                                     const result = nunjucks.render(templateFileName, this.helperTemplateConfig);
                                     const filePath = path.join(this.automationsDirectoryPath, 'helpers', 'askui-helper.ts');
@@ -187,24 +184,6 @@ export class CreateExampleProject {
             yield runCommand(frameworkDependencies.jest);
         });
     }
-    addUserCredentials() {
-        return __awaiter(this, void 0, void 0, function* () {
-            return [
-                {
-                    /* eslint-disable sort-keys */
-                    title: 'Add user credentials',
-                    enabled: () => !this.cliOptions.skipCredentials,
-                    task: () => __awaiter(this, void 0, void 0, function* () {
-                        this.helperTemplateConfig['credentials'] = ` credentials: {
-        workspaceId: '${this.cliOptions.workspaceId}',
-        token: '${this.cliOptions.accessToken}',
-      },`;
-                    }),
-                },
-                /* eslint-enable */
-            ];
-        });
-    }
     copyESLintConfigFiles() {
         return __awaiter(this, void 0, void 0, function* () {
             const esLintRcFilePath = path.join('example_projects_templates', 'typescript', '.eslintrc.json-template');
@@ -276,20 +255,14 @@ export class CreateExampleProject {
                 ...(yield this.copyESLintConfigFiles()),
                 ...(yield this.copyGitignore()),
                 ...(yield this.copyTsConfigFile()),
-                ...(yield this.addUserCredentials()),
                 ...(yield this.createAskUIHelperFromTemplate()),
             ]);
             yield tasks.run();
             /* eslint-disable no-console */
             console.log(chalk.greenBright('\nCongratulations!'));
             console.log(`askui example was created under ${chalk.gray(this.automationsDirectoryPath)}`);
-            if (this.cliOptions.operatingSystem === 'windows') {
-                console.log(chalk.redBright(`\nPlease make sure the AskUI Controller is running: ${this.askUIControllerUrl}\n`));
-                console.log(`You can start your automation with this command ${chalk.green('AskUI-RunProject')}`);
-            }
-            else {
-                console.log(`You can start your automation with this command ${chalk.green('npm run askui')}`);
-            }
+            console.log(chalk.redBright(`\nPlease make sure the AskUI Controller is running: ${this.askUIControllerUrl}\n`));
+            console.log(`You can start your automation with this command ${chalk.green('AskUI-RunProject')}`);
             /* eslint-enable no-console */
         });
     }

package/dist/esm/lib/ui-controller.d.ts

@@ -1,4 +1,8 @@
 import { UiControllerArgs } from './ui-controller-args';
+/**
+ * @deprecated Will be removed soon because the AskUI Controller
+ *             gets started as an operating system service.
+ */
 export declare class UiController {
     private args?;
     private server;

package/dist/esm/lib/ui-controller.js

@@ -11,6 +11,10 @@ import { UiControllerDarwin } from './ui-controller-darwin';
 import { UiControllerLinux } from './ui-controller-linux';
 import { UiControllerWin32 } from './ui-controller-win32';
 import { platform } from './download-binaries';
+/**
+ * @deprecated Will be removed soon because the AskUI Controller
+ *             gets started as an operating system service.
+ */
 export class UiController {
     constructor(args) {
         this.args = args;

package/dist/example_projects_templates/templates/askui-helper.nj

@@ -1,27 +1,14 @@
-import { UiControlClient, UiController } from 'askui';
+import { UiControlClient } from 'askui';
 {{ allure_stepreporter_import }}
 
-// Server for controlling the operating system
-let uiController: UiController;
-
 // Client is necessary to use the askui API
+// eslint-disable-next-line import/no-mutable-exports
 let aui: UiControlClient;
 
 {{ timeout_placeholder }}
 
 beforeAll(async () => {
-  uiController = new UiController({
-    /**
-      * Select the display you want to run your tests on, display 0 is your main display;
-      * ignore if you have only one display
-      */
-    display: 0,
-  });
-
-  await uiController.start();
-
   aui = await UiControlClient.build({
-    {{ credentials }}
     {{ reporter_placeholder }}
   });
 
@@ -43,8 +30,6 @@ afterEach(async () => {
 
 afterAll(async () => {
   aui.disconnect();
-
-  await uiController.stop();
 });
 
 export { aui };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "askui",
-    "version": "0.20.6",
+    "version": "0.20.8",
     "license": "MIT",
     "author": "askui GmbH <info@askui.com> (http://www.askui.com/)",
     "description": "Reliable, automated end-to-end-testing that depends on what is shown on your screen instead of the technology you are running on",

package/dist/example_projects_templates/templates/askui-helper-windows.nj

@@ -1,36 +0,0 @@
-import { UiControlClient } from 'askui';
-{{ allure_stepreporter_import }}
-
-// Client is necessary to use the askui API
-// eslint-disable-next-line import/no-mutable-exports
-let aui: UiControlClient;
-
-{{ timeout_placeholder }}
-
-beforeAll(async () => {
-  aui = await UiControlClient.build({
-    {{ credentials }}
-    {{ reporter_placeholder }}
-  });
-
-  await aui.connect();
-});
-
-beforeEach(async () => {
-  /* Uncomment to enable video recording
-  await aui.startVideoRecording();
-  */
-});
-
-afterEach(async () => {
-  /* Uncomment to enable video recording
-  await aui.stopVideoRecording();
-  {{ allure_stepreporter_attach_video }}
-  */
-});
-
-afterAll(async () => {
-  aui.disconnect();
-});
-
-export { aui };