askui 0.18.0 → 0.19.0

package/dist/cjs/execution/index.d.ts

@@ -1 +1 @@
-export { UiControlClient, RelationsForConvenienceMethods } from './ui-control-client';
+export * from './ui-control-client';

package/dist/cjs/execution/index.js

@@ -1,5 +1,13 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.UiControlClient = void 0;
-var ui_control_client_1 = require("./ui-control-client");
-Object.defineProperty(exports, "UiControlClient", { enumerable: true, get: function () { return ui_control_client_1.UiControlClient; } });
+__exportStar(require("./ui-control-client"), exports);

package/dist/cjs/execution/inference-client.js

@@ -62,12 +62,17 @@ class InferenceClient {
     inference(customElements = [], image, instruction) {
         return __awaiter(this, void 0, void 0, function* () {
             const resizedImage = yield this.resizeIfNeeded(customElements, image);
-            const response = yield this.httpClient.post(this.urls.inference, {
-                customElements,
+            const response = yield this.httpClient.post(this.urls.inference, this.urls.inference.includes('v4-experimental') ? {
                 image: resizedImage.base64Image,
                 instruction,
-                modelComposition: this.modelComposition,
-            });
+                tasks: ['OCR'],
+            }
+                : {
+                    customElements,
+                    image: resizedImage.base64Image,
+                    instruction,
+                    modelComposition: this.modelComposition,
+                });
             InferenceClient.logMetaInformation(response);
             return ui_control_commands_1.InferenceResponse.fromJson(response.body, resizedImage.resizeRatio, image);
         });

package/dist/cjs/execution/ui-control-client-dependency-builder.d.ts

@@ -8,6 +8,7 @@ export declare class UiControlClientDependencyBuilder {
     static build(clientArgs: ClientArgsWithDefaults): Promise<{
         executionRuntime: ExecutionRuntime;
         stepReporter: StepReporter;
+        workspaceId: string | undefined;
     }>;
     static getClientArgsWithDefaults(clientArgs: ClientArgs): Promise<ClientArgsWithDefaults>;
 }

package/dist/cjs/execution/ui-control-client-dependency-builder.js

@@ -36,29 +36,32 @@ class UiControlClientDependencyBuilder {
         var _a;
         return __awaiter(this, void 0, void 0, function* () {
             const httpClient = yield UiControlClientDependencyBuilder.buildHttpClient(clientArgs);
-            return new inference_client_1.InferenceClient(clientArgs.inferenceServerUrl, httpClient, clientArgs.resize, (_a = clientArgs.credentials) === null || _a === void 0 ? void 0 : _a.workspaceId, clientArgs.modelComposition);
+            return new inference_client_1.InferenceClient(clientArgs.inferenceServerUrl, httpClient, clientArgs.resize, (_a = clientArgs.credentials) === null || _a === void 0 ? void 0 : _a.workspaceId, clientArgs.modelComposition, clientArgs.inferenceServerApiVersion);
         });
     }
     static buildUiControllerClient(clientArgs) {
         return new ui_controller_client_1.UiControllerClient(clientArgs.uiControllerUrl);
     }
     static build(clientArgs) {
+        var _a;
         return __awaiter(this, void 0, void 0, function* () {
             const uiControllerClient = UiControlClientDependencyBuilder.buildUiControllerClient(clientArgs);
             const inferenceClient = yield UiControlClientDependencyBuilder.buildInferenceClient(clientArgs);
             const stepReporter = new reporting_1.StepReporter(clientArgs.reporter);
+            const workspaceId = (_a = clientArgs.credentials) === null || _a === void 0 ? void 0 : _a.workspaceId;
             return {
                 executionRuntime: new execution_runtime_1.ExecutionRuntime(uiControllerClient, inferenceClient, stepReporter),
                 stepReporter,
+                workspaceId,
             };
         });
     }
     static getClientArgsWithDefaults(clientArgs) {
-        var _a, _b, _c, _d, _e;
+        var _a, _b, _c, _d, _e, _f;
         return __awaiter(this, void 0, void 0, function* () {
             return Object.assign(Object.assign({}, clientArgs), { context: {
                     isCi: (_b = (_a = clientArgs.context) === null || _a === void 0 ? void 0 : _a.isCi) !== null && _b !== void 0 ? _b : is_ci_1.default,
-                }, credentials: (0, read_credentials_1.readCredentials)(clientArgs), inferenceServerUrl: (_c = clientArgs.inferenceServerUrl) !== null && _c !== void 0 ? _c : 'https://inference.askui.com', proxyAgents: (_d = clientArgs.proxyAgents) !== null && _d !== void 0 ? _d : (yield (0, proxy_builder_1.envProxyAgents)()), uiControllerUrl: (_e = clientArgs.uiControllerUrl) !== null && _e !== void 0 ? _e : 'http://127.0.0.1:6769' });
+                }, credentials: (0, read_credentials_1.readCredentials)(clientArgs), inferenceServerApiVersion: (_c = clientArgs.inferenceServerApiVersion) !== null && _c !== void 0 ? _c : 'v3', inferenceServerUrl: (_d = clientArgs.inferenceServerUrl) !== null && _d !== void 0 ? _d : 'https://inference.askui.com', proxyAgents: (_e = clientArgs.proxyAgents) !== null && _e !== void 0 ? _e : (yield (0, proxy_builder_1.envProxyAgents)()), uiControllerUrl: (_f = clientArgs.uiControllerUrl) !== null && _f !== void 0 ? _f : 'http://127.0.0.1:6769' });
         });
     }
 }

package/dist/cjs/execution/ui-control-client.d.ts

@@ -1,12 +1,34 @@
-import { CustomElementJson } from '../core/model/custom-element-json';
-import { Exec, Executable, FluentFilters, ApiCommands, PC_AND_MODIFIER_KEY } from './dsl';
+import { Exec, Executable, FluentFilters, ApiCommands, PC_AND_MODIFIER_KEY, CommandExecutorContext } from './dsl';
 import { UiControllerClientConnectionState } from './ui-controller-client-connection-state';
 import { Annotation } from '../core/annotation/annotation';
 import { AnnotationRequest } from '../core/model/annotation-result/annotation-interface';
 import { DetectedElement } from '../core/model/annotation-result/detected-element';
 import { ClientArgs } from './ui-controller-client-interface';
 export declare type RelationsForConvenienceMethods = 'nearestTo' | 'leftOf' | 'above' | 'rightOf' | 'below' | 'contains';
+export declare type TextMatchingOption = 'similar' | 'exact' | 'regex';
+export declare type ElementExistsQueryType = 'otherElement' | 'switch' | 'element' | 'container' | 'checkbox' | 'element' | 'button' | 'table' | 'text' | 'icon' | 'image' | 'textfield';
+export interface ElementExistsQueryText {
+    value: string;
+    matching?: TextMatchingOption;
+}
+export interface ElementExistsQueryRelation {
+    type: RelationsForConvenienceMethods;
+    text: string;
+}
+export interface ElementExistsQuery {
+    type: keyof Pick<FluentFilters, ElementExistsQueryType>;
+    text?: ElementExistsQueryText;
+    relation?: ElementExistsQueryRelation;
+}
+export interface ExpectExistenceElement extends ElementExistsQuery {
+    exists: boolean;
+}
+export interface ExpectAllExistResult {
+    allExist: boolean;
+    elements: ExpectExistenceElement[];
+}
 export declare class UiControlClient extends ApiCommands {
+    private workspaceId;
     private executionRuntime;
     private stepReporter;
     private constructor();
@@ -34,8 +56,9 @@ export declare class UiControlClient extends ApiCommands {
     annotateInteractively(): Promise<void>;
     private escapeSeparatorString;
     private buildInstruction;
-    fluentCommandExecutor(instructionString: string, customElementJson?: CustomElementJson[]): Promise<void>;
-    getterExecutor(instruction: string, customElementJson?: CustomElementJson[]): Promise<DetectedElement[]>;
+    private getAIElementsByNames;
+    fluentCommandExecutor(instructionString: string, context?: CommandExecutorContext): Promise<void>;
+    getterExecutor(instruction: string, context?: CommandExecutorContext): Promise<DetectedElement[]>;
     private secretText;
     private getAndResetSecretText;
     /**
@@ -190,8 +213,8 @@ export declare class UiControlClient extends ApiCommands {
      * @param {Object} params - Object containing required `label` property and
      *                          optional `relation` property.
      * @property {string} params.label - The label for the checkbox.
-     * @property {Object} params.relation - Object describing the relationship between
-     *                                      the clicked checkbox and another element.
+     * @property {Object} [params.relation] - Object describing the relationship between
+     *                                        the clicked checkbox and another element.
      * @property {RelationsForConvenienceMethods} params.relation.type - The type of relation.
      */
     clickCheckbox(params: {
@@ -213,7 +236,7 @@ export declare class UiControlClient extends ApiCommands {
      * @param {Object} params - Object containing required `label` property and
      *                          optional `relation` property.
      * @property {string} params.label - The label for the checkbox.
-     * @property {Object} params.relation - Object describing the relationship between
+     * @property {Object} [params.relation] - Object describing the relationship between
      *                                      the clicked checkbox and another element.
      * @property {RelationsForConvenienceMethods} params.relation.type - The type of relation.
      */
@@ -266,41 +289,103 @@ export declare class UiControlClient extends ApiCommands {
     }): Promise<void>;
     /**
      * Click on a specific text.
-     * You can also use a RegEx or match the text exactly by specifyicing the specific flag.
+     * You can also use a RegEx or match the text exactly by specifying the specific flag.
      * Use a relation to find the text in relation to a specific text.
      *
      * **Examples:**
      * ```typescript
      * // Click text that matches exactly
-     * await aui.clickText({text: 'askui', type: 'similar'})
+     * await aui.clickText({text: 'askui', matching: 'similar'})
      *
      * // Click text that contains 'pie' or 'cake' or 'Pie' or 'Cake'
-     * await aui.clickText({text: '.*([Pp]ie|[Cc]ake).*', type: 'regex'})
+     * await aui.clickText({text: '.*([Pp]ie|[Cc]ake).*', matching: 'regex'})
      *
      * // Click the text 'TERMINAL' that is left of the text 'Ports'
      * await aui.clickText({
      *     text: 'TERMINAL',
-     *     type: "exact",
+     *     matching: "exact",
      *     relation: { type: 'leftOf', text: 'PORTS' }
      *   })
      * ```
+     *
      * @param {Object} params - Object containing required `text` property and optional properties
      *                          for regular expression matching and relation.
      * @property {string} params.text - The text to be clicked.
-     * @property {string} params.type - Whether the text is matched using similarity,
-     *                                  exact match or a regular expression.
-     * @property {Object} params.relation - Object describing the relationship between the
-     *                                      clicked text and another element.
+     * @property {string} params.matching - Whether the text is matched using similarity,
+     *                                      exact match or a regular expression.
+     * @property {Object} [params.relation] - Object describing the relationship between the
+     *                                        clicked text and another element.
      * @property {RelationsForConvenienceMethods} params.relation.type - The type of relation.
      * @property {string} params.relation.text - The label or text associated with the
      *                                           related element or state.
      */
     clickText(params: {
         text: string;
-        type: 'similar' | 'exact' | 'regex';
+        matching: TextMatchingOption;
         relation?: {
             type: RelationsForConvenienceMethods;
             text: string;
         };
     }): Promise<void>;
+    private evaluateMatchingProperty;
+    /**
+     * Check if one or multiple elements are detected.
+     *
+     * **Examples:**
+     * ```typescript
+     * await aui.expectAllExist([
+     *   {
+     *     type: 'text',
+     *     text: {
+     *       value: 'Switch to Dark',
+     *       matching: 'similar'
+     *     }
+     *   },
+     * ]);
+     *
+     * // Check for existence of multiple elements
+     * await aui.expectAllExist([
+     *   {
+     *     type: 'textfield',
+     *     relation: {
+     *       type: 'rightOf',
+     *       text: 'Email:'
+     *     }
+     *   },
+     *   {
+     *     type: 'element',
+     *     text: {
+     *       value: 'Switch to Dark'
+     *     }
+     *   },
+     * ]);
+     *
+     * // Validate existence
+     * const exists = await aui.expectAllExist([...]);
+     * exists.allExist // true when every element exists
+     *
+     * // Check which elements do not exist
+     * // with the elements property
+     * const nonExistentElements = exists.elements.filter((e) => e.exists===false)
+     * ```
+     *
+     * @param {ElementExistsQuery[]} query - Objects containing the required property
+     *                                                   'type' and the optional properties
+     *                                                   'text' and 'relation'.
+     * @property {string} query.type - The type of the element: 'otherElement' | 'switch' |
+     *                                 'element' | 'container' | 'checkbox' | 'element' |
+     *                                 'button' | 'table' | 'text' | 'icon' | 'image' | 'textfield'
+     * @property {Object} [query.text] - Object containing value and matching strategy.
+     * @property {string} query.text.value - The text to match for.
+     * @property {string} [query.text.matching] - Whether the text is matched using similarity,
+     *                                            exact match or a regular expression.
+     * @property {Object} [query.relation] - Object describing the relationship between the
+     *                                       clicked text and another element.
+     * @property {RelationsForConvenienceMethods} query.relation.type - The type of relation.
+     * @property {string} query.relation.text - The label or text associated with the
+     *                                          related element or state.
+     * @returns {ExpectAllExistResult.allExist} - If every element exists.
+     * @returns {ExpectAllExistResult.elements} - ExpectExistenceElement[].
+     */
+    expectAllExist(query: ElementExistsQuery[]): Promise<ExpectAllExistResult>;
 }

package/dist/cjs/execution/ui-control-client.js

@@ -8,20 +8,19 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.UiControlClient = void 0;
-const ValidationError_1 = __importDefault(require("yup/lib/ValidationError"));
+const yup_1 = require("yup");
 const custom_element_1 = require("../core/model/custom-element");
 const dsl_1 = require("./dsl");
 const annotation_writer_1 = require("../core/annotation/annotation-writer");
 const logger_1 = require("../lib/logger");
 const ui_control_client_dependency_builder_1 = require("./ui-control-client-dependency-builder");
+const ai_element_collection_1 = require("../core/ai-element/ai-element-collection");
 class UiControlClient extends dsl_1.ApiCommands {
-    constructor(executionRuntime, stepReporter) {
+    constructor(workspaceId, executionRuntime, stepReporter) {
         super();
+        this.workspaceId = workspaceId;
         this.executionRuntime = executionRuntime;
         this.stepReporter = stepReporter;
         this.secretText = undefined;
@@ -30,8 +29,8 @@ class UiControlClient extends dsl_1.ApiCommands {
         return __awaiter(this, void 0, void 0, function* () {
             const builder = ui_control_client_dependency_builder_1.UiControlClientDependencyBuilder;
             const clientArgsWithDefaults = yield builder.getClientArgsWithDefaults(clientArgs);
-            const { executionRuntime, stepReporter } = yield builder.build(clientArgsWithDefaults);
-            return new UiControlClient(executionRuntime, stepReporter);
+            const { workspaceId, executionRuntime, stepReporter, } = yield builder.build(clientArgsWithDefaults);
+            return new UiControlClient(workspaceId, executionRuntime, stepReporter);
         });
     }
     /**
@@ -125,9 +124,20 @@ class UiControlClient extends dsl_1.ApiCommands {
             };
         });
     }
-    fluentCommandExecutor(instructionString, customElementJson = []) {
+    getAIElementsByNames(names) {
+        return __awaiter(this, void 0, void 0, function* () {
+            // eslint-disable-next-line max-len
+            const workspaceAIElementCollection = yield ai_element_collection_1.AIElementCollection.collectForWorkspaceId(this.workspaceId);
+            return workspaceAIElementCollection.getByNames(names);
+        });
+    }
+    fluentCommandExecutor(instructionString, context = { customElementsJson: [], aiElementNames: [] }) {
         return __awaiter(this, void 0, void 0, function* () {
-            const instruction = yield this.buildInstruction(instructionString, customElementJson);
+            const aiElements = yield this.getAIElementsByNames(context.aiElementNames);
+            const instruction = yield this.buildInstruction(instructionString, [
+                ...context.customElementsJson,
+                ...aiElements,
+            ]);
             logger_1.logger.debug(instruction);
             try {
                 yield this.stepReporter.resetStep(instruction);
@@ -141,12 +151,16 @@ class UiControlClient extends dsl_1.ApiCommands {
             }
         });
     }
-    getterExecutor(instruction, customElementJson = []) {
+    getterExecutor(instruction, context = { customElementsJson: [], aiElementNames: [] }) {
         return __awaiter(this, void 0, void 0, function* () {
-            const customElements = yield custom_element_1.CustomElement.fromJsonListWithImagePathOrImage(customElementJson);
+            const aiElements = yield this.getAIElementsByNames(context.aiElementNames);
+            const customElements = yield custom_element_1.CustomElement.fromJsonListWithImagePathOrImage(context.customElementsJson);
             const stringWithoutSeparators = this.escapeSeparatorString(instruction);
             logger_1.logger.debug(stringWithoutSeparators);
-            return this.executionRuntime.getDetectedElements(instruction, customElements);
+            return this.executionRuntime.getDetectedElements(instruction, [
+                ...customElements,
+                ...aiElements,
+            ]);
         });
     }
     getAndResetSecretText() {
@@ -331,30 +345,22 @@ class UiControlClient extends dsl_1.ApiCommands {
     }
     // eslint-disable-next-line class-methods-use-this
     evaluateRelation(command, relation, text) {
-        let commando = command;
         switch (relation) {
-            case 'nearestTo':
-                commando = command.nearestTo().text(text);
-                break;
             case 'leftOf':
-                commando = command.leftOf().text(text);
-                break;
+                return command.leftOf().text(text);
             case 'above':
-                commando = command.above().text(text);
-                break;
+                return command.above().text(text);
             case 'rightOf':
-                commando = command.rightOf().text(text);
-                break;
+                return command.rightOf().text(text);
             case 'below':
-                commando = command.below().text(text);
-                break;
+                return command.below().text(text);
             case 'contains':
-                commando = command.contains().text(text);
-                break;
+                return command.contains().text(text);
+            case 'nearestTo':
+                return command.nearestTo().text(text);
             default:
-                throw new ValidationError_1.default('No valid Relation.Type was passed.');
+                throw new yup_1.ValidationError(`'relation' has to be 'nearestTo', 'leftOf', 'above', 'rightOf', 'below' or 'contains' but was '${relation}'`);
         }
-        return commando;
     }
     /**
      * Click a button with a specific label.
@@ -400,8 +406,8 @@ class UiControlClient extends dsl_1.ApiCommands {
      * @param {Object} params - Object containing required `label` property and
      *                          optional `relation` property.
      * @property {string} params.label - The label for the checkbox.
-     * @property {Object} params.relation - Object describing the relationship between
-     *                                      the clicked checkbox and another element.
+     * @property {Object} [params.relation] - Object describing the relationship between
+     *                                        the clicked checkbox and another element.
      * @property {RelationsForConvenienceMethods} params.relation.type - The type of relation.
      */
     clickCheckbox(params) {
@@ -429,7 +435,7 @@ class UiControlClient extends dsl_1.ApiCommands {
      * @param {Object} params - Object containing required `label` property and
      *                          optional `relation` property.
      * @property {string} params.label - The label for the checkbox.
-     * @property {Object} params.relation - Object describing the relationship between
+     * @property {Object} [params.relation] - Object describing the relationship between
      *                                      the clicked checkbox and another element.
      * @property {RelationsForConvenienceMethods} params.relation.type - The type of relation.
      */
@@ -493,31 +499,32 @@ class UiControlClient extends dsl_1.ApiCommands {
     }
     /**
      * Click on a specific text.
-     * You can also use a RegEx or match the text exactly by specifyicing the specific flag.
+     * You can also use a RegEx or match the text exactly by specifying the specific flag.
      * Use a relation to find the text in relation to a specific text.
      *
      * **Examples:**
      * ```typescript
      * // Click text that matches exactly
-     * await aui.clickText({text: 'askui', type: 'similar'})
+     * await aui.clickText({text: 'askui', matching: 'similar'})
      *
      * // Click text that contains 'pie' or 'cake' or 'Pie' or 'Cake'
-     * await aui.clickText({text: '.*([Pp]ie|[Cc]ake).*', type: 'regex'})
+     * await aui.clickText({text: '.*([Pp]ie|[Cc]ake).*', matching: 'regex'})
      *
      * // Click the text 'TERMINAL' that is left of the text 'Ports'
      * await aui.clickText({
      *     text: 'TERMINAL',
-     *     type: "exact",
+     *     matching: "exact",
      *     relation: { type: 'leftOf', text: 'PORTS' }
      *   })
      * ```
+     *
      * @param {Object} params - Object containing required `text` property and optional properties
      *                          for regular expression matching and relation.
      * @property {string} params.text - The text to be clicked.
-     * @property {string} params.type - Whether the text is matched using similarity,
-     *                                  exact match or a regular expression.
-     * @property {Object} params.relation - Object describing the relationship between the
-     *                                      clicked text and another element.
+     * @property {string} params.matching - Whether the text is matched using similarity,
+     *                                      exact match or a regular expression.
+     * @property {Object} [params.relation] - Object describing the relationship between the
+     *                                        clicked text and another element.
      * @property {RelationsForConvenienceMethods} params.relation.type - The type of relation.
      * @property {string} params.relation.text - The label or text associated with the
      *                                           related element or state.
@@ -525,24 +532,107 @@ class UiControlClient extends dsl_1.ApiCommands {
     clickText(params) {
         return __awaiter(this, void 0, void 0, function* () {
             let command = this.click().text();
-            switch (params.type) {
-                case 'similar':
-                    command = command.withText(params.text);
-                    break;
-                case 'exact':
-                    command = command.withExactText(params.text);
-                    break;
-                case 'regex':
-                    command = command.withTextRegex(params.text);
-                    break;
-                default:
-                    throw new ValidationError_1.default('"type" must be "similar", "exact" or "regex"');
-            }
+            command = this.evaluateMatchingProperty(command, { value: params.text, matching: params.matching });
             if (params.relation) {
                 command = this.evaluateRelation(command, params.relation.type, params.relation.text);
             }
             yield command.exec();
         });
     }
+    // eslint-disable-next-line class-methods-use-this
+    evaluateMatchingProperty(command, text) {
+        var _a;
+        switch ((_a = text.matching) !== null && _a !== void 0 ? _a : 'similar') {
+            case 'exact':
+                return command.withExactText(text.value);
+            case 'regex':
+                return command.withTextRegex(text.value);
+            case 'similar':
+                return command.withText(text.value);
+            default:
+                throw new yup_1.ValidationError(`'text.matching' property has to be 'similar', 'exact' or 'regex' but was '${text.matching}'`);
+        }
+    }
+    /**
+     * Check if one or multiple elements are detected.
+     *
+     * **Examples:**
+     * ```typescript
+     * await aui.expectAllExist([
+     *   {
+     *     type: 'text',
+     *     text: {
+     *       value: 'Switch to Dark',
+     *       matching: 'similar'
+     *     }
+     *   },
+     * ]);
+     *
+     * // Check for existence of multiple elements
+     * await aui.expectAllExist([
+     *   {
+     *     type: 'textfield',
+     *     relation: {
+     *       type: 'rightOf',
+     *       text: 'Email:'
+     *     }
+     *   },
+     *   {
+     *     type: 'element',
+     *     text: {
+     *       value: 'Switch to Dark'
+     *     }
+     *   },
+     * ]);
+     *
+     * // Validate existence
+     * const exists = await aui.expectAllExist([...]);
+     * exists.allExist // true when every element exists
+     *
+     * // Check which elements do not exist
+     * // with the elements property
+     * const nonExistentElements = exists.elements.filter((e) => e.exists===false)
+     * ```
+     *
+     * @param {ElementExistsQuery[]} query - Objects containing the required property
+     *                                                   'type' and the optional properties
+     *                                                   'text' and 'relation'.
+     * @property {string} query.type - The type of the element: 'otherElement' | 'switch' |
+     *                                 'element' | 'container' | 'checkbox' | 'element' |
+     *                                 'button' | 'table' | 'text' | 'icon' | 'image' | 'textfield'
+     * @property {Object} [query.text] - Object containing value and matching strategy.
+     * @property {string} query.text.value - The text to match for.
+     * @property {string} [query.text.matching] - Whether the text is matched using similarity,
+     *                                            exact match or a regular expression.
+     * @property {Object} [query.relation] - Object describing the relationship between the
+     *                                       clicked text and another element.
+     * @property {RelationsForConvenienceMethods} query.relation.type - The type of relation.
+     * @property {string} query.relation.text - The label or text associated with the
+     *                                          related element or state.
+     * @returns {ExpectAllExistResult.allExist} - If every element exists.
+     * @returns {ExpectAllExistResult.elements} - ExpectExistenceElement[].
+     */
+    expectAllExist(query) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const elements = yield query.reduce((accumulatorPromise, subquery) => __awaiter(this, void 0, void 0, function* () {
+                const acc = yield accumulatorPromise;
+                const command = this.get()[subquery.type]();
+                let finalCommand = subquery.text !== undefined
+                    ? this.evaluateMatchingProperty(command, subquery.text)
+                    : command;
+                if (subquery.relation) {
+                    finalCommand = this.evaluateRelation(finalCommand, subquery.relation.type, subquery.relation.text);
+                }
+                return [
+                    ...acc,
+                    Object.assign(Object.assign({}, subquery), { exists: (yield finalCommand.exec()).length > 0 }),
+                ];
+            }), Promise.resolve([]));
+            return {
+                elements,
+                allExist: elements.every((el) => el.exists),
+            };
+        });
+    }
 }
 exports.UiControlClient = UiControlClient;

package/dist/cjs/execution/ui-controller-client-interface.d.ts

@@ -53,9 +53,11 @@ export interface ClientArgs {
     readonly modelComposition?: ModelCompositionBranch[];
     readonly reporter?: Reporter | Reporter[] | undefined;
     readonly context?: ContextArgs | undefined;
+    readonly inferenceServerApiVersion?: string;
 }
 export interface ClientArgsWithDefaults extends ClientArgs {
     readonly uiControllerUrl: string;
     readonly inferenceServerUrl: string;
     readonly context: Context;
+    readonly inferenceServerApiVersion: string;
 }

package/dist/cjs/execution/ui-controller-client.d.ts

@@ -18,6 +18,7 @@ export declare class UiControllerClient {
     private onMessage;
     connect(): Promise<UiControllerClientConnectionState>;
     disconnect(): void;
+    private checkConnection;
     private sendAndReceive;
     private send;
     requestScreenshot(): Promise<CaptureScreenshotResponse>;

package/dist/cjs/execution/ui-controller-client.js

@@ -10,6 +10,7 @@ const logger_1 = require("../lib/logger");
 const ui_controller_client_connection_state_1 = require("./ui-controller-client-connection-state");
 const read_recording_response_stream_handler_1 = require("./read-recording-response-stream-handler");
 const ui_controller_client_error_1 = require("./ui-controller-client-error");
+const ui_controller_not_connected_error_1 = require("./ui-controller-not-connected-error");
 class UiControllerClient {
     constructor(url) {
         this.url = url;
@@ -40,7 +41,9 @@ class UiControllerClient {
         return new Promise((resolve, reject) => {
             try {
                 this.ws = new ws_1.default(this.url);
-                this.ws.on('message', (data) => { this.onMessage(data); });
+                this.ws.on('message', (data) => {
+                    this.onMessage(data);
+                });
                 this.ws.on('open', () => {
                     this.connectionState = ui_controller_client_connection_state_1.UiControllerClientConnectionState.CONNECTED;
                     resolve(this.connectionState);
@@ -61,7 +64,13 @@ class UiControllerClient {
         var _a;
         (_a = this.ws) === null || _a === void 0 ? void 0 : _a.close();
     }
+    checkConnection() {
+        if (this.connectionState !== ui_controller_client_connection_state_1.UiControllerClientConnectionState.CONNECTED) {
+            throw new ui_controller_not_connected_error_1.UiControllerNotConnectedError();
+        }
+    }
     sendAndReceive(msg, requestTimeout = UiControllerClient.REQUEST_TIMEOUT_IN_MS) {
+        this.checkConnection();
         return new Promise((resolve, reject) => {
             this.currentResolve = resolve;
             this.currentReject = reject;
@@ -75,6 +84,7 @@ class UiControllerClient {
         });
     }
     send(msg, _requestTimeout = UiControllerClient.REQUEST_TIMEOUT_IN_MS) {
+        this.checkConnection();
         if (!this.currentReject || !this.currentResolve) {
             throw new ui_controller_client_error_1.UiControllerClientError('Request is not finished! It is not possible to have multiple requests at the same time.');
         }

package/dist/cjs/execution/ui-controller-not-connected-error.d.ts

@@ -0,0 +1,4 @@
+import { UiControllerClientError } from './ui-controller-client-error';
+export declare class UiControllerNotConnectedError extends UiControllerClientError {
+    constructor();
+}

package/dist/cjs/execution/ui-controller-not-connected-error.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.UiControllerNotConnectedError = void 0;
+const ui_controller_client_error_1 = require("./ui-controller-client-error");
+class UiControllerNotConnectedError extends ui_controller_client_error_1.UiControllerClientError {
+    constructor() {
+        super('UI Controller is not connected. Did you call `UiControlClient.connect()`'
+            + ' before trying to interact with the UI Controller / OS?');
+        this.name = 'UiControllerNotConnectedError';
+    }
+}
+exports.UiControllerNotConnectedError = UiControllerNotConnectedError;