askui 0.20.9 → 0.21.0

package/dist/esm/execution/dsl.d.ts

@@ -1,5 +1,6 @@
 import { CustomElementJson } from '../core/model/custom-element-json';
 import { DetectedElement } from '../core/model/annotation-result/detected-element';
+import { ModelCompositionBranch } from './model-composition-branch';
 export declare enum Separators {
     STRING = "<|string|>"
 }
@@ -19,7 +20,7 @@ declare abstract class FluentBase {
     protected _textStr: string;
     protected _params: Map<string, unknown>;
     protected static addParams(paramsList: Map<string, unknown[]>, params: Map<string, unknown>): Map<string, unknown[]>;
-    protected fluentCommandStringBuilder(currentInstruction?: string, paramsList?: Map<string, unknown[]>): Promise<void>;
+    protected fluentCommandStringBuilder(modelComposition: ModelCompositionBranch[], currentInstruction?: string, paramsList?: Map<string, unknown[]>): Promise<void>;
     protected getterStringBuilder(currentInstruction?: string, paramsList?: Map<string, unknown[]>): Promise<DetectedElement[]>;
     protected get textStr(): string;
     protected get params(): Map<string, unknown>;
@@ -28,7 +29,7 @@ export interface Executable {
     exec(): Promise<void>;
 }
 export declare class Exec extends FluentBase implements Executable {
-    exec(): Promise<void>;
+    exec(modelComposition?: ModelCompositionBranch[]): Promise<void>;
 }
 export declare class FluentFilters extends FluentBase {
     /**
@@ -151,6 +152,22 @@ export declare class FluentFilters extends FluentBase {
      * @return {FluentFiltersOrRelations}
      */
     text(text?: string): FluentFiltersOrRelations;
+    /**
+     * Filters for an UI element 'wordlevel'.
+     *
+     * Takes an optional parameter to filter for a specific text.
+     * See the examples below.
+     *
+     * **Examples:**
+     * ```typescript
+     * await aui.fluentCommandExecutor('Click on button right of wordlevel 'User_PRG in:'');
+     * ```
+     *
+     * @param {string} [wordlevel] - A text to be matched.
+     *
+     * @return {FluentFiltersOrRelations}
+     */
+    wordlevel(wordlevel?: string): FluentFiltersOrRelations;
     /**
      * Filters for a UI element 'icon'.
      *
@@ -211,7 +228,7 @@ export declare class FluentFilters extends FluentBase {
      */
     customElement(customElement: CustomElementJson): FluentFiltersOrRelations;
     /**
-     * Detects an AI Element created with the [snipping workflow](https://docs.askui.com/docs/general/Components/aielement#snipping-workflow).
+     * Detects an AI Element created with the [snipping workflow](https://docs.askui.com/docs/general/Element%20Selection/aielement#snipping-workflow).
      *
      * **Examples:**
      *
@@ -362,6 +379,14 @@ export declare class FluentFilters extends FluentBase {
      * @return {FluentFiltersOrRelations}
      */
     containsText(text: string): FluentFiltersOrRelations;
+    /**
+     * Filters for PTA locating the text provided as an argument.
+     *
+     * @param {string} text - A text to be located.
+     *
+     * @return {FluentFiltersOrRelations}
+     */
+    pta(text: string): FluentFiltersOrRelations;
     /**
      * Filters elements based on a textual description.
      *
@@ -838,7 +863,7 @@ export declare class FluentFiltersOrRelations extends FluentFilters {
      * @return {FluentFilters}
      */
     contains(): FluentFilters;
-    exec(): Promise<void>;
+    exec(modelComposition?: ModelCompositionBranch[]): Promise<void>;
 }
 export declare class FluentFiltersCondition extends FluentBase {
     /**
@@ -961,6 +986,22 @@ export declare class FluentFiltersCondition extends FluentBase {
      * @return {FluentFiltersOrRelationsCondition}
      */
     text(text?: string): FluentFiltersOrRelationsCondition;
+    /**
+     * Filters for an UI element 'wordlevel'.
+     *
+     * Takes an optional parameter to filter for a specific text.
+     * See the examples below.
+     *
+     * **Examples:**
+     * ```typescript
+     * await aui.fluentCommandExecutor('Click on button right of wordlevel 'User_PRG in:'');
+     * ```
+     *
+     * @param {string} [wordlevel] - A text to be matched.
+     *
+     * @return {FluentFiltersOrRelationsCondition}
+     */
+    wordlevel(wordlevel?: string): FluentFiltersOrRelationsCondition;
     /**
      * Filters for a UI element 'icon'.
      *
@@ -1021,7 +1062,7 @@ export declare class FluentFiltersCondition extends FluentBase {
      */
     customElement(customElement: CustomElementJson): FluentFiltersOrRelationsCondition;
     /**
-     * Detects an AI Element created with the [snipping workflow](https://docs.askui.com/docs/general/Components/aielement#snipping-workflow).
+     * Detects an AI Element created with the [snipping workflow](https://docs.askui.com/docs/general/Element%20Selection/aielement#snipping-workflow).
      *
      * **Examples:**
      *
@@ -1172,6 +1213,14 @@ export declare class FluentFiltersCondition extends FluentBase {
      * @return {FluentFiltersOrRelationsCondition}
      */
     containsText(text: string): FluentFiltersOrRelationsCondition;
+    /**
+     * Filters for PTA locating the text provided as an argument.
+     *
+     * @param {string} text - A text to be located.
+     *
+     * @return {FluentFiltersOrRelationsCondition}
+     */
+    pta(text: string): FluentFiltersOrRelationsCondition;
     /**
      * Filters elements based on a textual description.
      *
@@ -2146,7 +2195,7 @@ export declare abstract class FluentCommand extends FluentBase {
      * @return {Exec}
      */
     pressAndroidKey(key: ANDROID_KEY): Exec;
-    abstract fluentCommandExecutor(instruction: string, context: CommandExecutorContext): Promise<void>;
+    abstract fluentCommandExecutor(instruction: string, modelComposition: ModelCompositionBranch[], context: CommandExecutorContext): Promise<void>;
 }
 export interface ExecutableGetter {
     exec(): Promise<DetectedElement[]>;
@@ -2275,6 +2324,22 @@ export declare class FluentFiltersGetter extends FluentBase {
      * @return {FluentFiltersOrRelationsGetter}
      */
     text(text?: string): FluentFiltersOrRelationsGetter;
+    /**
+     * Filters for an UI element 'wordlevel'.
+     *
+     * Takes an optional parameter to filter for a specific text.
+     * See the examples below.
+     *
+     * **Examples:**
+     * ```typescript
+     * await aui.fluentCommandExecutor('Click on button right of wordlevel 'User_PRG in:'');
+     * ```
+     *
+     * @param {string} [wordlevel] - A text to be matched.
+     *
+     * @return {FluentFiltersOrRelationsGetter}
+     */
+    wordlevel(wordlevel?: string): FluentFiltersOrRelationsGetter;
     /**
      * Filters for a UI element 'icon'.
      *
@@ -2335,7 +2400,7 @@ export declare class FluentFiltersGetter extends FluentBase {
      */
     customElement(customElement: CustomElementJson): FluentFiltersOrRelationsGetter;
     /**
-     * Detects an AI Element created with the [snipping workflow](https://docs.askui.com/docs/general/Components/aielement#snipping-workflow).
+     * Detects an AI Element created with the [snipping workflow](https://docs.askui.com/docs/general/Element%20Selection/aielement#snipping-workflow).
      *
      * **Examples:**
      *
@@ -2486,6 +2551,14 @@ export declare class FluentFiltersGetter extends FluentBase {
      * @return {FluentFiltersOrRelationsGetter}
      */
     containsText(text: string): FluentFiltersOrRelationsGetter;
+    /**
+     * Filters for PTA locating the text provided as an argument.
+     *
+     * @param {string} text - A text to be located.
+     *
+     * @return {FluentFiltersOrRelationsGetter}
+     */
+    pta(text: string): FluentFiltersOrRelationsGetter;
     /**
      * Filters elements based on a textual description.
      *

package/dist/esm/execution/dsl.js

@@ -49,14 +49,14 @@ class FluentBase {
         });
         return paramsList;
     }
-    fluentCommandStringBuilder(currentInstruction = '', paramsList = new Map()) {
+    fluentCommandStringBuilder(modelComposition, currentInstruction = '', paramsList = new Map()) {
         const newCurrentInstruction = `${this.textStr} ${currentInstruction}`;
         const newParamsList = FluentBase.addParams(paramsList, this._params);
         if (this instanceof FluentCommand) {
             const fluentCommand = this;
             const customElements = newParamsList.has('customElement') ? newParamsList.get('customElement') : [];
             const aiElementNames = newParamsList.has('aiElementName') ? newParamsList.get('aiElementName') : [];
-            return fluentCommand.fluentCommandExecutor(newCurrentInstruction.trim(), {
+            return fluentCommand.fluentCommandExecutor(newCurrentInstruction.trim(), modelComposition, {
                 customElementsJson: customElements,
                 aiElementNames,
             });
@@ -64,7 +64,7 @@ class FluentBase {
         if (!this.prev) {
             throw new Error('Prev element not defined');
         }
-        return this.prev.fluentCommandStringBuilder(newCurrentInstruction, newParamsList);
+        return this.prev.fluentCommandStringBuilder(modelComposition, newCurrentInstruction, newParamsList);
     }
     getterStringBuilder(currentInstruction = '', paramsList = new Map()) {
         const newCurrentInstruction = `${this.textStr} ${currentInstruction}`;
@@ -87,10 +87,10 @@ class FluentBase {
     get params() { return this._params; }
 }
 export class Exec extends FluentBase {
-    exec() {
+    exec(modelComposition = []) {
         const originStacktrace = { stack: '' };
         Error.captureStackTrace(originStacktrace, this.exec);
-        return this.fluentCommandStringBuilder().catch((err) => Promise.reject(rewriteStackTraceForError(err, originStacktrace.stack)));
+        return this.fluentCommandStringBuilder(modelComposition).catch((err) => Promise.reject(rewriteStackTraceForError(err, originStacktrace.stack)));
     }
 }
 // Filters
@@ -274,6 +274,29 @@ export class FluentFilters extends FluentBase {
         }
         return new FluentFiltersOrRelations(this);
     }
+    /**
+     * Filters for an UI element 'wordlevel'.
+     *
+     * Takes an optional parameter to filter for a specific text.
+     * See the examples below.
+     *
+     * **Examples:**
+     * ```typescript
+     * await aui.fluentCommandExecutor('Click on button right of wordlevel 'User_PRG in:'');
+     * ```
+     *
+     * @param {string} [wordlevel] - A text to be matched.
+     *
+     * @return {FluentFiltersOrRelations}
+     */
+    wordlevel(wordlevel) {
+        this._textStr = '';
+        this._textStr += 'wordlevel';
+        if (wordlevel !== undefined) {
+            this._textStr += ` ${Separators.STRING}${wordlevel}${Separators.STRING}`;
+        }
+        return new FluentFiltersOrRelations(this);
+    }
     /**
      * Filters for a UI element 'icon'.
      *
@@ -344,7 +367,7 @@ export class FluentFilters extends FluentBase {
         return new FluentFiltersOrRelations(this);
     }
     /**
-     * Detects an AI Element created with the [snipping workflow](https://docs.askui.com/docs/general/Components/aielement#snipping-workflow).
+     * Detects an AI Element created with the [snipping workflow](https://docs.askui.com/docs/general/Element%20Selection/aielement#snipping-workflow).
      *
      * **Examples:**
      *
@@ -540,6 +563,19 @@ export class FluentFilters extends FluentBase {
         this._textStr += ` ${Separators.STRING}${text}${Separators.STRING}`;
         return new FluentFiltersOrRelations(this);
     }
+    /**
+     * Filters for PTA locating the text provided as an argument.
+     *
+     * @param {string} text - A text to be located.
+     *
+     * @return {FluentFiltersOrRelations}
+     */
+    pta(text) {
+        this._textStr = '';
+        this._textStr += 'pta';
+        this._textStr += ` ${Separators.STRING}${text}${Separators.STRING}`;
+        return new FluentFiltersOrRelations(this);
+    }
     /**
      * Filters elements based on a textual description.
      *
@@ -1085,10 +1121,10 @@ export class FluentFiltersOrRelations extends FluentFilters {
         this._textStr += 'contains';
         return new FluentFilters(this);
     }
-    exec() {
+    exec(modelComposition = []) {
         const originStacktrace = { stack: '' };
         Error.captureStackTrace(originStacktrace, this.exec);
-        return this.fluentCommandStringBuilder().catch((err) => Promise.reject(rewriteStackTraceForError(err, originStacktrace.stack)));
+        return this.fluentCommandStringBuilder(modelComposition).catch((err) => Promise.reject(rewriteStackTraceForError(err, originStacktrace.stack)));
     }
 }
 // Filters
@@ -1272,6 +1308,29 @@ export class FluentFiltersCondition extends FluentBase {
         }
         return new FluentFiltersOrRelationsCondition(this);
     }
+    /**
+     * Filters for an UI element 'wordlevel'.
+     *
+     * Takes an optional parameter to filter for a specific text.
+     * See the examples below.
+     *
+     * **Examples:**
+     * ```typescript
+     * await aui.fluentCommandExecutor('Click on button right of wordlevel 'User_PRG in:'');
+     * ```
+     *
+     * @param {string} [wordlevel] - A text to be matched.
+     *
+     * @return {FluentFiltersOrRelationsCondition}
+     */
+    wordlevel(wordlevel) {
+        this._textStr = '';
+        this._textStr += 'wordlevel';
+        if (wordlevel !== undefined) {
+            this._textStr += ` ${Separators.STRING}${wordlevel}${Separators.STRING}`;
+        }
+        return new FluentFiltersOrRelationsCondition(this);
+    }
     /**
      * Filters for a UI element 'icon'.
      *
@@ -1342,7 +1401,7 @@ export class FluentFiltersCondition extends FluentBase {
         return new FluentFiltersOrRelationsCondition(this);
     }
     /**
-     * Detects an AI Element created with the [snipping workflow](https://docs.askui.com/docs/general/Components/aielement#snipping-workflow).
+     * Detects an AI Element created with the [snipping workflow](https://docs.askui.com/docs/general/Element%20Selection/aielement#snipping-workflow).
      *
      * **Examples:**
      *
@@ -1538,6 +1597,19 @@ export class FluentFiltersCondition extends FluentBase {
         this._textStr += ` ${Separators.STRING}${text}${Separators.STRING}`;
         return new FluentFiltersOrRelationsCondition(this);
     }
+    /**
+     * Filters for PTA locating the text provided as an argument.
+     *
+     * @param {string} text - A text to be located.
+     *
+     * @return {FluentFiltersOrRelationsCondition}
+     */
+    pta(text) {
+        this._textStr = '';
+        this._textStr += 'pta';
+        this._textStr += ` ${Separators.STRING}${text}${Separators.STRING}`;
+        return new FluentFiltersOrRelationsCondition(this);
+    }
     /**
      * Filters elements based on a textual description.
      *
@@ -2951,6 +3023,29 @@ export class FluentFiltersGetter extends FluentBase {
         }
         return new FluentFiltersOrRelationsGetter(this);
     }
+    /**
+     * Filters for an UI element 'wordlevel'.
+     *
+     * Takes an optional parameter to filter for a specific text.
+     * See the examples below.
+     *
+     * **Examples:**
+     * ```typescript
+     * await aui.fluentCommandExecutor('Click on button right of wordlevel 'User_PRG in:'');
+     * ```
+     *
+     * @param {string} [wordlevel] - A text to be matched.
+     *
+     * @return {FluentFiltersOrRelationsGetter}
+     */
+    wordlevel(wordlevel) {
+        this._textStr = '';
+        this._textStr += 'wordlevel';
+        if (wordlevel !== undefined) {
+            this._textStr += ` ${Separators.STRING}${wordlevel}${Separators.STRING}`;
+        }
+        return new FluentFiltersOrRelationsGetter(this);
+    }
     /**
      * Filters for a UI element 'icon'.
      *
@@ -3021,7 +3116,7 @@ export class FluentFiltersGetter extends FluentBase {
         return new FluentFiltersOrRelationsGetter(this);
     }
     /**
-     * Detects an AI Element created with the [snipping workflow](https://docs.askui.com/docs/general/Components/aielement#snipping-workflow).
+     * Detects an AI Element created with the [snipping workflow](https://docs.askui.com/docs/general/Element%20Selection/aielement#snipping-workflow).
      *
      * **Examples:**
      *
@@ -3217,6 +3312,19 @@ export class FluentFiltersGetter extends FluentBase {
         this._textStr += ` ${Separators.STRING}${text}${Separators.STRING}`;
         return new FluentFiltersOrRelationsGetter(this);
     }
+    /**
+     * Filters for PTA locating the text provided as an argument.
+     *
+     * @param {string} text - A text to be located.
+     *
+     * @return {FluentFiltersOrRelationsGetter}
+     */
+    pta(text) {
+        this._textStr = '';
+        this._textStr += 'pta';
+        this._textStr += ` ${Separators.STRING}${text}${Separators.STRING}`;
+        return new FluentFiltersOrRelationsGetter(this);
+    }
     /**
      * Filters elements based on a textual description.
      *

package/dist/esm/execution/execution-runtime.d.ts

@@ -6,6 +6,7 @@ import { DetectedElement } from '../core/model/annotation-result/detected-elemen
 import { UiControllerClientConnectionState } from './ui-controller-client-connection-state';
 import { Instruction, StepReporter } from '../core/reporting';
 import { RetryStrategy } from './retry-strategies/retry-strategy';
+import { ModelCompositionBranch } from './model-composition-branch';
 export declare class ExecutionRuntime {
     private uiControllerClient;
     private inferenceClient;
@@ -18,7 +19,7 @@ export declare class ExecutionRuntime {
     stopVideoRecording(): Promise<void>;
     readVideoRecording(): Promise<string>;
     private requestControl;
-    executeInstruction(instruction: Instruction): Promise<void>;
+    executeInstruction(instruction: Instruction, modelComposition: ModelCompositionBranch[]): Promise<void>;
     private readonly EXEC_REPETITION_COUNT;
     private executeCommandRepeatedly;
     /**

package/dist/esm/execution/execution-runtime.js

@@ -52,21 +52,21 @@ export class ExecutionRuntime {
             yield this.uiControllerClient.requestControl(controlCommand);
         });
     }
-    executeInstruction(instruction) {
+    executeInstruction(instruction, modelComposition) {
         return __awaiter(this, void 0, void 0, function* () {
             var _a, _b, _c;
-            const controlCommand = yield this.predictCommandWithRetry(instruction);
+            const controlCommand = yield this.predictCommandWithRetry(instruction, modelComposition);
             if (controlCommand.code === ControlCommandCode.OK) {
                 return this.requestControl(controlCommand);
             }
             if (controlCommand.tryToRepeat) {
                 yield this.requestControl(controlCommand);
-                return this.executeCommandRepeatedly(instruction);
+                return this.executeCommandRepeatedly(instruction, modelComposition);
             }
             throw new ControlCommandError((_c = (_b = (_a = controlCommand.actions) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.text) !== null && _c !== void 0 ? _c : '');
         });
     }
-    executeCommandRepeatedly(instruction) {
+    executeCommandRepeatedly(instruction, modelComposition) {
         return __awaiter(this, void 0, void 0, function* () {
             var _a, _b, _c;
             /* eslint-disable no-await-in-loop */
@@ -76,7 +76,7 @@ export class ExecutionRuntime {
                         + 'from a single test step reached');
                 }
                 logger.debug('Repeat command execution....');
-                const controlCommand = yield this.predictCommandWithRetry(instruction);
+                const controlCommand = yield this.predictCommandWithRetry(instruction, modelComposition);
                 if (controlCommand.code === ControlCommandCode.OK) {
                     break;
                 }
@@ -95,10 +95,10 @@ export class ExecutionRuntime {
      * --> retry with linear back-off
      */
     /* eslint-disable-next-line consistent-return */
-    predictCommandWithRetry(instruction) {
+    predictCommandWithRetry(instruction, modelComposition) {
         return __awaiter(this, void 0, void 0, function* () {
             var _a, _b, _c;
-            let command = yield this.predictCommand(instruction);
+            let command = yield this.predictCommand(instruction, modelComposition);
             /* eslint-disable no-await-in-loop */
             for (let k = 0; k < this.retryStrategy.retryCount; k += 1) {
                 if (command.code === ControlCommandCode.OK) {
@@ -107,7 +107,7 @@ export class ExecutionRuntime {
                 const msUntilRetry = this.retryStrategy.getDelay(k + 1);
                 logger.debug(`Wait ${msUntilRetry} and retry predicting command...`);
                 yield delay(msUntilRetry);
-                command = yield this.predictCommand(instruction, new ControlCommandError((_c = (_b = (_a = command.actions) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.text) !== null && _c !== void 0 ? _c : ''));
+                command = yield this.predictCommand(instruction, modelComposition, new ControlCommandError((_c = (_b = (_a = command.actions) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.text) !== null && _c !== void 0 ? _c : ''));
             }
             /* eslint-enable no-await-in-loop */
             return command;
@@ -147,14 +147,14 @@ export class ExecutionRuntime {
             };
         });
     }
-    predictCommand(instruction, retryError) {
+    predictCommand(instruction, modelComposition, retryError) {
         return __awaiter(this, void 0, void 0, function* () {
             const snapshot = yield this.buildSnapshot(instruction.value);
             if (retryError !== undefined)
                 this.stepReporter.onStepRetry(snapshot, retryError);
             else
                 this.stepReporter.onStepBegin(snapshot);
-            const controlCommand = yield this.inferenceClient.predictControlCommand(instruction.value, instruction.customElements, snapshot.screenshot);
+            const controlCommand = yield this.inferenceClient.predictControlCommand(instruction.value, modelComposition, instruction.customElements, snapshot.screenshot);
             if (instruction.secretText !== undefined) {
                 controlCommand.setTextToBeTyped(instruction.secretText);
             }

package/dist/esm/execution/inference-client.d.ts

@@ -16,10 +16,10 @@ export declare class InferenceClient {
     constructor(baseUrl: string, httpClient: HttpClientGot, resize?: number | undefined, workspaceId?: string | undefined, modelComposition?: ModelCompositionBranch[] | undefined, apiVersion?: string);
     isImageRequired(instruction: string): Promise<boolean>;
     private resizeIfNeeded;
-    inference(customElements?: CustomElement[], image?: string, instruction?: string): Promise<ControlCommand | Annotation>;
+    inference(customElements?: CustomElement[], image?: string, instruction?: string, modelComposition?: ModelCompositionBranch[]): Promise<ControlCommand | Annotation>;
     vqaInference(image: string, prompt: string, config?: object): Promise<VQAInferenceResponseBody>;
     private static logMetaInformation;
-    predictControlCommand(instruction: string, customElements?: CustomElement[], image?: string): Promise<ControlCommand>;
+    predictControlCommand(instruction: string, modelComposition: ModelCompositionBranch[], customElements?: CustomElement[], image?: string): Promise<ControlCommand>;
     getDetectedElements(instruction: string, image: string, customElements?: CustomElement[]): Promise<DetectedElement[]>;
     predictImageAnnotation(image: string, customElements?: CustomElement[]): Promise<Annotation>;
     predictVQAAnswer(prompt: string, image: string, config?: object): Promise<any>;

package/dist/esm/execution/inference-client.js

@@ -55,7 +55,7 @@ export class InferenceClient {
         });
     }
     inference() {
-        return __awaiter(this, arguments, void 0, function* (customElements = [], image, instruction) {
+        return __awaiter(this, arguments, void 0, function* (customElements = [], image, instruction, modelComposition = []) {
             const resizedImage = yield this.resizeIfNeeded(customElements, image);
             const response = yield this.httpClient.post(this.urls.inference, this.urls.inference.includes('v4-experimental') ? {
                 image: resizedImage.base64Image,
@@ -66,7 +66,7 @@ export class InferenceClient {
                     customElements,
                     image: resizedImage.base64Image,
                     instruction,
-                    modelComposition: this.modelComposition,
+                    modelComposition: modelComposition.length > 0 ? modelComposition : this.modelComposition,
                 });
             InferenceClient.logMetaInformation(response);
             return InferenceResponse.fromJson(response.body, resizedImage.resizeRatio, image);
@@ -88,9 +88,9 @@ export class InferenceClient {
             logger.warn(response.headers['askui-usage-warnings']);
         }
     }
-    predictControlCommand(instruction_1) {
-        return __awaiter(this, arguments, void 0, function* (instruction, customElements = [], image) {
-            const inferenceResponse = yield this.inference(customElements, image, instruction);
+    predictControlCommand(instruction_1, modelComposition_1) {
+        return __awaiter(this, arguments, void 0, function* (instruction, modelComposition, customElements = [], image) {
+            const inferenceResponse = yield this.inference(customElements, image, instruction, modelComposition);
             if (!(inferenceResponse instanceof ControlCommand)) {
                 throw new InferenceResponseError('Internal Error. Can not execute command');
             }

package/dist/esm/execution/ui-control-client.d.ts

@@ -4,6 +4,7 @@ import { Annotation } from '../core/annotation/annotation';
 import { AnnotationRequest } from '../core/model/annotation-result/annotation-interface';
 import { DetectedElement } from '../core/model/annotation-result/detected-element';
 import { ClientArgs } from './ui-controller-client-interface';
+import { ModelCompositionBranch } from './model-composition-branch';
 export type RelationsForConvenienceMethods = 'nearestTo' | 'leftOf' | 'above' | 'rightOf' | 'below' | 'contains';
 export type TextMatchingOption = 'similar' | 'exact' | 'regex';
 export type ElementExistsQueryType = 'otherElement' | 'switch' | 'element' | 'container' | 'checkbox' | 'element' | 'button' | 'table' | 'text' | 'icon' | 'image' | 'textfield';
@@ -57,7 +58,7 @@ export declare class UiControlClient extends ApiCommands {
     private escapeSeparatorString;
     private buildInstruction;
     private getAIElementsByNames;
-    fluentCommandExecutor(instructionString: string, context?: CommandExecutorContext): Promise<void>;
+    fluentCommandExecutor(instructionString: string, modelComposition: ModelCompositionBranch[], context?: CommandExecutorContext): Promise<void>;
     getterExecutor(instruction: string, context?: CommandExecutorContext): Promise<DetectedElement[]>;
     /**
      * Takes a prompt that contains a question you want to be answered

package/dist/esm/execution/ui-control-client.js

@@ -131,8 +131,8 @@ export class UiControlClient extends ApiCommands {
             return workspaceAIElementCollection.getByNames(names);
         });
     }
-    fluentCommandExecutor(instructionString_1) {
-        return __awaiter(this, arguments, void 0, function* (instructionString, context = { customElementsJson: [], aiElementNames: [] }) {
+    fluentCommandExecutor(instructionString_1, modelComposition_1) {
+        return __awaiter(this, arguments, void 0, function* (instructionString, modelComposition, context = { customElementsJson: [], aiElementNames: [] }) {
             const aiElements = yield this.getAIElementsByNames(context.aiElementNames);
             const instruction = yield this.buildInstruction(instructionString, [
                 ...context.customElementsJson,
@@ -141,7 +141,7 @@ export class UiControlClient extends ApiCommands {
             logger.debug(instruction);
             try {
                 yield this.stepReporter.resetStep(instruction);
-                yield this.executionRuntime.executeInstruction(instruction);
+                yield this.executionRuntime.executeInstruction(instruction, modelComposition);
                 yield this.afterCommandExecution(instruction);
                 return yield Promise.resolve();
             }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "askui",
-    "version": "0.20.9",
+    "version": "0.21.0",
     "license": "MIT",
     "author": "askui GmbH <info@askui.com> (http://www.askui.com/)",
     "description": "Reliable, automated end-to-end-testing that depends on what is shown on your screen instead of the technology you are running on",