askui 0.18.1 → 0.19.0

package/dist/esm/core/ai-element/ai-element.js

@@ -0,0 +1,28 @@
+import { logger } from '../../lib';
+import { AIElementError } from './ai-element-error';
+class AIElement {
+    constructor(name, imagePath, mask) {
+        this.name = name;
+        this.imagePath = imagePath;
+        this.mask = mask;
+    }
+    static fromJson(json, imagePath) {
+        var _a;
+        if (json.version === 1) {
+            return new AIElement(json.name, imagePath, (_a = json.image) === null || _a === void 0 ? void 0 : _a.mask);
+        }
+        throw new AIElementError(`Unsupported AIElement version '${json.version}'.`);
+    }
+    toCustomElement() {
+        logger.debug('Converting AIElement to CustomElementJson.');
+        return {
+            customImage: this.imagePath,
+            mask: this.mask,
+            name: this.name,
+        };
+    }
+    hasName(name) {
+        return this.name === name;
+    }
+}
+export { AIElement };

package/dist/esm/execution/dsl.d.ts

@@ -8,6 +8,10 @@ export declare type ANDROID_KEY = 'home' | 'back' | 'call' | 'endcall' | '0' | '
 export declare type MODIFIER_KEY = 'command' | 'alt' | 'control' | 'shift' | 'right_shift';
 export declare type COLOR = 'black' | 'white' | 'red' | 'green' | 'yellow green' | 'orange' | 'yellow' | 'purple' | 'pink' | 'gray' | 'lime green' | 'royal blue';
 export declare type PC_AND_MODIFIER_KEY = 'command' | 'alt' | 'control' | 'shift' | 'right_shift' | 'backspace' | 'delete' | 'enter' | 'tab' | 'escape' | 'up' | 'down' | 'right' | 'left' | 'home' | 'end' | 'pageup' | 'pagedown' | 'f1' | 'f2' | 'f3' | 'f4' | 'f5' | 'f6' | 'f7' | 'f8' | 'f9' | 'f10' | 'f11' | 'f12' | 'space' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '!' | '"' | '#' | '$' | '%' | '&' | "'" | '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | ':' | ';' | '<' | '=' | '>' | '?' | '@' | '[' | '\\' | ']' | '^' | '_' | '`' | '{' | '|' | '}' | '~ ';
+export interface CommandExecutorContext {
+    customElementsJson: CustomElementJson[];
+    aiElementNames: string[];
+}
 declare abstract class FluentBase {
     protected prev?: FluentBase | undefined;
     constructor(prev?: FluentBase | undefined);
@@ -202,6 +206,14 @@ export declare class FluentFilters extends FluentBase {
      * @return {FluentFiltersOrRelations}
      */
     customElement(customElement: CustomElementJson): FluentFiltersOrRelations;
+    /**
+     * Detects an AI Element created with the workflow creator.
+     *
+     * @param {string} aiElementName - Name of the AI Element.
+     *
+     * @return {FluentFiltersOrRelations}
+     */
+    aiElement(aiElementName: string): FluentFiltersOrRelations;
     /**
      * Filters for a UI element 'image'.
      *
@@ -354,7 +366,17 @@ export declare class FluentFilters extends FluentBase {
      *
      * **Examples:**
      * ```typescript
-     * await aui.click().matching('a mask on purple background and a firefox logo').exec()
+     * // Select the black sneaker from a bunch of sneakers
+     * await aui.click().element().matching('a black sneaker shoe').exec();
+     *
+     * // Select an image that has text in it
+     * await aui.click().element().matching('has Burger King in it').exec();
+     * await aui.click().element().matching('has adidas in it').exec();
+     *
+     * // Target a logo/image by describing it
+     * await aui.click().element().matching('a mask on purple background and a firefox logo').exec();
+     * await aui.click().element().matching('logo looking like an apple with one bite bitten off').exec();
+     * await aui.click().element().matching('logo looking like a seashell').exec();
      * ```
      *
      * @param {string} text - A description of the target element.
@@ -827,6 +849,14 @@ export declare class FluentFiltersCondition extends FluentBase {
      * @return {FluentFiltersOrRelationsCondition}
      */
     customElement(customElement: CustomElementJson): FluentFiltersOrRelationsCondition;
+    /**
+     * Detects an AI Element created with the workflow creator.
+     *
+     * @param {string} aiElementName - Name of the AI Element.
+     *
+     * @return {FluentFiltersOrRelationsCondition}
+     */
+    aiElement(aiElementName: string): FluentFiltersOrRelationsCondition;
     /**
      * Filters for a UI element 'image'.
      *
@@ -979,7 +1009,17 @@ export declare class FluentFiltersCondition extends FluentBase {
      *
      * **Examples:**
      * ```typescript
-     * await aui.click().matching('a mask on purple background and a firefox logo').exec()
+     * // Select the black sneaker from a bunch of sneakers
+     * await aui.click().element().matching('a black sneaker shoe').exec();
+     *
+     * // Select an image that has text in it
+     * await aui.click().element().matching('has Burger King in it').exec();
+     * await aui.click().element().matching('has adidas in it').exec();
+     *
+     * // Target a logo/image by describing it
+     * await aui.click().element().matching('a mask on purple background and a firefox logo').exec();
+     * await aui.click().element().matching('logo looking like an apple with one bite bitten off').exec();
+     * await aui.click().element().matching('logo looking like a seashell').exec();
      * ```
      *
      * @param {string} text - A description of the target element.
@@ -1770,7 +1810,7 @@ export declare abstract class FluentCommand extends FluentBase {
      * @return {Exec}
      */
     pressAndroidKey(key: ANDROID_KEY): Exec;
-    abstract fluentCommandExecutor(instruction: string, customElements: CustomElementJson[]): Promise<void>;
+    abstract fluentCommandExecutor(instruction: string, context: CommandExecutorContext): Promise<void>;
 }
 export interface ExecutableGetter {
     exec(): Promise<DetectedElement[]>;
@@ -1955,6 +1995,14 @@ export declare class FluentFiltersGetter extends FluentBase {
      * @return {FluentFiltersOrRelationsGetter}
      */
     customElement(customElement: CustomElementJson): FluentFiltersOrRelationsGetter;
+    /**
+     * Detects an AI Element created with the workflow creator.
+     *
+     * @param {string} aiElementName - Name of the AI Element.
+     *
+     * @return {FluentFiltersOrRelationsGetter}
+     */
+    aiElement(aiElementName: string): FluentFiltersOrRelationsGetter;
     /**
      * Filters for a UI element 'image'.
      *
@@ -2107,7 +2155,17 @@ export declare class FluentFiltersGetter extends FluentBase {
      *
      * **Examples:**
      * ```typescript
-     * await aui.click().matching('a mask on purple background and a firefox logo').exec()
+     * // Select the black sneaker from a bunch of sneakers
+     * await aui.click().element().matching('a black sneaker shoe').exec();
+     *
+     * // Select an image that has text in it
+     * await aui.click().element().matching('has Burger King in it').exec();
+     * await aui.click().element().matching('has adidas in it').exec();
+     *
+     * // Target a logo/image by describing it
+     * await aui.click().element().matching('a mask on purple background and a firefox logo').exec();
+     * await aui.click().element().matching('logo looking like an apple with one bite bitten off').exec();
+     * await aui.click().element().matching('logo looking like a seashell').exec();
      * ```
      *
      * @param {string} text - A description of the target element.
@@ -2420,20 +2478,72 @@ export declare abstract class Getter extends FluentCommand {
      * ```typescript
      * const text = await aui.get().text('Sign').exec();
      * console.log(text);
+     *
+     * // Console output
+     * [
+     *   DetectedElement {
+     *     name: 'TEXT',
+     *     text: 'Sign In',
+     *     bndbox: BoundingBox {
+     *       xmin: 1128.2720982142857,
+     *       ymin: 160.21332310267857,
+     *       xmax: 1178.8204241071428,
+     *       ymax: 180.83512834821428
+     *     }
+     *   }
+     * ]
      * ```
-     * ```text
-     *  console output: [
+     *
+     * // *************************************************** //
+     * // Examples on how to work with the returned elements  //
+     * // *************************************************** //
+     * const texts = await aui.get().text().below().textfield().exec();
+     *
+     * // We can get a lot of elements this way
+     * console.log(texts);
+     *
+     * // Console output
+     * [
      *   DetectedElement {
-     *      name: 'TEXT',
-     *      text: 'Sign In',
-     *      bndbox: BoundingBox {
-     *         xmin: 1128.2720982142857,
-     *         ymin: 160.21332310267857,
-     *         xmax: 1178.8204241071428,
-     *         ymax: 180.83512834821428
-     *      }
-     *    }
-     *  ]
+     *     name: 'TEXT',
+     *     text: 'Sign In',
+     *     bndbox: BoundingBox {
+     *       xmin: 1128.2720982142857,
+     *       ymin: 160.21332310267857,
+     *       xmax: 1178.8204241071428,
+     *       ymax: 180.83512834821428
+     *     },
+     *   },
+     *   DetectedElement {
+     *     name: 'TEXT',
+     *     text: 'Login',
+     *     bndbox: BoundingBox {
+     *       xmin: 250.8204241071428,
+     *       ymin: 300.21332310267857,
+     *       xmax: 450.6304241071428,
+     *       ymax: 950.47812834821428
+     *     },
+     *   },
+     *   ... 10 more items
+     * ]
+     *
+     * // Extract the FIRST element
+     * // Arrays start with index 0!
+     * const firstTextElement = texts[0];
+     * const textOfFirstElement = firstElement.text;
+     *
+     * console.log(textOfFirstElement);
+     *
+     * // Console output
+     * Sign In
+     *
+     * // Log the text of the SECOND element
+     * // with shorter code
+     * const texts = await aui.get().text().below().textfield().exec();
+     * console.log(texts[1].text)
+     *
+     * // Console output
+     * Login
      * ```
      *
      * @return {FluentFiltersGetter}
@@ -2479,7 +2589,7 @@ export declare abstract class Getter extends FluentCommand {
      * @return {ExecGetter}
      */
     getAll(): ExecGetter;
-    abstract getterExecutor(instruction: string, customElements: CustomElementJson[]): Promise<DetectedElement[]>;
+    abstract getterExecutor(instruction: string, context: CommandExecutorContext): Promise<DetectedElement[]>;
 }
 export declare abstract class ApiCommands extends Getter {
 }

package/dist/esm/execution/dsl.js

@@ -28,7 +28,11 @@ class FluentBase {
         if (this instanceof FluentCommand) {
             const fluentCommand = this;
             const customElements = newParamsList.has('customElement') ? newParamsList.get('customElement') : [];
-            return fluentCommand.fluentCommandExecutor(newCurrentInstruction.trim(), customElements);
+            const aiElementNames = newParamsList.has('aiElementName') ? newParamsList.get('aiElementName') : [];
+            return fluentCommand.fluentCommandExecutor(newCurrentInstruction.trim(), {
+                customElementsJson: customElements,
+                aiElementNames,
+            });
         }
         if (!this.prev) {
             throw new Error('Prev element not defined');
@@ -41,7 +45,11 @@ class FluentBase {
         if (this instanceof Getter) {
             const getter = this;
             const customElements = newParamsList.has('customElement') ? newParamsList.get('customElement') : [];
-            return getter.getterExecutor(newCurrentInstruction.trim(), customElements);
+            const aiElementNames = newParamsList.has('aiElementName') ? newParamsList.get('aiElementName') : [];
+            return getter.getterExecutor(newCurrentInstruction.trim(), {
+                customElementsJson: customElements,
+                aiElementNames,
+            });
         }
         if (!this.prev) {
             throw new Error('Prev element not defined');
@@ -303,6 +311,23 @@ export class FluentFilters extends FluentBase {
         this._params.set('customElement', customElement);
         return new FluentFiltersOrRelations(this);
     }
+    /**
+     * Detects an AI Element created with the workflow creator.
+     *
+     * @param {string} aiElementName - Name of the AI Element.
+     *
+     * @return {FluentFiltersOrRelations}
+     */
+    aiElement(aiElementName) {
+        this._textStr = '';
+        this._textStr += 'ai';
+        this._textStr += ' element';
+        this._textStr += ' with';
+        this._textStr += ' name';
+        this._textStr += ` ${Separators.STRING}${aiElementName}${Separators.STRING}`;
+        this._params.set('aiElementName', aiElementName);
+        return new FluentFiltersOrRelations(this);
+    }
     /**
      * Filters for a UI element 'image'.
      *
@@ -491,7 +516,17 @@ export class FluentFilters extends FluentBase {
      *
      * **Examples:**
      * ```typescript
-     * await aui.click().matching('a mask on purple background and a firefox logo').exec()
+     * // Select the black sneaker from a bunch of sneakers
+     * await aui.click().element().matching('a black sneaker shoe').exec();
+     *
+     * // Select an image that has text in it
+     * await aui.click().element().matching('has Burger King in it').exec();
+     * await aui.click().element().matching('has adidas in it').exec();
+     *
+     * // Target a logo/image by describing it
+     * await aui.click().element().matching('a mask on purple background and a firefox logo').exec();
+     * await aui.click().element().matching('logo looking like an apple with one bite bitten off').exec();
+     * await aui.click().element().matching('logo looking like a seashell').exec();
      * ```
      *
      * @param {string} text - A description of the target element.
@@ -1093,6 +1128,23 @@ export class FluentFiltersCondition extends FluentBase {
         this._params.set('customElement', customElement);
         return new FluentFiltersOrRelationsCondition(this);
     }
+    /**
+     * Detects an AI Element created with the workflow creator.
+     *
+     * @param {string} aiElementName - Name of the AI Element.
+     *
+     * @return {FluentFiltersOrRelationsCondition}
+     */
+    aiElement(aiElementName) {
+        this._textStr = '';
+        this._textStr += 'ai';
+        this._textStr += ' element';
+        this._textStr += ' with';
+        this._textStr += ' name';
+        this._textStr += ` ${Separators.STRING}${aiElementName}${Separators.STRING}`;
+        this._params.set('aiElementName', aiElementName);
+        return new FluentFiltersOrRelationsCondition(this);
+    }
     /**
      * Filters for a UI element 'image'.
      *
@@ -1281,7 +1333,17 @@ export class FluentFiltersCondition extends FluentBase {
      *
      * **Examples:**
      * ```typescript
-     * await aui.click().matching('a mask on purple background and a firefox logo').exec()
+     * // Select the black sneaker from a bunch of sneakers
+     * await aui.click().element().matching('a black sneaker shoe').exec();
+     *
+     * // Select an image that has text in it
+     * await aui.click().element().matching('has Burger King in it').exec();
+     * await aui.click().element().matching('has adidas in it').exec();
+     *
+     * // Target a logo/image by describing it
+     * await aui.click().element().matching('a mask on purple background and a firefox logo').exec();
+     * await aui.click().element().matching('logo looking like an apple with one bite bitten off').exec();
+     * await aui.click().element().matching('logo looking like a seashell').exec();
      * ```
      *
      * @param {string} text - A description of the target element.
@@ -2562,6 +2624,23 @@ export class FluentFiltersGetter extends FluentBase {
         this._params.set('customElement', customElement);
         return new FluentFiltersOrRelationsGetter(this);
     }
+    /**
+     * Detects an AI Element created with the workflow creator.
+     *
+     * @param {string} aiElementName - Name of the AI Element.
+     *
+     * @return {FluentFiltersOrRelationsGetter}
+     */
+    aiElement(aiElementName) {
+        this._textStr = '';
+        this._textStr += 'ai';
+        this._textStr += ' element';
+        this._textStr += ' with';
+        this._textStr += ' name';
+        this._textStr += ` ${Separators.STRING}${aiElementName}${Separators.STRING}`;
+        this._params.set('aiElementName', aiElementName);
+        return new FluentFiltersOrRelationsGetter(this);
+    }
     /**
      * Filters for a UI element 'image'.
      *
@@ -2750,7 +2829,17 @@ export class FluentFiltersGetter extends FluentBase {
      *
      * **Examples:**
      * ```typescript
-     * await aui.click().matching('a mask on purple background and a firefox logo').exec()
+     * // Select the black sneaker from a bunch of sneakers
+     * await aui.click().element().matching('a black sneaker shoe').exec();
+     *
+     * // Select an image that has text in it
+     * await aui.click().element().matching('has Burger King in it').exec();
+     * await aui.click().element().matching('has adidas in it').exec();
+     *
+     * // Target a logo/image by describing it
+     * await aui.click().element().matching('a mask on purple background and a firefox logo').exec();
+     * await aui.click().element().matching('logo looking like an apple with one bite bitten off').exec();
+     * await aui.click().element().matching('logo looking like a seashell').exec();
      * ```
      *
      * @param {string} text - A description of the target element.
@@ -3123,20 +3212,72 @@ export class Getter extends FluentCommand {
      * ```typescript
      * const text = await aui.get().text('Sign').exec();
      * console.log(text);
+     *
+     * // Console output
+     * [
+     *   DetectedElement {
+     *     name: 'TEXT',
+     *     text: 'Sign In',
+     *     bndbox: BoundingBox {
+     *       xmin: 1128.2720982142857,
+     *       ymin: 160.21332310267857,
+     *       xmax: 1178.8204241071428,
+     *       ymax: 180.83512834821428
+     *     }
+     *   }
+     * ]
      * ```
-     * ```text
-     *  console output: [
+     *
+     * // *************************************************** //
+     * // Examples on how to work with the returned elements  //
+     * // *************************************************** //
+     * const texts = await aui.get().text().below().textfield().exec();
+     *
+     * // We can get a lot of elements this way
+     * console.log(texts);
+     *
+     * // Console output
+     * [
      *   DetectedElement {
-     *      name: 'TEXT',
-     *      text: 'Sign In',
-     *      bndbox: BoundingBox {
-     *         xmin: 1128.2720982142857,
-     *         ymin: 160.21332310267857,
-     *         xmax: 1178.8204241071428,
-     *         ymax: 180.83512834821428
-     *      }
-     *    }
-     *  ]
+     *     name: 'TEXT',
+     *     text: 'Sign In',
+     *     bndbox: BoundingBox {
+     *       xmin: 1128.2720982142857,
+     *       ymin: 160.21332310267857,
+     *       xmax: 1178.8204241071428,
+     *       ymax: 180.83512834821428
+     *     },
+     *   },
+     *   DetectedElement {
+     *     name: 'TEXT',
+     *     text: 'Login',
+     *     bndbox: BoundingBox {
+     *       xmin: 250.8204241071428,
+     *       ymin: 300.21332310267857,
+     *       xmax: 450.6304241071428,
+     *       ymax: 950.47812834821428
+     *     },
+     *   },
+     *   ... 10 more items
+     * ]
+     *
+     * // Extract the FIRST element
+     * // Arrays start with index 0!
+     * const firstTextElement = texts[0];
+     * const textOfFirstElement = firstElement.text;
+     *
+     * console.log(textOfFirstElement);
+     *
+     * // Console output
+     * Sign In
+     *
+     * // Log the text of the SECOND element
+     * // with shorter code
+     * const texts = await aui.get().text().below().textfield().exec();
+     * console.log(texts[1].text)
+     *
+     * // Console output
+     * Login
      * ```
      *
      * @return {FluentFiltersGetter}

package/dist/esm/execution/index.d.ts

@@ -1 +1 @@
-export { UiControlClient, RelationsForConvenienceMethods } from './ui-control-client';
+export * from './ui-control-client';

package/dist/esm/execution/index.js

@@ -1 +1 @@
-export { UiControlClient } from './ui-control-client';
+export * from './ui-control-client';

package/dist/esm/execution/ui-control-client-dependency-builder.d.ts

@@ -8,6 +8,7 @@ export declare class UiControlClientDependencyBuilder {
     static build(clientArgs: ClientArgsWithDefaults): Promise<{
         executionRuntime: ExecutionRuntime;
         stepReporter: StepReporter;
+        workspaceId: string | undefined;
     }>;
     static getClientArgsWithDefaults(clientArgs: ClientArgs): Promise<ClientArgsWithDefaults>;
 }

package/dist/esm/execution/ui-control-client-dependency-builder.js

@@ -37,13 +37,16 @@ export class UiControlClientDependencyBuilder {
         return new UiControllerClient(clientArgs.uiControllerUrl);
     }
     static build(clientArgs) {
+        var _a;
         return __awaiter(this, void 0, void 0, function* () {
             const uiControllerClient = UiControlClientDependencyBuilder.buildUiControllerClient(clientArgs);
             const inferenceClient = yield UiControlClientDependencyBuilder.buildInferenceClient(clientArgs);
             const stepReporter = new StepReporter(clientArgs.reporter);
+            const workspaceId = (_a = clientArgs.credentials) === null || _a === void 0 ? void 0 : _a.workspaceId;
             return {
                 executionRuntime: new ExecutionRuntime(uiControllerClient, inferenceClient, stepReporter),
                 stepReporter,
+                workspaceId,
             };
         });
     }