askui 0.18.0 → 0.19.0

package/dist/cjs/core/ai-element/ai-element-collection.d.ts

@@ -0,0 +1,10 @@
+import { AIElement } from './ai-element';
+import { CustomElementJson } from '../model/custom-element-json';
+export declare class AIElementCollection {
+    private elements;
+    static AI_ELEMENT_FOLDER: string;
+    constructor(elements: AIElement[]);
+    static collectForWorkspaceId(workspaceId: string | undefined): Promise<AIElementCollection>;
+    getByName(name: string): CustomElementJson[];
+    getByNames(names: string[]): CustomElementJson[];
+}

package/dist/cjs/core/ai-element/ai-element-collection.js

@@ -0,0 +1,78 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AIElementCollection = void 0;
+const os_1 = __importDefault(require("os"));
+const path_1 = __importDefault(require("path"));
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const ai_element_1 = require("./ai-element");
+const lib_1 = require("../../lib");
+const ai_element_error_1 = require("./ai-element-error");
+class AIElementCollection {
+    constructor(elements) {
+        this.elements = elements;
+    }
+    static collectForWorkspaceId(workspaceId) {
+        return __awaiter(this, void 0, void 0, function* () {
+            lib_1.logger.debug(`Collecting AIElements for workspace '${workspaceId}' ...`);
+            if (workspaceId === undefined) {
+                throw new ai_element_error_1.AIElementError("Value of 'workspaceId' must be defined.");
+            }
+            const workspaceAIElementFolder = path_1.default.join(AIElementCollection.AI_ELEMENT_FOLDER, workspaceId);
+            if (!(yield fs_extra_1.default.pathExists(workspaceAIElementFolder))) {
+                throw new ai_element_error_1.AIElementError(`Missing AIElement folder for workspace '${workspaceId}' at '${workspaceAIElementFolder}'.`);
+            }
+            const files = yield fs_extra_1.default.readdir(workspaceAIElementFolder);
+            if (files.length === 0) {
+                throw new ai_element_error_1.AIElementError(`'${workspaceAIElementFolder}' is empty. No AIElement files found for workspace '${workspaceId}'.`);
+            }
+            const aiElements = yield Promise.all(files
+                .filter((file) => path_1.default.extname(file) === '.json')
+                .map((file) => __awaiter(this, void 0, void 0, function* () {
+                const jsonFile = path_1.default.join(workspaceAIElementFolder, file);
+                const baseName = path_1.default.basename(jsonFile, '.json');
+                const pngFile = path_1.default.join(workspaceAIElementFolder, `${baseName}.png`);
+                if (yield fs_extra_1.default.pathExists(pngFile)) {
+                    const metadata = JSON.parse(yield fs_extra_1.default.readFile(jsonFile, 'utf-8'));
+                    return ai_element_1.AIElement.fromJson(metadata, pngFile);
+                }
+                return null;
+            })));
+            const validAIElements = aiElements.filter((element) => element !== null);
+            if (validAIElements.length === 0) {
+                throw new ai_element_error_1.AIElementError(`No AIElement files found for workspace '${workspaceId}' at '${workspaceAIElementFolder}'.`);
+            }
+            return new AIElementCollection(validAIElements);
+        });
+    }
+    getByName(name) {
+        if (name === '') {
+            throw new ai_element_error_1.AIElementError("Parameter 'name' must be non-empty. This might be due to corrupted metadata.");
+        }
+        lib_1.logger.debug(`Getting all CustomElementJson with the name '${name}' ...`);
+        const elements = this.elements.filter((element) => element.hasName(name));
+        if (elements.length === 0) {
+            throw new ai_element_error_1.AIElementError(`No AIElement with the name '${name}' was found.`);
+        }
+        return elements.map((element) => element.toCustomElement());
+    }
+    getByNames(names) {
+        if (names.length === 0) {
+            return [];
+        }
+        return names.flatMap((name) => this.getByName(name));
+    }
+}
+exports.AIElementCollection = AIElementCollection;
+AIElementCollection.AI_ELEMENT_FOLDER = path_1.default.join(os_1.default.homedir(), '.askui', 'SnippingTool', 'AIElement');

package/dist/cjs/core/ai-element/ai-element-error.d.ts

@@ -0,0 +1,2 @@
+export declare class AIElementError extends Error {
+}

package/dist/cjs/core/ai-element/ai-element-error.js

@@ -0,0 +1,6 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AIElementError = void 0;
+class AIElementError extends Error {
+}
+exports.AIElementError = AIElementError;

package/dist/cjs/core/ai-element/ai-element.d.ts

@@ -0,0 +1,27 @@
+import { CustomElementJson } from '../model/custom-element-json';
+interface AIElementJson {
+    version: number;
+    name: string;
+    image?: {
+        mask?: {
+            x: number;
+            y: number;
+        }[];
+    };
+}
+declare class AIElement {
+    name: string;
+    imagePath: string;
+    mask?: {
+        x: number;
+        y: number;
+    }[] | undefined;
+    constructor(name: string, imagePath: string, mask?: {
+        x: number;
+        y: number;
+    }[] | undefined);
+    static fromJson(json: AIElementJson, imagePath: string): AIElement;
+    toCustomElement(): CustomElementJson;
+    hasName(name: string): boolean;
+}
+export { AIElement, AIElementJson };

package/dist/cjs/core/ai-element/ai-element.js

@@ -0,0 +1,31 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AIElement = void 0;
+const lib_1 = require("../../lib");
+const ai_element_error_1 = require("./ai-element-error");
+class AIElement {
+    constructor(name, imagePath, mask) {
+        this.name = name;
+        this.imagePath = imagePath;
+        this.mask = mask;
+    }
+    static fromJson(json, imagePath) {
+        var _a;
+        if (json.version === 1) {
+            return new AIElement(json.name, imagePath, (_a = json.image) === null || _a === void 0 ? void 0 : _a.mask);
+        }
+        throw new ai_element_error_1.AIElementError(`Unsupported AIElement version '${json.version}'.`);
+    }
+    toCustomElement() {
+        lib_1.logger.debug('Converting AIElement to CustomElementJson.');
+        return {
+            customImage: this.imagePath,
+            mask: this.mask,
+            name: this.name,
+        };
+    }
+    hasName(name) {
+        return this.name === name;
+    }
+}
+exports.AIElement = AIElement;

package/dist/cjs/core/model/custom-element-json.d.ts

@@ -1,17 +1,16 @@
 /**
  * Defines a 'custom element'. This is a UI element which is defined by
  * providing an image and other parameters such as degree of rotation.
- * It allows filtering for a UI element that is not recognized
- * by our machine learning models by default.
- * It can also be used for pixel assertions of elements using classical
- * [template matching](https://en.wikipedia.org/wiki/Template_matching).
+ * It allows filtering for a UI element based on an image instead of using
+ * text or element descriptions like `button().withText('Submit')` in
+ * `await aui.click().button().withText('Submit').exec()`.
  *
- * **Important:** The `CustomElementJson` needs to capture as accurately as possible
- * what the custom element looks like during test execution as otherwise
- * our machine learning models cannot find it, even with the additional data
- * provided. This is especially true for the resolution used while cropping
- * the `CustomElementJson.customImage` which should match the resolution during
- * test execution.
+ * **Important:** The `CustomElementJson` needs to capture as accurately as
+ * possible what the custom element looks like during test execution as
+ * otherwise our machine learning models cannot find it, even with the
+ * additional data provided. This is especially true for the resolution used
+ * while cropping the `CustomElementJson.customImage` which should be as
+ * similar as possible to the resolution during test execution.
  *
  * Rotated custom elements can be filtered for using
  * `CustomElementJson.rotationDegreePerStep`.
@@ -46,8 +45,22 @@ export interface CustomElementJson {
      * probably not what you want) and `1.0` (= elements need to look exactly
      * like defined by `CustomElementJson` which is unlikely to be achieved
      * as even minor differences count). Defaults to `0.9`.
+     *
+     * **Important**: The `threshold` impacts the prediction quality.
      */
     threshold?: number | undefined;
+    /**
+     * A threshold for when to stop searching for UI elements similar to the
+     * custom element. As soon as UI elements have been found that are at least
+     * as similar as the `stopThreshold`, the search is going to stop. After that
+     * elements are filtered using the `threshold`. Because of that the
+     * `stopThreshold` should be greater than or equal to `threshold`. It is
+     * primarily to be used as a speed improvement (by lowering the value).
+     * Takes values between `0.0` and `1.0`. Defaults to `0.9`.
+     *
+     * **Important**: The `stopThreshold` impacts the prediction speed.
+     */
+    stopThreshold?: number | undefined;
     /**
      * A step size in rotation degree. Rotates the custom image by
      * `rotationDegreePerStep` until 360° is exceeded. Range is between
@@ -60,13 +73,17 @@ export interface CustomElementJson {
     /**
      * A color compare style. Allows matching a custom element by color, e.g.,
      * instead of filtering for all icons (blue and green ones),
-     * only the green ones captured by `customImage` are filtered for using 'RGB'.
-     * Defaults to 'grayscale'.
+     * only the green ones captured by `customImage` are filtered for using
+     * 'RGB'. Defaults to 'grayscale'.
      *
-     * **Important**: This increases the prediction time quite a bit. So only use
-     * it when absolutely necessary.
+     * **Important**: The `imageCompareFormat` impacts the prediction time as
+     * well as quality. Although this highly depends on the use case, the other
+     * parameters and the actual scene in which to find the UI element, as a
+     * rule of thumb, 'edges' is likely to be a bit faster than 'grayscale' and
+     * 'grayscale' is likely to be a bit faster than 'RGB'. For quality it is
+     * most often the other way around.
      */
-    imageCompareFormat?: 'RGB' | 'grayscale' | undefined;
+    imageCompareFormat?: 'RGB' | 'grayscale' | 'edges' | undefined;
     /** A polygon to match only a certain area of the custom element. */
     mask?: ({
         x: number;

package/dist/cjs/core/model/custom-element.d.ts

@@ -3,14 +3,15 @@ export declare class CustomElement implements CustomElementJson {
     customImage: string;
     name?: string | undefined;
     threshold?: number | undefined;
+    stopThreshold?: number | undefined;
     rotationDegreePerStep?: number | undefined;
-    imageCompareFormat?: "RGB" | "grayscale" | undefined;
+    imageCompareFormat?: "RGB" | "grayscale" | "edges" | undefined;
     mask?: {
         x: number;
         y: number;
     }[] | undefined;
     private static schema;
-    constructor(customImage: string, name?: string | undefined, threshold?: number | undefined, rotationDegreePerStep?: number | undefined, imageCompareFormat?: "RGB" | "grayscale" | undefined, mask?: {
+    constructor(customImage: string, name?: string | undefined, threshold?: number | undefined, stopThreshold?: number | undefined, rotationDegreePerStep?: number | undefined, imageCompareFormat?: "RGB" | "grayscale" | "edges" | undefined, mask?: {
         x: number;
         y: number;
     }[] | undefined);

package/dist/cjs/core/model/custom-element.js

@@ -13,10 +13,11 @@ exports.CustomElement = void 0;
 const yup_1 = require("yup");
 const base_64_image_1 = require("../../utils/base_64_image/base-64-image");
 class CustomElement {
-    constructor(customImage, name, threshold, rotationDegreePerStep, imageCompareFormat, mask) {
+    constructor(customImage, name, threshold, stopThreshold, rotationDegreePerStep, imageCompareFormat, mask) {
         this.customImage = customImage;
         this.name = name;
         this.threshold = threshold;
+        this.stopThreshold = stopThreshold;
         this.rotationDegreePerStep = rotationDegreePerStep;
         this.imageCompareFormat = imageCompareFormat;
         this.mask = mask;
@@ -35,7 +36,7 @@ class CustomElement {
         });
     }
     static fromJson(ceJson) {
-        return new CustomElement(ceJson.customImage, ceJson.name, ceJson.threshold, ceJson.rotationDegreePerStep, ceJson.imageCompareFormat, ceJson.mask);
+        return new CustomElement(ceJson.customImage, ceJson.name, ceJson.threshold, ceJson.stopThreshold, ceJson.rotationDegreePerStep, ceJson.imageCompareFormat, ceJson.mask);
     }
     validate() {
         try {
@@ -50,5 +51,6 @@ exports.CustomElement = CustomElement;
 CustomElement.schema = (0, yup_1.object)({
     mask: (0, yup_1.array)().optional().min(3, 'mask must contain at least 3 points'),
     rotationDegreePerStep: (0, yup_1.number)().optional().min(0).lessThan(360),
+    stopThreshold: (0, yup_1.number)().optional().min(0).max(1),
     threshold: (0, yup_1.number)().optional().min(0).max(1),
 });

package/dist/cjs/core/reporting/step-reporter.js

@@ -58,37 +58,46 @@ class StepReporter {
     onStepBegin(snapshot) {
         return __awaiter(this, void 0, void 0, function* () {
             if (this._currentStep === undefined) {
-                throw new Error('Cannot begin step if step is undefined.');
+                logger_1.logger.error('Cannot begin step if step is undefined.');
+                return Promise.resolve();
             }
             if (this._currentStep.status !== 'pending') {
-                throw new Error('Cannot begin step that is not pending.');
+                logger_1.logger.error('Cannot begin step that is not pending.');
+                return Promise.resolve();
             }
             this._currentStep = this._currentStep.onBegin(snapshot);
             this.enqueueReporterCalls('onStepBegin', this._currentStep);
+            return Promise.resolve();
         });
     }
     onStepRetry(snapshot, error) {
         return __awaiter(this, void 0, void 0, function* () {
             if (this._currentStep === undefined) {
-                throw new Error('Cannot retry step if step is undefined.');
+                logger_1.logger.error('Cannot retry step if step is undefined.');
+                return Promise.resolve();
             }
             if (this._currentStep.status !== 'running') {
-                throw new Error('Cannot retry step that has not been running.');
+                logger_1.logger.error('Cannot retry step that has not been running.');
+                return Promise.resolve();
             }
             this._currentStep = this._currentStep.onRetry(snapshot, error);
             this.enqueueReporterCalls('onStepRetry', this._currentStep);
+            return Promise.resolve();
         });
     }
     onStepEnd(snapshot, error) {
         return __awaiter(this, void 0, void 0, function* () {
             if (this._currentStep === undefined) {
-                throw new Error('Cannot end step if step is undefined.');
+                logger_1.logger.error('Cannot end step if step is undefined.');
+                return Promise.resolve();
             }
             if (this._currentStep.status !== 'running') {
-                throw new Error('Cannot end step that has not been running.');
+                logger_1.logger.error('Cannot end step that has not been running.');
+                return Promise.resolve();
             }
             this._currentStep = this._currentStep.onEnd(snapshot, error);
             this.enqueueReporterCalls('onStepEnd', this._currentStep);
+            return Promise.resolve();
         });
     }
 }