askui 0.18.0 → 0.19.0

package/dist/cjs/main.d.ts

@@ -1,5 +1,5 @@
 export { UiController } from './lib';
-export { UiControlClient, RelationsForConvenienceMethods } from './execution';
+export * from './execution';
 export { Instruction, Reporter, ReporterConfig, Snapshot, SnapshotDetailLevel, Step, StepStatus, StepStatusEnd, } from './core/reporting';
 export { Annotation } from './core/annotation/annotation';
 export { DetectedElement } from './core/model/annotation-result/detected-element';

package/dist/cjs/main.js

@@ -1,10 +1,19 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.LogLevels = exports.DetectedElement = exports.Annotation = exports.UiControlClient = exports.UiController = void 0;
+exports.LogLevels = exports.DetectedElement = exports.Annotation = exports.UiController = void 0;
 var lib_1 = require("./lib");
 Object.defineProperty(exports, "UiController", { enumerable: true, get: function () { return lib_1.UiController; } });
-var execution_1 = require("./execution");
-Object.defineProperty(exports, "UiControlClient", { enumerable: true, get: function () { return execution_1.UiControlClient; } });
+__exportStar(require("./execution"), exports);
 var annotation_1 = require("./core/annotation/annotation");
 Object.defineProperty(exports, "Annotation", { enumerable: true, get: function () { return annotation_1.Annotation; } });
 var detected_element_1 = require("./core/model/annotation-result/detected-element");

package/dist/cjs/utils/analytics/analytics.d.ts

@@ -1,6 +1,8 @@
 import { AnalyticsInterface } from './analytics-interface';
-import { Context } from '@/execution/context';
+import { Context } from '../../execution/context';
 export declare class Analytics implements AnalyticsInterface {
+    private static clientSessionId;
+    private clientId;
     private userIdentifier;
     getAnalyticsHeaders(context: Context): Promise<Record<string, string>>;
     getAnalyticsCookies(): Promise<Record<string, string>>;

package/dist/cjs/utils/analytics/analytics.js

@@ -14,16 +14,20 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Analytics = void 0;
 const os_1 = __importDefault(require("os"));
+const crypto_1 = __importDefault(require("crypto"));
 const user_identifier_1 = require("./user-identifier");
 const installation_timestamp_1 = require("./installation-timestamp");
 class Analytics {
     constructor() {
+        this.clientId = crypto_1.default.randomUUID();
         this.userIdentifier = new user_identifier_1.UserIdentifier();
     }
     getAnalyticsHeaders(context) {
         return __awaiter(this, void 0, void 0, function* () {
             const userID = yield this.userIdentifier.userId();
             const headers = {
+                'askui-client-id': this.clientId,
+                'askui-client-session-id': Analytics.clientSessionId,
                 'askui-is-ci': String(context.isCi),
                 'askui-user-agent': `os:${os_1.default.platform()};arch:${os_1.default.arch()}`,
                 'askui-user-id': userID,
@@ -45,3 +49,4 @@ class Analytics {
     }
 }
 exports.Analytics = Analytics;
+Analytics.clientSessionId = crypto_1.default.randomUUID();

package/dist/cjs/utils/http/custom-errors/index.js

@@ -8,7 +8,7 @@ const unkown_http_client_error_1 = require("./unkown-http-client-error");
 var general_http_client_error_1 = require("./general-http-client-error");
 Object.defineProperty(exports, "GeneralHttpClientError", { enumerable: true, get: function () { return general_http_client_error_1.GeneralHttpClientError; } });
 function httpClientErrorHandler(responseCode, errorMessage) {
-    const diplayedMessage = `HTTP Status Code: ${responseCode}. Message:${errorMessage}`;
+    const diplayedMessage = `HTTP Status Code: ${responseCode}. Message:\n${errorMessage}`;
     if (responseCode >= 400 && responseCode < 500) {
         if (responseCode === 401 || responseCode === 403) {
             return new authentication_http_client_error_1.AuthenticationHttpClientError(diplayedMessage);

package/dist/cjs/utils/http/http-client-got.js

@@ -18,6 +18,22 @@ const tough_cookie_1 = require("tough-cookie");
 const lib_1 = require("../../lib");
 const credentials_1 = require("./credentials");
 const custom_errors_1 = require("./custom-errors");
+function buildRetryLog(requestUrl, errorCode, statusCode, errorMessage, delayInMs, delayReason, attemptCount) {
+    const failureReasons = [];
+    if (statusCode !== undefined && statusCode >= 400) {
+        failureReasons.push(`status code ${statusCode}`);
+    }
+    if (errorCode !== undefined) {
+        failureReasons.push(`error code ${errorCode}`);
+    }
+    if (failureReasons.length === 0) {
+        failureReasons.push('unknown error');
+    }
+    const requestText = requestUrl ? `Request to ${requestUrl}` : 'Request';
+    return (`${requestText} failed with ${failureReasons.join(', ')}.`
+        + ` Retrying in ${delayInMs} ms... (based on ${delayReason}; attempt ${attemptCount})`
+        + `\nFull message:\n${errorMessage}`);
+}
 class HttpClientGot {
     constructor(token, customHeaders, cookies = {}, proxyAgents) {
         this.token = token;
@@ -33,33 +49,43 @@ class HttpClientGot {
     buildGotExtendOptions(proxyAgents) {
         const gotExtendOptions = {
             retry: {
-                calculateDelay: ({ attemptCount, retryOptions, error, computedValue, }) => {
+                calculateDelay: ({ attemptCount, retryOptions, error, retryAfter, }) => {
                     var _a, _b, _c;
-                    if (attemptCount > retryOptions.limit
-                        || !this.shouldRetryOnError(error)) {
+                    if (!this.shouldRetryOnError(error)) {
+                        return 0;
+                    }
+                    if (attemptCount > retryOptions.limit) {
+                        return 0;
+                    }
+                    const hasMethod = retryOptions.methods.includes(error.options.method);
+                    const hasErrorCode = retryOptions.errorCodes.includes(error.code);
+                    const hasStatusCode = error.response
+                        && retryOptions.statusCodes.includes(error.response.statusCode);
+                    if (!hasMethod || (!hasErrorCode && !hasStatusCode)) {
                         return 0;
                     }
-                    if (error.response !== undefined
-                        && error.response.headers['retry-after'] === undefined) {
-                        lib_1.logger.debug(`Request to ${(_a = error.request) === null || _a === void 0 ? void 0 : _a.requestUrl} failed with status code ${(_b = error.response) === null || _b === void 0 ? void 0 : _b.statusCode}.\n${error.message}\nRetrying... (attempt ${attemptCount})`);
-                        return Math.min(1000 * Math.pow(2, (attemptCount - 1)) + Math.random() * 100, Number.MAX_SAFE_INTEGER);
+                    if (error.response) {
+                        if (retryAfter) {
+                            if (retryOptions.maxRetryAfter === undefined
+                                || retryAfter > retryOptions.maxRetryAfter) {
+                                return 0;
+                            }
+                            lib_1.logger.debug(buildRetryLog((_a = error.request) === null || _a === void 0 ? void 0 : _a.requestUrl, error.code, error.response.statusCode, error.message, retryAfter, 'retry-after header', attemptCount));
+                            return retryAfter;
+                        }
+                        if (error.response.statusCode === 413) {
+                            return 0;
+                        }
                     }
-                    lib_1.logger.debug(`Request to ${(_c = error.request) === null || _c === void 0 ? void 0 : _c.requestUrl} failed.\n${error.message}\nRetrying... (attempt ${attemptCount})`);
-                    return computedValue;
+                    const baseDelayInMs = 1000;
+                    const noiseToPreventCollisions = Math.random() * 100;
+                    const delayInMs = Math.min(Math.pow(2, (attemptCount - 1)) * baseDelayInMs + noiseToPreventCollisions, Number.MAX_SAFE_INTEGER);
+                    lib_1.logger.debug(buildRetryLog((_b = error.request) === null || _b === void 0 ? void 0 : _b.requestUrl, error.code, (_c = error.response) === null || _c === void 0 ? void 0 : _c.statusCode, error.message, delayInMs, 'retry-after header', attemptCount));
+                    return delayInMs;
                 },
-                errorCodes: [
-                    'ETIMEDOUT',
-                    'ECONNRESET',
-                    'EADDRINUSE',
-                    'ECONNREFUSED',
-                    'EPIPE',
-                    'ENOTFOUND',
-                    'ENETUNREACH',
-                    'EAI_AGAIN',
-                ],
                 limit: 5,
+                maxRetryAfter: 10 * 60,
                 methods: ['POST', 'GET', 'PUT', 'HEAD', 'DELETE', 'OPTIONS', 'TRACE'],
-                statusCodes: [408, 413, 429, 500, 502, 503, 504, 521, 522, 524],
             },
         };
         if (proxyAgents) {

package/dist/esm/core/ai-element/ai-element-collection.d.ts

@@ -0,0 +1,10 @@
+import { AIElement } from './ai-element';
+import { CustomElementJson } from '../model/custom-element-json';
+export declare class AIElementCollection {
+    private elements;
+    static AI_ELEMENT_FOLDER: string;
+    constructor(elements: AIElement[]);
+    static collectForWorkspaceId(workspaceId: string | undefined): Promise<AIElementCollection>;
+    getByName(name: string): CustomElementJson[];
+    getByNames(names: string[]): CustomElementJson[];
+}

package/dist/esm/core/ai-element/ai-element-collection.js

@@ -0,0 +1,71 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import os from 'os';
+import path from 'path';
+import fs from 'fs-extra';
+import { AIElement } from './ai-element';
+import { logger } from '../../lib';
+import { AIElementError } from './ai-element-error';
+export class AIElementCollection {
+    constructor(elements) {
+        this.elements = elements;
+    }
+    static collectForWorkspaceId(workspaceId) {
+        return __awaiter(this, void 0, void 0, function* () {
+            logger.debug(`Collecting AIElements for workspace '${workspaceId}' ...`);
+            if (workspaceId === undefined) {
+                throw new AIElementError("Value of 'workspaceId' must be defined.");
+            }
+            const workspaceAIElementFolder = path.join(AIElementCollection.AI_ELEMENT_FOLDER, workspaceId);
+            if (!(yield fs.pathExists(workspaceAIElementFolder))) {
+                throw new AIElementError(`Missing AIElement folder for workspace '${workspaceId}' at '${workspaceAIElementFolder}'.`);
+            }
+            const files = yield fs.readdir(workspaceAIElementFolder);
+            if (files.length === 0) {
+                throw new AIElementError(`'${workspaceAIElementFolder}' is empty. No AIElement files found for workspace '${workspaceId}'.`);
+            }
+            const aiElements = yield Promise.all(files
+                .filter((file) => path.extname(file) === '.json')
+                .map((file) => __awaiter(this, void 0, void 0, function* () {
+                const jsonFile = path.join(workspaceAIElementFolder, file);
+                const baseName = path.basename(jsonFile, '.json');
+                const pngFile = path.join(workspaceAIElementFolder, `${baseName}.png`);
+                if (yield fs.pathExists(pngFile)) {
+                    const metadata = JSON.parse(yield fs.readFile(jsonFile, 'utf-8'));
+                    return AIElement.fromJson(metadata, pngFile);
+                }
+                return null;
+            })));
+            const validAIElements = aiElements.filter((element) => element !== null);
+            if (validAIElements.length === 0) {
+                throw new AIElementError(`No AIElement files found for workspace '${workspaceId}' at '${workspaceAIElementFolder}'.`);
+            }
+            return new AIElementCollection(validAIElements);
+        });
+    }
+    getByName(name) {
+        if (name === '') {
+            throw new AIElementError("Parameter 'name' must be non-empty. This might be due to corrupted metadata.");
+        }
+        logger.debug(`Getting all CustomElementJson with the name '${name}' ...`);
+        const elements = this.elements.filter((element) => element.hasName(name));
+        if (elements.length === 0) {
+            throw new AIElementError(`No AIElement with the name '${name}' was found.`);
+        }
+        return elements.map((element) => element.toCustomElement());
+    }
+    getByNames(names) {
+        if (names.length === 0) {
+            return [];
+        }
+        return names.flatMap((name) => this.getByName(name));
+    }
+}
+AIElementCollection.AI_ELEMENT_FOLDER = path.join(os.homedir(), '.askui', 'SnippingTool', 'AIElement');

package/dist/esm/core/ai-element/ai-element-error.d.ts

@@ -0,0 +1,2 @@
+export declare class AIElementError extends Error {
+}

package/dist/esm/core/ai-element/ai-element-error.js

@@ -0,0 +1,2 @@
+export class AIElementError extends Error {
+}

package/dist/esm/core/ai-element/ai-element.d.ts

@@ -0,0 +1,27 @@
+import { CustomElementJson } from '../model/custom-element-json';
+interface AIElementJson {
+    version: number;
+    name: string;
+    image?: {
+        mask?: {
+            x: number;
+            y: number;
+        }[];
+    };
+}
+declare class AIElement {
+    name: string;
+    imagePath: string;
+    mask?: {
+        x: number;
+        y: number;
+    }[] | undefined;
+    constructor(name: string, imagePath: string, mask?: {
+        x: number;
+        y: number;
+    }[] | undefined);
+    static fromJson(json: AIElementJson, imagePath: string): AIElement;
+    toCustomElement(): CustomElementJson;
+    hasName(name: string): boolean;
+}
+export { AIElement, AIElementJson };

package/dist/esm/core/ai-element/ai-element.js

@@ -0,0 +1,28 @@
+import { logger } from '../../lib';
+import { AIElementError } from './ai-element-error';
+class AIElement {
+    constructor(name, imagePath, mask) {
+        this.name = name;
+        this.imagePath = imagePath;
+        this.mask = mask;
+    }
+    static fromJson(json, imagePath) {
+        var _a;
+        if (json.version === 1) {
+            return new AIElement(json.name, imagePath, (_a = json.image) === null || _a === void 0 ? void 0 : _a.mask);
+        }
+        throw new AIElementError(`Unsupported AIElement version '${json.version}'.`);
+    }
+    toCustomElement() {
+        logger.debug('Converting AIElement to CustomElementJson.');
+        return {
+            customImage: this.imagePath,
+            mask: this.mask,
+            name: this.name,
+        };
+    }
+    hasName(name) {
+        return this.name === name;
+    }
+}
+export { AIElement };

package/dist/esm/core/model/custom-element-json.d.ts

@@ -1,17 +1,16 @@
 /**
  * Defines a 'custom element'. This is a UI element which is defined by
  * providing an image and other parameters such as degree of rotation.
- * It allows filtering for a UI element that is not recognized
- * by our machine learning models by default.
- * It can also be used for pixel assertions of elements using classical
- * [template matching](https://en.wikipedia.org/wiki/Template_matching).
+ * It allows filtering for a UI element based on an image instead of using
+ * text or element descriptions like `button().withText('Submit')` in
+ * `await aui.click().button().withText('Submit').exec()`.
  *
- * **Important:** The `CustomElementJson` needs to capture as accurately as possible
- * what the custom element looks like during test execution as otherwise
- * our machine learning models cannot find it, even with the additional data
- * provided. This is especially true for the resolution used while cropping
- * the `CustomElementJson.customImage` which should match the resolution during
- * test execution.
+ * **Important:** The `CustomElementJson` needs to capture as accurately as
+ * possible what the custom element looks like during test execution as
+ * otherwise our machine learning models cannot find it, even with the
+ * additional data provided. This is especially true for the resolution used
+ * while cropping the `CustomElementJson.customImage` which should be as
+ * similar as possible to the resolution during test execution.
  *
  * Rotated custom elements can be filtered for using
  * `CustomElementJson.rotationDegreePerStep`.
@@ -46,8 +45,22 @@ export interface CustomElementJson {
      * probably not what you want) and `1.0` (= elements need to look exactly
      * like defined by `CustomElementJson` which is unlikely to be achieved
      * as even minor differences count). Defaults to `0.9`.
+     *
+     * **Important**: The `threshold` impacts the prediction quality.
      */
     threshold?: number | undefined;
+    /**
+     * A threshold for when to stop searching for UI elements similar to the
+     * custom element. As soon as UI elements have been found that are at least
+     * as similar as the `stopThreshold`, the search is going to stop. After that
+     * elements are filtered using the `threshold`. Because of that the
+     * `stopThreshold` should be greater than or equal to `threshold`. It is
+     * primarily to be used as a speed improvement (by lowering the value).
+     * Takes values between `0.0` and `1.0`. Defaults to `0.9`.
+     *
+     * **Important**: The `stopThreshold` impacts the prediction speed.
+     */
+    stopThreshold?: number | undefined;
     /**
      * A step size in rotation degree. Rotates the custom image by
      * `rotationDegreePerStep` until 360° is exceeded. Range is between
@@ -60,13 +73,17 @@ export interface CustomElementJson {
     /**
      * A color compare style. Allows matching a custom element by color, e.g.,
      * instead of filtering for all icons (blue and green ones),
-     * only the green ones captured by `customImage` are filtered for using 'RGB'.
-     * Defaults to 'grayscale'.
+     * only the green ones captured by `customImage` are filtered for using
+     * 'RGB'. Defaults to 'grayscale'.
      *
-     * **Important**: This increases the prediction time quite a bit. So only use
-     * it when absolutely necessary.
+     * **Important**: The `imageCompareFormat` impacts the prediction time as
+     * well as quality. Although this highly depends on the use case, the other
+     * parameters and the actual scene in which to find the UI element, as a
+     * rule of thumb, 'edges' is likely to be a bit faster than 'grayscale' and
+     * 'grayscale' is likely to be a bit faster than 'RGB'. For quality it is
+     * most often the other way around.
      */
-    imageCompareFormat?: 'RGB' | 'grayscale' | undefined;
+    imageCompareFormat?: 'RGB' | 'grayscale' | 'edges' | undefined;
     /** A polygon to match only a certain area of the custom element. */
     mask?: ({
         x: number;

package/dist/esm/core/model/custom-element.d.ts

@@ -3,14 +3,15 @@ export declare class CustomElement implements CustomElementJson {
     customImage: string;
     name?: string | undefined;
     threshold?: number | undefined;
+    stopThreshold?: number | undefined;
     rotationDegreePerStep?: number | undefined;
-    imageCompareFormat?: "RGB" | "grayscale" | undefined;
+    imageCompareFormat?: "RGB" | "grayscale" | "edges" | undefined;
     mask?: {
         x: number;
         y: number;
     }[] | undefined;
     private static schema;
-    constructor(customImage: string, name?: string | undefined, threshold?: number | undefined, rotationDegreePerStep?: number | undefined, imageCompareFormat?: "RGB" | "grayscale" | undefined, mask?: {
+    constructor(customImage: string, name?: string | undefined, threshold?: number | undefined, stopThreshold?: number | undefined, rotationDegreePerStep?: number | undefined, imageCompareFormat?: "RGB" | "grayscale" | "edges" | undefined, mask?: {
         x: number;
         y: number;
     }[] | undefined);

package/dist/esm/core/model/custom-element.js

@@ -10,10 +10,11 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 import { array, number, object, ValidationError, } from 'yup';
 import { Base64Image } from '../../utils/base_64_image/base-64-image';
 export class CustomElement {
-    constructor(customImage, name, threshold, rotationDegreePerStep, imageCompareFormat, mask) {
+    constructor(customImage, name, threshold, stopThreshold, rotationDegreePerStep, imageCompareFormat, mask) {
         this.customImage = customImage;
         this.name = name;
         this.threshold = threshold;
+        this.stopThreshold = stopThreshold;
         this.rotationDegreePerStep = rotationDegreePerStep;
         this.imageCompareFormat = imageCompareFormat;
         this.mask = mask;
@@ -32,7 +33,7 @@ export class CustomElement {
         });
     }
     static fromJson(ceJson) {
-        return new CustomElement(ceJson.customImage, ceJson.name, ceJson.threshold, ceJson.rotationDegreePerStep, ceJson.imageCompareFormat, ceJson.mask);
+        return new CustomElement(ceJson.customImage, ceJson.name, ceJson.threshold, ceJson.stopThreshold, ceJson.rotationDegreePerStep, ceJson.imageCompareFormat, ceJson.mask);
     }
     validate() {
         try {
@@ -46,5 +47,6 @@ export class CustomElement {
 CustomElement.schema = object({
     mask: array().optional().min(3, 'mask must contain at least 3 points'),
     rotationDegreePerStep: number().optional().min(0).lessThan(360),
+    stopThreshold: number().optional().min(0).max(1),
     threshold: number().optional().min(0).max(1),
 });

package/dist/esm/core/reporting/step-reporter.js

@@ -55,37 +55,46 @@ export class StepReporter {
     onStepBegin(snapshot) {
         return __awaiter(this, void 0, void 0, function* () {
             if (this._currentStep === undefined) {
-                throw new Error('Cannot begin step if step is undefined.');
+                logger.error('Cannot begin step if step is undefined.');
+                return Promise.resolve();
             }
             if (this._currentStep.status !== 'pending') {
-                throw new Error('Cannot begin step that is not pending.');
+                logger.error('Cannot begin step that is not pending.');
+                return Promise.resolve();
             }
             this._currentStep = this._currentStep.onBegin(snapshot);
             this.enqueueReporterCalls('onStepBegin', this._currentStep);
+            return Promise.resolve();
         });
     }
     onStepRetry(snapshot, error) {
         return __awaiter(this, void 0, void 0, function* () {
             if (this._currentStep === undefined) {
-                throw new Error('Cannot retry step if step is undefined.');
+                logger.error('Cannot retry step if step is undefined.');
+                return Promise.resolve();
             }
             if (this._currentStep.status !== 'running') {
-                throw new Error('Cannot retry step that has not been running.');
+                logger.error('Cannot retry step that has not been running.');
+                return Promise.resolve();
             }
             this._currentStep = this._currentStep.onRetry(snapshot, error);
             this.enqueueReporterCalls('onStepRetry', this._currentStep);
+            return Promise.resolve();
         });
     }
     onStepEnd(snapshot, error) {
         return __awaiter(this, void 0, void 0, function* () {
             if (this._currentStep === undefined) {
-                throw new Error('Cannot end step if step is undefined.');
+                logger.error('Cannot end step if step is undefined.');
+                return Promise.resolve();
             }
             if (this._currentStep.status !== 'running') {
-                throw new Error('Cannot end step that has not been running.');
+                logger.error('Cannot end step that has not been running.');
+                return Promise.resolve();
             }
             this._currentStep = this._currentStep.onEnd(snapshot, error);
             this.enqueueReporterCalls('onStepEnd', this._currentStep);
+            return Promise.resolve();
         });
     }
 }