askui 0.10.0 → 0.10.2

package/dist/esm/core/model/annotation-result/annotation-interface.d.ts

@@ -1,4 +1,4 @@
-import { CustomElementJson } from '../test-case-dto';
+import { CustomElementJson } from '../custom-element-json';
 export interface AnnotationRequest {
     imagePath?: string;
     outputPath?: string;

package/dist/esm/core/model/custom-element-json.d.ts

@@ -0,0 +1,75 @@
+/**
+ * Defines a 'custom element'. This is a UI element which is defined by
+ * providing an image and other parameters such as degree of rotation.
+ * It allows filtering for a UI element that is not recognized
+ * by our machine learning models by default.
+ * It can also be used for pixel assertions of elements using classical
+ * [template matching](https://en.wikipedia.org/wiki/Template_matching).
+ *
+ * **Important:** The `CustomElementJson` needs to capture as accurately as possible
+ * what the custom element looks like during test execution as otherwise
+ * our machine learning models cannot find it, even with the additional data
+ * provided. This is especially true for the resolution used while cropping
+ * the `CustomElementJson.customImage` which should match the resolution during
+ * test execution.
+ *
+ * Rotated custom elements can be filtered for using
+ * `CustomElementJson.rotationDegreePerStep`.
+ */
+export interface CustomElementJson {
+    /**
+     * An cropped image in form of a base64 string or file path,
+     * e.g., "./custom.png".
+     */
+    customImage: string;
+    /**
+     * A unique name which can be used for filtering for the custom element,
+     * e.g.,
+     * ```typescript
+     * ...customElement({
+     *   name: 'unique-name',
+     *   // ... (rest of the custom element)
+     * }).withText('unique-name')
+     * ```
+     *
+     * If not set, the text inside the custom element is
+     * detected via
+     * [OCR](https://en.wikipedia.org/wiki/Optical_character_recognition).
+     * It can also be used for filtering for the custom element.
+     */
+    name?: string | undefined;
+    /**
+     * A threshold for how similar UI elements shown during test execution
+     * need to be to the custom element as defined by the other fields in
+     * `CustomElementJson` to be recognized as such. Takes values between
+     * `0.0` (= all elements are recognized as the custom element which is
+     * probably not what you want) and `1.0` (= elements need to look exactly
+     * like defined by `CustomElementJson` which is unlikely to be achieved
+     * as even minor differences count). Defaults to `0.9`.
+     */
+    threshold?: number | undefined;
+    /**
+     * A step size in rotation degree. Rotates the custom image by
+     * `rotationDegreePerStep` until 360° is exceeded. Range is between
+     * 0° - 360°. Defaults to 0°.
+     *
+     * **Important**: This increases the prediction time quite a bit. So only use
+     * it when absolutely necessary.
+     */
+    rotationDegreePerStep?: number | undefined;
+    /**
+     * A color compare style. Allows matching a custom element by color, e.g.,
+     * instead of filtering for all icons (blue and green ones),
+     * only the green ones captured by `customImage` are filtered for using 'RGB'.
+     * Defaults to 'grayscale'.
+     *
+     * **Important**: This increases the prediction time quite a bit. So only use
+     * it when absolutely necessary.
+     */
+    imageCompareFormat?: 'RGB' | 'grayscale' | undefined;
+    /** A polygon to match only a certain area of the custom element. */
+    mask?: ({
+        x: number;
+        y: number;
+    })[] | undefined;
+}

package/dist/esm/core/model/custom-element-json.js

@@ -0,0 +1 @@
+export {};

package/dist/esm/core/model/custom-element.d.ts

@@ -0,0 +1,21 @@
+import { CustomElementJson } from './custom-element-json';
+export declare class CustomElement implements CustomElementJson {
+    customImage: string;
+    name?: string | undefined;
+    threshold?: number | undefined;
+    rotationDegreePerStep?: number | undefined;
+    imageCompareFormat?: "RGB" | "grayscale" | undefined;
+    mask?: {
+        x: number;
+        y: number;
+    }[] | undefined;
+    private static schema;
+    constructor(customImage: string, name?: string | undefined, threshold?: number | undefined, rotationDegreePerStep?: number | undefined, imageCompareFormat?: "RGB" | "grayscale" | undefined, mask?: {
+        x: number;
+        y: number;
+    }[] | undefined);
+    static fromJsonListWithImagePathOrImage(ceJson?: CustomElementJson[]): Promise<CustomElement[]>;
+    static fromJsonWithImagePathOrImage(ceJson: CustomElementJson): Promise<CustomElement>;
+    static fromJson(ceJson: CustomElementJson): CustomElement;
+    validate(): void;
+}

package/dist/esm/core/model/custom-element.js

@@ -0,0 +1,50 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { array, number, object, ValidationError, } from 'yup';
+import { Base64Image } from '../../utils/base_64_image/base-64-image';
+export class CustomElement {
+    constructor(customImage, name, threshold, rotationDegreePerStep, imageCompareFormat, mask) {
+        this.customImage = customImage;
+        this.name = name;
+        this.threshold = threshold;
+        this.rotationDegreePerStep = rotationDegreePerStep;
+        this.imageCompareFormat = imageCompareFormat;
+        this.mask = mask;
+    }
+    static fromJsonListWithImagePathOrImage(ceJson = []) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return Promise.all(ceJson.map((customElement) => CustomElement.fromJsonWithImagePathOrImage(customElement)));
+        });
+    }
+    static fromJsonWithImagePathOrImage(ceJson) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const customImage = (yield Base64Image.fromPathOrString(ceJson.customImage)).toString();
+            const customElement = CustomElement.fromJson(Object.assign(Object.assign({}, ceJson), { customImage }));
+            customElement.validate();
+            return customElement;
+        });
+    }
+    static fromJson(ceJson) {
+        return new CustomElement(ceJson.customImage, ceJson.name, ceJson.threshold, ceJson.rotationDegreePerStep, ceJson.imageCompareFormat, ceJson.mask);
+    }
+    validate() {
+        try {
+            CustomElement.schema.validateSync(this, { abortEarly: false, strict: true });
+        }
+        catch (e) { // ValidationError
+            throw new ValidationError(e.errors.join(', '));
+        }
+    }
+}
+CustomElement.schema = object({
+    threshold: number().optional().min(0).max(1),
+    rotationDegreePerStep: number().optional().min(0).lessThan(360),
+    mask: array().optional().min(3, 'mask must contain at least 3 points'),
+});

package/dist/esm/core/reporting/default-reporter.d.ts

@@ -0,0 +1,10 @@
+import { Step } from './step';
+export declare const DEFAULT_REPORTER: {
+    config: {
+        readonly withScreenshots: "onFailure";
+        readonly withDetectedElements: "onFailure";
+    };
+    onStepBegin: (_step: Step) => Promise<void>;
+    onStepRetry: (_step: Step) => Promise<void>;
+    onStepEnd: (_step: Step) => Promise<void>;
+};

package/dist/esm/core/reporting/default-reporter.js

@@ -0,0 +1,9 @@
+export const DEFAULT_REPORTER = {
+    config: {
+        withScreenshots: 'onFailure',
+        withDetectedElements: 'onFailure',
+    },
+    onStepBegin: (_step) => Promise.resolve(),
+    onStepRetry: (_step) => Promise.resolve(),
+    onStepEnd: (_step) => Promise.resolve(),
+};

package/dist/esm/core/reporting/default-step.d.ts

@@ -0,0 +1,24 @@
+import { Instruction } from './instruction';
+import { Snapshot } from './snapshot';
+import { Step } from './step';
+import { StepRun } from './step-run';
+import { StepStatus } from './step-status';
+export declare class DefaultStep implements Step {
+    instruction: Instruction;
+    runs: Readonly<StepRun[]>;
+    constructor(instruction: Instruction);
+    get status(): StepStatus;
+    get begin(): Snapshot | undefined;
+    get end(): Snapshot | undefined;
+    get error(): Error | undefined;
+    get retries(): StepRun[];
+    get retryCount(): number;
+    get firstRun(): StepRun | undefined;
+    get lastRun(): StepRun | undefined;
+    get flaky(): boolean;
+    get duration(): number | undefined;
+    onBegin(snapshot: Snapshot): DefaultStep;
+    onRetry(snapshot: Snapshot, error: Error): DefaultStep;
+    onEnd(snapshot: Snapshot, error?: Error): DefaultStep;
+    private static determineLastRunStatus;
+}

package/dist/esm/core/reporting/default-step.js

@@ -0,0 +1,71 @@
+export class DefaultStep {
+    constructor(instruction) {
+        this.instruction = instruction;
+        this.runs = [];
+    }
+    get status() {
+        var _a, _b;
+        return (_b = (_a = this.lastRun) === null || _a === void 0 ? void 0 : _a.status) !== null && _b !== void 0 ? _b : 'pending';
+    }
+    get begin() {
+        var _a;
+        return (_a = this.firstRun) === null || _a === void 0 ? void 0 : _a.begin;
+    }
+    get end() {
+        var _a;
+        return (_a = this.lastRun) === null || _a === void 0 ? void 0 : _a.end;
+    }
+    get error() {
+        var _a;
+        return (_a = this.lastRun) === null || _a === void 0 ? void 0 : _a.error;
+    }
+    get retries() {
+        var _a, _b;
+        return (_b = (_a = this.runs) === null || _a === void 0 ? void 0 : _a.slice(1)) !== null && _b !== void 0 ? _b : [];
+    }
+    get retryCount() {
+        var _a, _b;
+        return (_b = (_a = this.retries) === null || _a === void 0 ? void 0 : _a.length) !== null && _b !== void 0 ? _b : 0;
+    }
+    get firstRun() {
+        var _a;
+        return (_a = this.runs) === null || _a === void 0 ? void 0 : _a[0];
+    }
+    get lastRun() {
+        var _a;
+        return (_a = this.runs) === null || _a === void 0 ? void 0 : _a[this.runs.length - 1];
+    }
+    get flaky() {
+        return this.retryCount > 0;
+    }
+    get duration() {
+        if (this.begin !== undefined && this.end !== undefined) {
+            return this.end.createdAt.getTime() - this.begin.createdAt.getTime();
+        }
+        return undefined;
+    }
+    onBegin(snapshot) {
+        this.runs = [...this.runs, {
+                status: 'running',
+                begin: snapshot,
+            }];
+        return this;
+    }
+    onRetry(snapshot, error) {
+        this.onEnd(snapshot, error);
+        this.onBegin(snapshot);
+        return this;
+    }
+    onEnd(snapshot, error) {
+        this.runs = [...this.runs.slice(0, -1), Object.assign(Object.assign({}, this.lastRun), { status: DefaultStep.determineLastRunStatus(error), end: snapshot, error, 
+                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                duration: snapshot.createdAt.getTime() - this.lastRun.begin.createdAt.getTime() })];
+        return this;
+    }
+    static determineLastRunStatus(error) {
+        if (error !== undefined) {
+            return 'failed';
+        }
+        return 'passed';
+    }
+}

package/dist/esm/core/reporting/index.d.ts

@@ -0,0 +1,11 @@
+export * from './default-reporter';
+export * from './instruction';
+export * from './reporter-config';
+export * from './reporter';
+export * from './snapshot-detail-level';
+export * from './snapshot';
+export * from './step-reporter';
+export * from './step-run';
+export * from './step-status-end';
+export * from './step-status';
+export * from './step';

package/dist/esm/core/reporting/index.js

@@ -0,0 +1,11 @@
+export * from './default-reporter';
+export * from './instruction';
+export * from './reporter-config';
+export * from './reporter';
+export * from './snapshot-detail-level';
+export * from './snapshot';
+export * from './step-reporter';
+export * from './step-run';
+export * from './step-status-end';
+export * from './step-status';
+export * from './step';

package/dist/esm/core/reporting/instruction.d.ts

@@ -0,0 +1,7 @@
+import { CustomElement } from '@/core/model/custom-element';
+export interface Instruction {
+    readonly value: string;
+    readonly valueHumanReadable: string;
+    readonly customElements?: Readonly<Readonly<CustomElement>>[];
+    readonly secretText?: string | undefined;
+}

package/dist/esm/core/reporting/instruction.js

@@ -0,0 +1 @@
+export {};

package/dist/esm/core/reporting/reporter-config.d.ts

@@ -0,0 +1,13 @@
+import { SnapshotDetailLevel } from './snapshot-detail-level';
+export interface ReporterConfig {
+    /**
+       * default: onFailure; makes step run slower
+       */
+    withScreenshots?: SnapshotDetailLevel;
+    /**
+       * overrides withScreenshot if higher level of detail as screenshot
+       *  is required for detecting elements;
+       * incurres additional cost; default: onFailure; makes step run slower
+       */
+    withDetectedElements?: SnapshotDetailLevel;
+}

package/dist/esm/core/reporting/reporter-config.js

@@ -0,0 +1 @@
+export {};

package/dist/esm/core/reporting/reporter.d.ts

@@ -0,0 +1,14 @@
+import { Step } from './step';
+import { ReporterConfig } from './reporter-config';
+/**
+ * Can be used to report on a step run.
+ *
+ * The reporter is an interface (instead of a class) so that the it may be implemented by a class
+ * that already extends another class, e.g., the reporter of a test framework.
+ */
+export interface Reporter {
+    config?: ReporterConfig;
+    onStepBegin?(step: Step): Promise<void>;
+    onStepRetry?(step: Step): Promise<void>;
+    onStepEnd?(step: Step): Promise<void>;
+}

package/dist/esm/core/reporting/reporter.js

@@ -0,0 +1 @@
+export {};

package/dist/esm/core/reporting/snapshot-detail-level.d.ts

@@ -0,0 +1,21 @@
+export declare type SnapshotDetailLevel = 
+/**
+ * Details of snapshot, e.g., screenshot or detected elements, may or may not be available
+ *  depending on if they are required by the step. There are not guarantees made.
+ */
+'required' | 
+/**
+ * Details are available when the step fails, e.g., for debugging.
+ * Includes everything of required.
+ */
+'onFailure' | 
+/**
+ * Details are available also when the command is started, e.g.,
+ *  for detecting why a certain element was interacted with. Includes everything of onFailure.
+ */
+'begin' | 
+/**
+ * Details are available always, e.g.,
+ *  before and after a step has been run no matter if it failed or not for debugging.
+ */
+'always';

package/dist/esm/core/reporting/snapshot-detail-level.js

@@ -0,0 +1 @@
+export {};

package/dist/esm/core/reporting/snapshot.d.ts

@@ -0,0 +1,6 @@
+import { DetectedElement } from '../model/annotation-result/detected-element';
+export interface Snapshot {
+    readonly createdAt: Date;
+    readonly screenshot?: string | undefined;
+    readonly detectedElements?: Readonly<Readonly<DetectedElement>>[] | undefined;
+}

package/dist/esm/core/reporting/snapshot.js

@@ -0,0 +1 @@
+export {};

package/dist/esm/core/reporting/step-reporter.d.ts

@@ -0,0 +1,17 @@
+import { Snapshot } from './snapshot';
+import { Instruction } from './instruction';
+import { Reporter } from './reporter';
+import { ReporterConfig } from './reporter-config';
+import { DefaultStep } from './default-step';
+export declare class StepReporter {
+    private reporter;
+    currentStep?: DefaultStep | undefined;
+    constructor(reporter: Required<Reporter> & {
+        config: Required<ReporterConfig>;
+    });
+    get config(): Required<ReporterConfig>;
+    resetStep(instruction: Instruction): void;
+    onStepBegin(snapshot: Snapshot): Promise<void>;
+    onStepRetry(snapshot: Snapshot, error: Error): Promise<void>;
+    onStepEnd(snapshot: Snapshot, error?: Error): Promise<void>;
+}

package/dist/esm/core/reporting/step-reporter.js

@@ -0,0 +1,60 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { DefaultStep } from './default-step';
+export class StepReporter {
+    constructor(reporter) {
+        this.reporter = reporter;
+    }
+    get config() {
+        return this.reporter.config;
+    }
+    resetStep(instruction) {
+        this.currentStep = new DefaultStep(instruction);
+    }
+    onStepBegin(snapshot) {
+        var _a, _b;
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.currentStep === undefined) {
+                throw new Error('Cannot begin step if step is undefined.');
+            }
+            if (this.currentStep.status !== 'pending') {
+                throw new Error('Cannot begin step that is not pending.');
+            }
+            this.currentStep.onBegin(snapshot);
+            yield ((_b = (_a = this.reporter).onStepBegin) === null || _b === void 0 ? void 0 : _b.call(_a, this.currentStep));
+        });
+    }
+    onStepRetry(snapshot, error) {
+        var _a, _b;
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.currentStep === undefined) {
+                throw new Error('Cannot retry step if step is undefined.');
+            }
+            if (this.currentStep.status !== 'running') {
+                throw new Error('Cannot retry step that has not been running.');
+            }
+            this.currentStep.onRetry(snapshot, error);
+            yield ((_b = (_a = this.reporter).onStepRetry) === null || _b === void 0 ? void 0 : _b.call(_a, this.currentStep));
+        });
+    }
+    onStepEnd(snapshot, error) {
+        var _a, _b;
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.currentStep === undefined) {
+                throw new Error('Cannot end step if step is undefined.');
+            }
+            if (this.currentStep.status !== 'running') {
+                throw new Error('Cannot end step that has not been running.');
+            }
+            this.currentStep.onEnd(snapshot, error);
+            yield ((_b = (_a = this.reporter).onStepEnd) === null || _b === void 0 ? void 0 : _b.call(_a, this.currentStep));
+        });
+    }
+}

package/dist/esm/core/reporting/step-run.d.ts

@@ -0,0 +1,9 @@
+import { StepStatus } from './step-status';
+import { Snapshot } from './snapshot';
+export interface StepRun {
+    readonly status: StepStatus;
+    readonly begin?: Snapshot | undefined;
+    readonly end?: Snapshot | undefined;
+    readonly duration?: number | undefined;
+    readonly error?: Error | undefined;
+}

package/dist/esm/core/reporting/step-run.js

@@ -0,0 +1 @@
+export {};

package/dist/esm/core/reporting/step-status-end.d.ts

@@ -0,0 +1 @@
+export declare type StepStatusEnd = 'passed' | 'failed' | 'erroneous';

package/dist/esm/core/reporting/step-status-end.js

@@ -0,0 +1 @@
+export {};

package/dist/esm/core/reporting/step-status.d.ts

@@ -0,0 +1,13 @@
+/**
+ * - `passed`: the step passed
+ * - `failed`: the step failed because of a failed (implicit/explicit) assertion,
+ *  e.g., an element could not be found
+ * - `pending`: the step is waiting for a previous step to pass
+ * - `running`: the step is currently running (including retries)
+ * - `skipped`: the step was skipped, e.g., because a previous step failed or
+ *  because workflow was skipped
+ * - `erroneous`: the step could not be run because of a runtime error,
+ *  e.g., the user has no usage left, response of inference backend cannot be processed,
+ *  e.g., because lib version is outdated, etc. (currently not supported, everything "failed")
+ */
+export declare type StepStatus = 'passed' | 'failed' | 'pending' | 'running' | 'erroneous';

package/dist/esm/core/reporting/step-status.js

@@ -0,0 +1 @@
+export {};

package/dist/esm/core/reporting/step.d.ts

@@ -0,0 +1,36 @@
+import { StepStatus } from './step-status';
+import { Snapshot } from './snapshot';
+import { Instruction } from './instruction';
+import { StepRun } from './step-run';
+export interface Step {
+    readonly instruction: Readonly<Instruction>;
+    readonly status: StepStatus;
+    /**
+       * A snapshot of the state of the screen before the step is
+       * ran which is nearly immediately after the call to `.exec()`
+       */
+    readonly begin?: Snapshot | undefined;
+    /**
+       * The duration of the step in milliseconds based on the start
+       * and end time. If the step is still running, this property is undefined.
+       */
+    readonly duration?: number | undefined;
+    /**
+       * A snapshot of the state of the screen after the step has been run.
+       * If the step is still running or still pending, this property is undefined.
+       */
+    readonly end?: Snapshot | undefined;
+    /**
+       * When a step failed or is erroneous, this property contains the error.
+       */
+    readonly error?: Error | undefined;
+    readonly runs: Readonly<StepRun[]>;
+    readonly retries: Readonly<StepRun[]>;
+    readonly retryCount: number;
+    readonly firstRun?: StepRun | undefined;
+    readonly lastRun?: StepRun | undefined;
+    /**
+       * retryCount > 0.
+       */
+    readonly flaky: boolean;
+}

package/dist/esm/core/reporting/step.js

@@ -0,0 +1 @@
+export {};

package/dist/esm/execution/dsl.d.ts

@@ -1,4 +1,4 @@
-import { CustomElementJson } from '../core/model/test-case-dto';
+import { CustomElementJson } from '../core/model/custom-element-json';
 import { DetectedElement } from '../core/model/annotation-result/detected-element';
 export declare enum Separators {
     STRING = "<|string|>"

package/dist/esm/execution/execution-runtime.d.ts

@@ -1,19 +1,22 @@
-import { TestStep } from '../core/model/test-case-dto';
 import { UiControllerClient } from './ui-controller-client';
 import { InferenceClient } from './inference-client';
 import { Annotation } from '../core/annotation/annotation';
-import { CustomElementJson } from '../core/model/test-case-dto/custom-element-json';
+import { CustomElementJson } from '../core/model/custom-element-json';
 import { DetectedElement } from '../core/model/annotation-result/detected-element';
+import { UiControllerClientConnectionState } from './ui-controller-client-connection-state';
+import { Instruction, StepReporter } from '../core/reporting';
 export declare class ExecutionRuntime {
     private uiControllerClient;
     private inferenceClient;
-    constructor(uiControllerClient: UiControllerClient, inferenceClient: InferenceClient);
-    executeTestStep(step: TestStep): Promise<void>;
+    private stepReporter;
+    constructor(uiControllerClient: UiControllerClient, inferenceClient: InferenceClient, stepReporter: StepReporter);
+    connect(): Promise<UiControllerClientConnectionState>;
+    disconnect(): void;
+    startVideoRecording(): Promise<void>;
+    stopVideoRecording(): Promise<void>;
+    readVideoRecording(): Promise<string>;
     private requestControl;
-    /**
-     * @param {TestStep} step - Test step used for predicting command.
-     */
-    private executeCommand;
+    executeInstruction(instruction: Instruction): Promise<void>;
     private readonly EXEC_REPETITION_COUNT;
     private executeCommandRepeatedly;
     private readonly PREDICT_COMMAND_RETRY_COUNT;
@@ -22,7 +25,11 @@ export declare class ExecutionRuntime {
      * --> retry with linear back-off
      */
     private predictCommandWithRetry;
-    private getImageIfRequired;
+    private isImageRequiredByConfig;
+    private isImageRequired;
+    private isAnnotationRequired;
+    getScreenshot(): Promise<string>;
+    private buildSnapshot;
     private predictCommand;
     annotateInteractively(): Promise<void>;
     takeScreenshotIfImageisNotProvided(imagePath?: string): Promise<string>;