askui 0.10.0 → 0.10.2

package/dist/cjs/execution/execution-runtime.js

@@ -11,22 +11,42 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ExecutionRuntime = void 0;
 const ui_control_commands_1 = require("../core/ui-control-commands");
-const test_case_dto_1 = require("../core/model/test-case-dto");
+const custom_element_1 = require("../core/model/custom-element");
 const repeat_error_1 = require("./repeat-error");
 const misc_1 = require("./misc");
 const control_command_error_1 = require("./control-command-error");
 const logger_1 = require("../lib/logger");
 const base_64_image_1 = require("../utils/base_64_image/base-64-image");
 class ExecutionRuntime {
-    constructor(uiControllerClient, inferenceClient) {
+    constructor(uiControllerClient, inferenceClient, stepReporter) {
         this.uiControllerClient = uiControllerClient;
         this.inferenceClient = inferenceClient;
+        this.stepReporter = stepReporter;
         this.EXEC_REPETITION_COUNT = 25;
         this.PREDICT_COMMAND_RETRY_COUNT = 2;
     }
-    executeTestStep(step) {
+    connect() {
         return __awaiter(this, void 0, void 0, function* () {
-            yield this.executeCommand(step);
+            return this.uiControllerClient.connect();
+        });
+    }
+    disconnect() {
+        this.uiControllerClient.disconnect();
+    }
+    startVideoRecording() {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.uiControllerClient.startVideoRecording();
+        });
+    }
+    stopVideoRecording() {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.uiControllerClient.stopVideoRecording();
+        });
+    }
+    readVideoRecording() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const response = yield this.uiControllerClient.readVideoRecording();
+            return response.data.video;
         });
     }
     requestControl(controlCommand) {
@@ -34,25 +54,22 @@ class ExecutionRuntime {
             yield this.uiControllerClient.requestControl(controlCommand);
         });
     }
-    /**
-     * @param {TestStep} step - Test step used for predicting command.
-     */
-    executeCommand(step) {
-        var _a;
+    executeInstruction(instruction) {
+        var _a, _b, _c;
         return __awaiter(this, void 0, void 0, function* () {
-            const controlCommand = yield this.predictCommandWithRetry(step);
+            const controlCommand = yield this.predictCommandWithRetry(instruction);
             if (controlCommand.code === ui_control_commands_1.ControlCommandCode.OK) {
                 return this.requestControl(controlCommand);
             }
             if (controlCommand.tryToRepeat) {
                 yield this.requestControl(controlCommand);
-                return this.executeCommandRepeatedly(step);
+                return this.executeCommandRepeatedly(instruction);
             }
-            throw new control_command_error_1.ControlCommandError(((_a = controlCommand.actions[0]) === null || _a === void 0 ? void 0 : _a.text) || '');
+            throw new control_command_error_1.ControlCommandError((_c = (_b = (_a = controlCommand.actions) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.text) !== null && _c !== void 0 ? _c : '');
         });
     }
-    executeCommandRepeatedly(step) {
-        var _a;
+    executeCommandRepeatedly(instruction) {
+        var _a, _b, _c;
         return __awaiter(this, void 0, void 0, function* () {
             /* eslint-disable no-await-in-loop */
             for (let repeatCount = this.EXEC_REPETITION_COUNT; repeatCount >= 0; repeatCount -= 1) {
@@ -61,7 +78,7 @@ class ExecutionRuntime {
                         + 'from a single test step reached');
                 }
                 logger_1.logger.debug('Repeat command execution....');
-                const controlCommand = yield this.predictCommandWithRetry(step);
+                const controlCommand = yield this.predictCommandWithRetry(instruction);
                 if (controlCommand.code === ui_control_commands_1.ControlCommandCode.OK) {
                     break;
                 }
@@ -69,7 +86,7 @@ class ExecutionRuntime {
                     yield this.requestControl(controlCommand);
                 }
                 else {
-                    throw new control_command_error_1.ControlCommandError(((_a = controlCommand.actions[0]) === null || _a === void 0 ? void 0 : _a.text) || '');
+                    throw new control_command_error_1.ControlCommandError((_c = (_b = (_a = controlCommand.actions) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.text) !== null && _c !== void 0 ? _c : '');
                 }
             }
         });
@@ -79,9 +96,10 @@ class ExecutionRuntime {
      * --> retry with linear back-off
      */
     /* eslint-disable-next-line consistent-return */
-    predictCommandWithRetry(step) {
+    predictCommandWithRetry(instruction) {
+        var _a, _b, _c;
         return __awaiter(this, void 0, void 0, function* () {
-            let command = yield this.predictCommand(step);
+            let command = yield this.predictCommand(instruction);
             /* eslint-disable no-await-in-loop */
             for (let k = 0; k < this.PREDICT_COMMAND_RETRY_COUNT; k += 1) {
                 if (command.code === ui_control_commands_1.ControlCommandCode.OK) {
@@ -90,28 +108,56 @@ class ExecutionRuntime {
                 const msUntilRetry = k * 1000;
                 logger_1.logger.debug(`Wait ${msUntilRetry} and retry predicting command...`);
                 yield (0, misc_1.delay)(msUntilRetry);
-                command = yield this.predictCommand(step);
+                command = yield this.predictCommand(instruction, new control_command_error_1.ControlCommandError((_c = (_b = (_a = command.actions) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.text) !== null && _c !== void 0 ? _c : ''));
             }
             /* eslint-enable no-await-in-loop */
             return command;
         });
     }
-    getImageIfRequired(instruction) {
+    isImageRequiredByConfig() {
+        return this.stepReporter.config.withScreenshots === 'begin'
+            || this.stepReporter.config.withScreenshots === 'always';
+    }
+    isImageRequired(instruction) {
         return __awaiter(this, void 0, void 0, function* () {
-            const isImageRequired = yield this.inferenceClient.isImageRequired(instruction);
-            if (!isImageRequired) {
-                return undefined;
-            }
-            const screenshotResponse = yield this.uiControllerClient.requestScreenshot();
-            return screenshotResponse.data.image;
+            if (this.isImageRequiredByConfig())
+                return Promise.resolve(true);
+            return this.inferenceClient.isImageRequired(instruction);
+        });
+    }
+    isAnnotationRequired() {
+        return this.stepReporter.config.withDetectedElements === 'begin'
+            || this.stepReporter.config.withDetectedElements === 'always';
+    }
+    getScreenshot() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const requestScreenshotResponse = yield this.uiControllerClient.requestScreenshot();
+            return requestScreenshotResponse.data.image;
+        });
+    }
+    buildSnapshot(instruction) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const createdAt = new Date();
+            const screenshot = (yield this.isImageRequired(instruction))
+                ? yield this.getScreenshot() : undefined;
+            const annotation = this.isAnnotationRequired() ? yield this.annotateImage() : undefined;
+            return {
+                createdAt,
+                screenshot,
+                detectedElements: annotation === null || annotation === void 0 ? void 0 : annotation.detected_elements,
+            };
         });
     }
-    predictCommand(step) {
+    predictCommand(instruction, retryError) {
         return __awaiter(this, void 0, void 0, function* () {
-            const image = yield this.getImageIfRequired(step.instruction);
-            const controlCommand = yield this.inferenceClient.predictControlCommand(step.instruction, step.customElements, image);
-            if (step.secretText !== undefined) {
-                controlCommand.setTextToBeTyped(step.secretText);
+            const snapshot = yield this.buildSnapshot(instruction.value);
+            if (retryError !== undefined)
+                this.stepReporter.onStepRetry(snapshot, retryError);
+            else
+                this.stepReporter.onStepBegin(snapshot);
+            const controlCommand = yield this.inferenceClient.predictControlCommand(instruction.value, instruction.customElements, snapshot.screenshot);
+            if (instruction.secretText !== undefined) {
+                controlCommand.setTextToBeTyped(instruction.secretText);
             }
             return controlCommand;
         });
@@ -140,7 +186,7 @@ class ExecutionRuntime {
             let customElements = [];
             const base64Image = yield this.takeScreenshotIfImageisNotProvided();
             if (customElementJson !== undefined) {
-                customElements = yield test_case_dto_1.CustomElement.fromJsonListWithImagePathOrImage(customElementJson);
+                customElements = yield custom_element_1.CustomElement.fromJsonListWithImagePathOrImage(customElementJson);
             }
             return this.inferenceClient.getDetectedElements(instruction, base64Image, customElements);
         });
@@ -150,7 +196,7 @@ class ExecutionRuntime {
             let customElements = [];
             const base64Image = yield this.takeScreenshotIfImageisNotProvided(imagePath);
             if (customElementJson !== undefined) {
-                customElements = yield test_case_dto_1.CustomElement.fromJsonListWithImagePathOrImage(customElementJson);
+                customElements = yield custom_element_1.CustomElement.fromJsonListWithImagePathOrImage(customElementJson);
             }
             return this.inferenceClient.predictImageAnnotation(base64Image, customElements);
         });

package/dist/cjs/execution/inference-client.d.ts

@@ -1,6 +1,6 @@
 import { HttpClientGot } from '../utils/http/http-client-got';
 import { ControlCommand } from '../core/ui-control-commands';
-import { CustomElement } from '../core/model/test-case-dto';
+import { CustomElement } from '../core/model/custom-element';
 import { Annotation } from '../core/annotation/annotation';
 import { DetectedElement } from '../core/model/annotation-result/detected-element';
 import { ModelCompositionBranch } from './model-composition-branch';

package/dist/cjs/execution/reporter.d.ts

@@ -0,0 +1,132 @@
+import { DetectedElement } from "@/core/model/annotation-result/detected-element";
+import { CustomElement } from "@/core/model/custom-element";
+export interface Snapshot {
+    readonly createdAt: Date;
+    readonly screenshot?: string | undefined;
+    readonly detectedElements?: Readonly<Readonly<DetectedElement>>[] | undefined;
+}
+export interface Instruction {
+    readonly value: string;
+    readonly valueHumanReadable: string;
+    readonly customElements?: Readonly<Readonly<CustomElement>>[];
+    readonly secretText?: string | undefined;
+}
+export interface Step {
+    readonly instruction: Readonly<Instruction>;
+    readonly status: StepStatus;
+    /**
+     * A snapshot of the state of the screen before the step is ran which is nearly immediately after the call to `.exec()`
+     */
+    readonly begin?: Snapshot | undefined;
+    /**
+     * The duration of the step in milliseconds based on the start and end time. If the step is still running, this property is undefined.
+     */
+    readonly duration?: number | undefined;
+    /**
+     * A snapshot of the state of the screen after the step has been run. If the step is still running or still pending, this property is undefined.
+     */
+    readonly end?: Snapshot | undefined;
+    /**
+     * When a step failed or is erroneous, this property contains the error.
+     */
+    readonly error?: Error | undefined;
+    readonly runs: Readonly<StepRun[]>;
+    readonly retries: Readonly<StepRun[]>;
+    readonly retryCount: number;
+    readonly firstRun?: StepRun | undefined;
+    readonly lastRun?: StepRun | undefined;
+    /**
+     * retryCount > 0.
+     */
+    readonly flaky: boolean;
+}
+export interface StepRun {
+    readonly status: StepStatus;
+    readonly begin?: Snapshot | undefined;
+    readonly end?: Snapshot | undefined;
+    readonly duration?: number | undefined;
+    readonly error?: Error | undefined;
+}
+declare class askuiStep implements Step {
+    instruction: Instruction;
+    runs: Readonly<StepRun[]>;
+    constructor(instruction: Instruction);
+    get status(): StepStatus;
+    get begin(): Snapshot | undefined;
+    get end(): Snapshot | undefined;
+    get error(): Error | undefined;
+    get retries(): StepRun[];
+    get retryCount(): number;
+    get firstRun(): StepRun | undefined;
+    get lastRun(): StepRun | undefined;
+    get flaky(): boolean;
+    get duration(): number | undefined;
+    onBegin(snapshot: Snapshot): askuiStep;
+    onRetry(snapshot: Snapshot, error: Error): askuiStep;
+    onEnd(snapshot: Snapshot, error?: Error): askuiStep;
+    private determineLastRunStatus;
+}
+/**
+ * - `passed`: the step passed
+ * - `failed`: the step failed because of a failed (implicit/explicit) assertion, e.g., an element could not be found
+ * - `pending`: the step is waiting for a previous step to pass
+ * - `running`: the step is currently running (including retries)
+ * - `skipped`: the step was skipped, e.g., because a previous step failed or because workflow was skipped
+ * - `erroneous`: the step could not be run because of a runtime error, e.g., the user has no usage left, response of inference backend cannot be processed, e.g., because lib version is outdated, etc. (currently not supported, everything "failed")
+ */
+export declare type StepStatus = 'passed' | 'failed' | 'pending' | 'running' | 'erroneous';
+export declare type StepStatusEnd = 'passed' | 'failed' | 'erroneous';
+/**
+ * Can be used to report on a step run.
+ *
+ * The reporter is an interface (instead of a class) so that the it may be implemented by a class that already extends another class, e.g., the reporter of a test framework.
+ */
+export interface Reporter {
+    config?: ReporterConfig;
+    onStepBegin?(step: Step): Promise<void>;
+    onStepRetry?(step: Step): Promise<void>;
+    onStepEnd?(step: Step): Promise<void>;
+}
+export declare type SnapshotDetailLevel = 
+/**
+ * Details of snapshot, e.g., screenshot or detected elements, may or may not be available depending on if they are required by the step. There are not guarantees made.
+ */
+'required' | 
+/**
+ * Details are available when the step fails, e.g., for debugging. Includes everything of required.
+ */
+'onFailure' | 
+/**
+ * Details are available also when the command is started, e.g., for detecting why a certain element was interacted with. Includes everything of onFailure.
+ */
+'begin' | 
+/**
+ * Details are available always, e.g., before and after a step has been run no matter if it failed or not for debugging.
+ */
+'always';
+export interface ReporterConfig {
+    withScreenshots?: SnapshotDetailLevel;
+    withDetectedElements?: SnapshotDetailLevel;
+}
+export declare const DEFAULT_REPORTER: {
+    config: {
+        readonly withScreenshots: "onFailure";
+        readonly withDetectedElements: "onFailure";
+    };
+    onStepBegin: (_step: Step) => Promise<void>;
+    onStepRetry: (_step: Step) => Promise<void>;
+    onStepEnd: (_step: Step) => Promise<void>;
+};
+export declare class StepReporter {
+    private reporter;
+    currentStep?: askuiStep | undefined;
+    constructor(reporter: Required<Reporter> & {
+        config: Required<ReporterConfig>;
+    });
+    get config(): Required<ReporterConfig>;
+    resetStep(instruction: Instruction): void;
+    onStepBegin(snapshot: Snapshot): Promise<void>;
+    onStepRetry(snapshot: Snapshot, error: Error): Promise<void>;
+    onStepEnd(snapshot: Snapshot, error?: Error): Promise<void>;
+}
+export {};

package/dist/cjs/execution/reporter.js

@@ -0,0 +1,146 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StepReporter = exports.DEFAULT_REPORTER = void 0;
+// retries could be modeled as totally new steps only that we set the retry no at the beginning, it is undefined if there is no retry or 0 --> user could filter out the retries for the report and has to get the last retry to find out when the step actually ended and the snapshot at that point in time
+// Where do I save the error of the first step, what state does the first step have?
+// We could point back from retry to original step, but then we have to save the original step in the retry step, which is not nice
+// retry should be handled like completely new step but the state of the whole state is not failed --> if a passed step can have an error, then this is fine 
+// how do we determine if it is erroneous or failed?
+class askuiStep {
+    constructor(instruction) {
+        this.instruction = instruction;
+        this.runs = [];
+    }
+    get status() {
+        var _a, _b;
+        return (_b = (_a = this.lastRun) === null || _a === void 0 ? void 0 : _a.status) !== null && _b !== void 0 ? _b : 'pending';
+    }
+    get begin() {
+        var _a;
+        return (_a = this.firstRun) === null || _a === void 0 ? void 0 : _a.begin;
+    }
+    get end() {
+        var _a;
+        return (_a = this.lastRun) === null || _a === void 0 ? void 0 : _a.end;
+    }
+    get error() {
+        var _a;
+        return (_a = this.lastRun) === null || _a === void 0 ? void 0 : _a.error;
+    }
+    get retries() {
+        var _a, _b;
+        return (_b = (_a = this.runs) === null || _a === void 0 ? void 0 : _a.slice(1)) !== null && _b !== void 0 ? _b : [];
+    }
+    get retryCount() {
+        var _a, _b;
+        return (_b = (_a = this.retries) === null || _a === void 0 ? void 0 : _a.length) !== null && _b !== void 0 ? _b : 0;
+    }
+    get firstRun() {
+        var _a;
+        return (_a = this.runs) === null || _a === void 0 ? void 0 : _a[0];
+    }
+    get lastRun() {
+        var _a;
+        return (_a = this.runs) === null || _a === void 0 ? void 0 : _a[this.runs.length - 1];
+    }
+    get flaky() {
+        return this.retryCount > 0;
+    }
+    get duration() {
+        if (this.begin !== undefined && this.end !== undefined) {
+            return this.end.createdAt.getTime() - this.begin.createdAt.getTime();
+        }
+        return undefined;
+    }
+    onBegin(snapshot) {
+        this.runs = [...this.runs, {
+                status: 'running',
+                begin: snapshot,
+            }];
+        return this;
+    }
+    onRetry(snapshot, error) {
+        this.onEnd(snapshot, error);
+        this.onBegin(snapshot);
+        return this;
+    }
+    onEnd(snapshot, error) {
+        this.runs = [...this.runs.slice(0, -1), Object.assign(Object.assign({}, this.lastRun), { status: this.determineLastRunStatus(error), end: snapshot, error, duration: snapshot.createdAt.getTime() - this.lastRun.begin.createdAt.getTime() })];
+        return this;
+    }
+    determineLastRunStatus(error) {
+        if (error !== undefined) {
+            return 'failed';
+        }
+        return 'passed';
+    }
+}
+exports.DEFAULT_REPORTER = {
+    config: {
+        withScreenshots: 'onFailure',
+        withDetectedElements: 'onFailure',
+    },
+    onStepBegin: (_step) => {
+        return Promise.resolve();
+    },
+    onStepRetry: (_step) => {
+        return Promise.resolve();
+    },
+    onStepEnd: (_step) => {
+        return Promise.resolve();
+    },
+};
+class StepReporter {
+    constructor(reporter) {
+        this.reporter = reporter;
+    }
+    get config() {
+        return this.reporter.config;
+    }
+    resetStep(instruction) {
+        this.currentStep = new askuiStep(instruction);
+    }
+    onStepBegin(snapshot) {
+        var _a, _b;
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.currentStep === undefined)
+                throw new Error('Cannot begin step if step is undefined.');
+            if (this.currentStep.status !== 'pending')
+                throw new Error('Cannot begin step that is not pending.');
+            this.currentStep.onBegin(snapshot);
+            yield ((_b = (_a = this.reporter).onStepBegin) === null || _b === void 0 ? void 0 : _b.call(_a, this.currentStep));
+        });
+    }
+    onStepRetry(snapshot, error) {
+        var _a, _b;
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.currentStep === undefined)
+                throw new Error('Cannot retry step if step is undefined.');
+            if (this.currentStep.status !== 'running')
+                throw new Error('Cannot retry step that has not been running.');
+            this.currentStep.onRetry(snapshot, error);
+            yield ((_b = (_a = this.reporter).onStepRetry) === null || _b === void 0 ? void 0 : _b.call(_a, this.currentStep));
+        });
+    }
+    onStepEnd(snapshot, error) {
+        var _a, _b;
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.currentStep === undefined)
+                throw new Error('Cannot end step if step is undefined.');
+            if (this.currentStep.status !== 'running')
+                throw new Error('Cannot end step that has not been running.');
+            this.currentStep.onEnd(snapshot, error);
+            yield ((_b = (_a = this.reporter).onStepEnd) === null || _b === void 0 ? void 0 : _b.call(_a, this.currentStep));
+        });
+    }
+}
+exports.StepReporter = StepReporter;

package/dist/cjs/execution/ui-control-client-dependency-builder.d.ts

@@ -0,0 +1,14 @@
+import { ClientArgs, ClientArgsWithDefaults } from './ui-controller-client-interface';
+import { ExecutionRuntime } from './execution-runtime';
+import { StepReporter } from '../core/reporting';
+export declare class UiControlClientDependencyBuilder {
+    private static buildHttpClient;
+    private static buildInferenceClient;
+    private static buildUiControllerClient;
+    static build(clientArgs: ClientArgsWithDefaults): Promise<{
+        executionRuntime: ExecutionRuntime;
+        stepReporter: StepReporter;
+    }>;
+    private static buildReporter;
+    static getClientArgsWithDefaults(clientArgs: ClientArgs): Promise<ClientArgsWithDefaults>;
+}

package/dist/cjs/execution/ui-control-client-dependency-builder.js

@@ -0,0 +1,72 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.UiControlClientDependencyBuilder = void 0;
+const http_client_got_1 = require("../utils/http/http-client-got");
+const ui_controller_client_1 = require("./ui-controller-client");
+const inference_client_1 = require("./inference-client");
+const read_environment_credentials_1 = require("./read-environment-credentials");
+const analytics_1 = require("../utils/analytics");
+const proxy_builder_1 = require("../utils/proxy/proxy-builder");
+const annotation_level_1 = require("./annotation-level");
+const execution_runtime_1 = require("./execution-runtime");
+const reporting_1 = require("../core/reporting");
+class UiControlClientDependencyBuilder {
+    static buildHttpClient(clientArgs) {
+        var _a;
+        return __awaiter(this, void 0, void 0, function* () {
+            const analytics = new analytics_1.Analytics();
+            const analyticsHeaders = yield analytics.getAnalyticsHeaders();
+            const analyticsCookies = yield analytics.getAnalyticsCookies();
+            return new http_client_got_1.HttpClientGot((_a = clientArgs.credentials) === null || _a === void 0 ? void 0 : _a.token, analyticsHeaders, analyticsCookies, clientArgs.proxyAgents);
+        });
+    }
+    static buildInferenceClient(clientArgs) {
+        var _a;
+        return __awaiter(this, void 0, void 0, function* () {
+            const httpClient = yield UiControlClientDependencyBuilder.buildHttpClient(clientArgs);
+            return new inference_client_1.InferenceClient(clientArgs.inferenceServerUrl, httpClient, clientArgs.resize, (_a = clientArgs.credentials) === null || _a === void 0 ? void 0 : _a.workspaceId, clientArgs.modelComposition);
+        });
+    }
+    static buildUiControllerClient(clientArgs) {
+        return new ui_controller_client_1.UiControllerClient(clientArgs.uiControllerUrl);
+    }
+    static build(clientArgs) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const uiControllerClient = UiControlClientDependencyBuilder.buildUiControllerClient(clientArgs);
+            const inferenceClient = yield UiControlClientDependencyBuilder.buildInferenceClient(clientArgs);
+            const stepReporter = new reporting_1.StepReporter(clientArgs.reporter);
+            return {
+                executionRuntime: new execution_runtime_1.ExecutionRuntime(uiControllerClient, inferenceClient, stepReporter),
+                stepReporter,
+            };
+        });
+    }
+    static buildReporter(reporterArg) {
+        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k;
+        return {
+            config: {
+                withScreenshots: (_b = (_a = reporterArg === null || reporterArg === void 0 ? void 0 : reporterArg.config) === null || _a === void 0 ? void 0 : _a.withScreenshots) !== null && _b !== void 0 ? _b : reporting_1.DEFAULT_REPORTER.config.withScreenshots,
+                withDetectedElements: (_d = (_c = reporterArg === null || reporterArg === void 0 ? void 0 : reporterArg.config) === null || _c === void 0 ? void 0 : _c.withDetectedElements) !== null && _d !== void 0 ? _d : reporting_1.DEFAULT_REPORTER.config.withDetectedElements,
+            },
+            onStepBegin: (_f = (_e = reporterArg === null || reporterArg === void 0 ? void 0 : reporterArg.onStepBegin) === null || _e === void 0 ? void 0 : _e.bind(reporterArg)) !== null && _f !== void 0 ? _f : reporting_1.DEFAULT_REPORTER.onStepBegin.bind(reporting_1.DEFAULT_REPORTER),
+            onStepRetry: (_h = (_g = reporterArg === null || reporterArg === void 0 ? void 0 : reporterArg.onStepRetry) === null || _g === void 0 ? void 0 : _g.bind(reporterArg)) !== null && _h !== void 0 ? _h : reporting_1.DEFAULT_REPORTER.onStepRetry.bind(reporting_1.DEFAULT_REPORTER),
+            onStepEnd: (_k = (_j = reporterArg === null || reporterArg === void 0 ? void 0 : reporterArg.onStepEnd) === null || _j === void 0 ? void 0 : _j.bind(reporterArg)) !== null && _k !== void 0 ? _k : reporting_1.DEFAULT_REPORTER.onStepEnd.bind(reporting_1.DEFAULT_REPORTER),
+        };
+    }
+    static getClientArgsWithDefaults(clientArgs) {
+        var _a, _b, _c, _d, _e;
+        return __awaiter(this, void 0, void 0, function* () {
+            return Object.assign(Object.assign({}, clientArgs), { uiControllerUrl: (_a = clientArgs.uiControllerUrl) !== null && _a !== void 0 ? _a : 'http://127.0.0.1:6769', inferenceServerUrl: (_b = clientArgs.inferenceServerUrl) !== null && _b !== void 0 ? _b : 'https://inference.askui.com', annotationLevel: (_c = clientArgs.annotationLevel) !== null && _c !== void 0 ? _c : annotation_level_1.AnnotationLevel.DISABLED, credentials: (_d = clientArgs.credentials) !== null && _d !== void 0 ? _d : (0, read_environment_credentials_1.envCredentials)(), proxyAgents: (_e = clientArgs.proxyAgents) !== null && _e !== void 0 ? _e : yield (0, proxy_builder_1.envProxyAgents)(), reporter: UiControlClientDependencyBuilder.buildReporter(clientArgs.reporter) });
+        });
+    }
+}
+exports.UiControlClientDependencyBuilder = UiControlClientDependencyBuilder;

package/dist/cjs/execution/ui-control-client.d.ts

@@ -1,28 +1,44 @@
-import { CustomElementJson } from '../core/model/test-case-dto';
+import { CustomElementJson } from '../core/model/custom-element-json';
 import { Exec, Executable, FluentFilters, ApiCommands } from './dsl';
 import { UiControllerClientConnectionState } from './ui-controller-client-connection-state';
 import { Annotation } from '../core/annotation/annotation';
 import { AnnotationRequest } from '../core/model/annotation-result/annotation-interface';
-import { ClientArgs } from './ui-controller-client-interface';
 import { DetectedElement } from '../core/model/annotation-result/detected-element';
+import { ClientArgs } from './ui-controller-client-interface';
 export declare class UiControlClient extends ApiCommands {
-    private httpClient;
-    private clientArgs;
-    private workspaceId?;
-    private _uiControllerClient?;
+    private config;
+    private executionRuntime;
+    private stepReporter;
     private constructor();
     static build(clientArgs?: ClientArgs): Promise<UiControlClient>;
-    private get uiControllerClient();
-    private get inferenceClient();
-    private get executionRuntime();
-    private annotateByDefault;
+    /**
+     * Connects to the askui UI Controller.
+     */
     connect(): Promise<UiControllerClientConnectionState>;
+    /**
+     * Disconnects from the askui UI Controller.
+     */
+    disconnect(): void;
+    /**
+     * Disconnects from the askui UI Controller.
+     *
+     * @deprecated Use {@link disconnect} instead.
+     */
+    close(): void;
+    startVideoRecording(): Promise<void>;
+    stopVideoRecording(): Promise<void>;
+    readVideoRecording(): Promise<string>;
+    private shouldWriteAnntotationAfterCommandExecution;
+    private shouldAnnotateAfterCommandExecution;
+    private afterCommandExecution;
     annotate(annotationRequest?: AnnotationRequest): Promise<Annotation>;
     annotateInteractively(): Promise<void>;
     private escapeSeparatorString;
-    fluentCommandExecutor(instruction: string, customElementJson?: CustomElementJson[]): Promise<void>;
+    private buildInstruction;
+    fluentCommandExecutor(instructionString: string, customElementJson?: CustomElementJson[]): Promise<void>;
     getterExecutor(instruction: string, customElementJson?: CustomElementJson[]): Promise<DetectedElement[]>;
     private secretText;
+    private getAndResetSecretText;
     /**
      * Types a text inside the filtered element.
      *
@@ -76,8 +92,4 @@ export declare class UiControlClient extends ApiCommands {
      * @return {Executable}
      */
     waitFor(delayInMs: number): Executable;
-    /**
-    * Closes the connection to the askui UI Controller
-    */
-    close(): void;
 }