askui 0.3.1 → 0.4.0

package/dist/esm/execution/dsl.spec.js

@@ -10,7 +10,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 import { FluentCommand } from './dsl';
 class TestCommand extends FluentCommand {
     // eslint-disable-next-line class-methods-use-this
-    exec(instruction, customElements) {
+    fluentCommandExecutor(instruction, customElements) {
         return __awaiter(this, void 0, void 0, function* () {
             // eslint-disable-next-line no-console
             console.log(`${instruction} ${customElements}`);
@@ -22,14 +22,14 @@ describe('DSL', () => {
     describe('custom element', () => {
         test('should call exec function with zero custom element', () => __awaiter(void 0, void 0, void 0, function* () {
             const underTest = new TestCommand();
-            const testCommandSpy = jest.spyOn(underTest, 'exec');
+            const testCommandSpy = jest.spyOn(underTest, 'fluentCommandExecutor');
             yield underTest.click().button()
                 .exec();
             expect(testCommandSpy).toHaveBeenCalledWith('Click on button', []);
         }));
         test('should call exec function with one custom element', () => __awaiter(void 0, void 0, void 0, function* () {
             const underTest = new TestCommand();
-            const testCommandSpy = jest.spyOn(underTest, 'exec');
+            const testCommandSpy = jest.spyOn(underTest, 'fluentCommandExecutor');
             yield underTest.click().customElement({
                 customImage: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z/C/HgAGgwJ/lK3Q6wAAAABJRU5ErkJggg==',
                 imageCompareFormat: 'grayscale',
@@ -44,7 +44,7 @@ describe('DSL', () => {
         }));
         test('should call exec function with two custom element', () => __awaiter(void 0, void 0, void 0, function* () {
             const underTest = new TestCommand();
-            const testCommandSpy = jest.spyOn(underTest, 'exec');
+            const testCommandSpy = jest.spyOn(underTest, 'fluentCommandExecutor');
             yield underTest.click().customElement({
                 customImage: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z/C/HgAGgwJ/lK3Q6wAAAABJRU5ErkJggg==',
                 imageCompareFormat: 'grayscale',

package/dist/esm/execution/execution-runtime.d.ts

@@ -3,6 +3,7 @@ import { UiControllerClient } from './ui-controller-client';
 import { InferenceClient } from './inference-client';
 import { Annotation } from '../core/annotation/annotation';
 import { CustomElementJson } from '../core/model/test-case-dto/custom-element-json';
+import { DetectedElement } from '../core/model/annotation-result/detected-element';
 export declare class ExecutionRuntime {
     private uiControllerClient;
     private inferenceClient;
@@ -25,5 +26,6 @@ export declare class ExecutionRuntime {
     private predictCommand;
     annotateInteractively(): Promise<void>;
     takeScreenshotIfImageisNotProvided(imagePath?: string): Promise<string>;
+    getDetectedElements(instruction: string, customElementJson?: CustomElementJson[]): Promise<DetectedElement[]>;
     annotateImage(imagePath?: string, customElementJson?: CustomElementJson[]): Promise<Annotation>;
 }

package/dist/esm/execution/execution-runtime.js

@@ -116,7 +116,7 @@ export class ExecutionRuntime {
     annotateInteractively() {
         return __awaiter(this, void 0, void 0, function* () {
             const annotationResponse = yield this.annotateImage();
-            yield this.uiControllerClient.annotateInteractively(annotationResponse.objects, annotationResponse.image);
+            yield this.uiControllerClient.annotateInteractively(annotationResponse.detected_elements, annotationResponse.image);
         });
     }
     takeScreenshotIfImageisNotProvided(imagePath) {
@@ -132,6 +132,16 @@ export class ExecutionRuntime {
             return base64Image;
         });
     }
+    getDetectedElements(instruction, customElementJson) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let customElements = [];
+            const base64Image = yield this.takeScreenshotIfImageisNotProvided();
+            if (customElementJson !== undefined) {
+                customElements = yield CustomElement.fromJsonListWithImagePathOrImage(customElementJson);
+            }
+            return this.inferenceClient.getDetectedElements(instruction, base64Image, customElements);
+        });
+    }
     annotateImage(imagePath, customElementJson) {
         return __awaiter(this, void 0, void 0, function* () {
             let customElements = [];

package/dist/esm/execution/inference-client.d.ts

@@ -2,6 +2,7 @@ import { HttpClientGot } from '../utils/http/http-client-got';
 import { ControlCommand } from '../core/ui-control-commands';
 import { CustomElement } from '../core/model/test-case-dto';
 import { Annotation } from '../core/annotation/annotation';
+import { DetectedElement } from '../core/model/annotation-result/detected-element';
 export declare class InferenceClient {
     baseUrl: string;
     httpClient: HttpClientGot;
@@ -11,6 +12,8 @@ export declare class InferenceClient {
     constructor(baseUrl: string, httpClient: HttpClientGot, workspaceId?: string | undefined, apiVersion?: string);
     isImageRequired(instruction: string): Promise<boolean>;
     private resizeIfNeeded;
+    inference(customElements?: CustomElement[], image?: string, instruction?: string): Promise<ControlCommand | Annotation>;
     predictControlCommand(instruction: string, customElements?: CustomElement[], image?: string): Promise<ControlCommand>;
+    getDetectedElements(instruction: string, image: string, customElements?: CustomElement[]): Promise<DetectedElement[]>;
     predictImageAnnotation(image: string, customElements?: CustomElement[]): Promise<Annotation>;
 }

package/dist/esm/execution/inference-client.js

@@ -8,11 +8,12 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 import urljoin from 'url-join';
-import { ControlCommand } from '../core/ui-control-commands';
+import { ControlCommand, InferenceResponse } from '../core/ui-control-commands';
 import { Annotation } from '../core/annotation/annotation';
 import { resizeBase64ImageWithSameRatio } from '../utils/transformations';
+import { InferenceResponseError } from './inference-response-error';
 export class InferenceClient {
-    constructor(baseUrl, httpClient, workspaceId, apiVersion = 'v2') {
+    constructor(baseUrl, httpClient, workspaceId, apiVersion = 'v3') {
         this.baseUrl = baseUrl;
         this.httpClient = httpClient;
         this.workspaceId = workspaceId;
@@ -39,7 +40,7 @@ export class InferenceClient {
             return resizeBase64ImageWithSameRatio(image);
         });
     }
-    predictControlCommand(instruction, customElements = [], image) {
+    inference(customElements = [], image, instruction) {
         return __awaiter(this, void 0, void 0, function* () {
             const resizedImage = yield this.resizeIfNeeded(customElements, image);
             const httpBody = {
@@ -47,21 +48,36 @@ export class InferenceClient {
                 instruction,
                 customElements,
             };
-            const url = urljoin(this.url, 'predict-command');
+            const url = urljoin(this.url, 'inference');
             const httpResponse = yield this.httpClient.post(url, httpBody);
-            return ControlCommand.fromJson(httpResponse, resizedImage.resizeRatio);
+            return InferenceResponse.fromJson(httpResponse, resizedImage.resizeRatio, image);
+        });
+    }
+    predictControlCommand(instruction, customElements = [], image) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const inferenceResponse = yield this.inference(customElements, image, instruction);
+            if (!(inferenceResponse instanceof ControlCommand)) {
+                throw new InferenceResponseError('Internal Error. Can not execute command');
+            }
+            return inferenceResponse;
+        });
+    }
+    getDetectedElements(instruction, image, customElements = []) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const inferenceResponse = yield this.inference(customElements, image, instruction);
+            if (!(inferenceResponse instanceof Annotation)) {
+                throw new InferenceResponseError('Internal Error. Unable to get the detected elements');
+            }
+            return inferenceResponse.detected_elements;
         });
     }
     predictImageAnnotation(image, customElements = []) {
         return __awaiter(this, void 0, void 0, function* () {
-            const resizedImage = yield this.resizeIfNeeded(customElements, image);
-            const httpBody = {
-                image: resizedImage.base64Image,
-                customElements,
-            };
-            const url = urljoin(this.url, 'annotate', '?format=json');
-            const httpResponse = yield this.httpClient.post(url, httpBody);
-            return Annotation.fromJson(Object.assign(Object.assign({}, httpResponse), { image }), resizedImage.resizeRatio);
+            const inferenceResponse = yield this.inference(customElements, image);
+            if (!(inferenceResponse instanceof Annotation)) {
+                throw new InferenceResponseError('Internal Error. Can not execute annotation');
+            }
+            return inferenceResponse;
         });
     }
 }

package/dist/esm/execution/inference-response-error.d.ts

@@ -0,0 +1,2 @@
+export declare class InferenceResponseError extends Error {
+}

package/dist/esm/execution/inference-response-error.js

@@ -0,0 +1,2 @@
+export class InferenceResponseError extends Error {
+}

package/dist/esm/execution/ui-control-client.d.ts

@@ -1,10 +1,11 @@
 import { CustomElementJson } from '../core/model/test-case-dto';
-import { Exec, Executable, FluentCommand, FluentFilters } from './dsl';
+import { Exec, Executable, FluentFilters, ApiCommands } from './dsl';
 import { UiControllerClientConnectionState } from './ui-controller-client-connection-state';
 import { Annotation } from '../core/annotation/annotation';
 import { AnnotationRequest } from '../core/model/annotation-result/annotation-interface';
 import { ClientArgs } from './ui-controller-client-interface';
-export declare class UiControlClient extends FluentCommand {
+import { DetectedElement } from '../core/model/annotation-result/detected-element';
+export declare class UiControlClient extends ApiCommands {
     private httpClient;
     private clientArgs;
     private workspaceId?;
@@ -19,7 +20,8 @@ export declare class UiControlClient extends FluentCommand {
     annotate(annotationRequest?: AnnotationRequest): Promise<Annotation>;
     annotateInteractively(): Promise<void>;
     private escapeSeparatorString;
-    exec(instruction: string, customElementJson?: CustomElementJson[]): Promise<void>;
+    fluentCommandExecutor(instruction: string, customElementJson?: CustomElementJson[]): Promise<void>;
+    getterExecutor(instruction: string, customElementJson?: CustomElementJson[]): Promise<DetectedElement[]>;
     private secretText;
     /**
      * Types a text inside the filtered element.

package/dist/esm/execution/ui-control-client.js

@@ -8,7 +8,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 import { CustomElement } from '../core/model/test-case-dto';
-import { FluentCommand, Separators, } from './dsl';
+import { ApiCommands, Separators, } from './dsl';
 import { HttpClientGot } from '../utils/http/http-client-got';
 import { UiControllerClient } from './ui-controller-client';
 import { ExecutionRuntime } from './execution-runtime';
@@ -21,7 +21,7 @@ import { UiControlClientError } from './ui-control-client-error';
 import { envCredentials } from './read-environment-credentials';
 import { Analytics } from '../utils/analytics';
 const getClientArgsWithDefaults = (clientArgs = {}) => (Object.assign({ uiControllerUrl: 'http://127.0.0.1:6769', inferenceServerUrl: 'https://inference.askui.com', annotationLevel: AnnotationLevel.DISABLED }, clientArgs));
-export class UiControlClient extends FluentCommand {
+export class UiControlClient extends ApiCommands {
     constructor(httpClient, clientArgs, workspaceId) {
         super();
         this.httpClient = httpClient;
@@ -33,9 +33,10 @@ export class UiControlClient extends FluentCommand {
         return __awaiter(this, void 0, void 0, function* () {
             const analytics = new Analytics();
             const analyticsHeaders = yield analytics.getAnalyticsHeaders();
+            const analyticsCookies = yield analytics.getAnalyticsCookies();
             const cas = getClientArgsWithDefaults(clientArgs);
             const credentialArgs = cas.credentials || envCredentials();
-            const httpClient = new HttpClientGot(credentialArgs === null || credentialArgs === void 0 ? void 0 : credentialArgs.token, analyticsHeaders);
+            const httpClient = new HttpClientGot(credentialArgs === null || credentialArgs === void 0 ? void 0 : credentialArgs.token, analyticsHeaders, analyticsCookies);
             return new UiControlClient(httpClient, cas, credentialArgs === null || credentialArgs === void 0 ? void 0 : credentialArgs.workspaceId);
         });
     }
@@ -92,7 +93,7 @@ export class UiControlClient extends FluentCommand {
     escapeSeparatorString(instruction) {
         return instruction.split(Separators.STRING).join('"');
     }
-    exec(instruction, customElementJson = []) {
+    fluentCommandExecutor(instruction, customElementJson = []) {
         return __awaiter(this, void 0, void 0, function* () {
             const { secretText } = this;
             const customElements = yield CustomElement.fromJsonListWithImagePathOrImage(customElementJson);
@@ -113,6 +114,14 @@ export class UiControlClient extends FluentCommand {
             }
         });
     }
+    getterExecutor(instruction, customElementJson = []) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const customElements = yield CustomElement.fromJsonListWithImagePathOrImage(customElementJson);
+            const stringWithoutSeparators = this.escapeSeparatorString(instruction);
+            logger.debug(stringWithoutSeparators);
+            return this.executionRuntime.getDetectedElements(instruction, customElements);
+        });
+    }
     /**
      * Types a text inside the filtered element.
      *

package/dist/esm/utils/analytics/analytics.d.ts

@@ -2,4 +2,5 @@ import { AnalyticsInterface } from './analytics-interface';
 export declare class Analytics implements AnalyticsInterface {
     private userIdentifier;
     getAnalyticsHeaders(): Promise<Record<string, string>>;
+    getAnalyticsCookies(): Promise<Record<string, string>>;
 }

package/dist/esm/utils/analytics/analytics.js

@@ -28,4 +28,12 @@ export class Analytics {
             return headers;
         });
     }
+    getAnalyticsCookies() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const userID = yield this.userIdentifier.userId();
+            return {
+                'askui-user-id': userID,
+            };
+        });
+    }
 }

package/dist/esm/utils/http/http-client-got.d.ts

@@ -2,10 +2,11 @@ import { OptionsOfJSONResponseBody } from 'got';
 export declare class HttpClientGot {
     readonly token?: string | undefined;
     readonly customHeaders?: Record<string, string> | undefined;
+    private readonly cookies;
     private headers;
-    constructor(token?: string | undefined, customHeaders?: Record<string, string> | undefined);
+    constructor(token?: string | undefined, customHeaders?: Record<string, string> | undefined, cookies?: Record<string, string>);
     private initHeaders;
-    private injectHeaders;
+    private injectHeadersAndCookies;
     post<T>(url: string, data: Record<string | number | symbol, unknown>): Promise<T>;
     get<T>(url: string, options?: OptionsOfJSONResponseBody): Promise<T>;
 }

package/dist/esm/utils/http/http-client-got.js

@@ -8,12 +8,15 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 import got from 'got';
+import { CookieJar } from 'tough-cookie';
+import { logger } from '../../lib';
 import { Credentials } from './credentials';
 import { httpClientErrorHandler } from './custom-errors';
 export class HttpClientGot {
-    constructor(token, customHeaders) {
+    constructor(token, customHeaders, cookies = {}) {
         this.token = token;
         this.customHeaders = customHeaders;
+        this.cookies = cookies;
         this.headers = {};
         this.initHeaders(token, customHeaders);
     }
@@ -21,13 +24,20 @@ export class HttpClientGot {
         const credentials = token ? new Credentials(token) : undefined;
         this.headers = Object.assign(Object.assign({}, (credentials ? { Authorization: `Basic ${credentials === null || credentials === void 0 ? void 0 : credentials.base64Encoded}` } : {})), customHeaders);
     }
-    injectHeaders(options) {
-        return Object.assign(Object.assign({}, options), { headers: this.headers });
+    injectHeadersAndCookies(url, options) {
+        const cookieJar = new CookieJar();
+        Object.keys(this.cookies).map((key) => `${key}=${this.cookies[key]}`).forEach((cookie) => {
+            cookieJar.setCookieSync(cookie, url);
+        });
+        return Object.assign(Object.assign({}, options), { headers: this.headers, cookieJar });
     }
     post(url, data) {
         return __awaiter(this, void 0, void 0, function* () {
-            const options = this.injectHeaders({ json: data, responseType: 'json', throwHttpErrors: false });
-            const { body, statusCode } = yield got.post(url, options);
+            const options = this.injectHeadersAndCookies(url, { json: data, responseType: 'json', throwHttpErrors: false });
+            const { body, statusCode, headers } = yield got.post(url, options);
+            if (headers['deprecation'] !== undefined) {
+                logger.warn(headers['deprecation']);
+            }
             if (statusCode !== 200) {
                 throw httpClientErrorHandler(statusCode, JSON.stringify(body));
             }
@@ -36,7 +46,7 @@ export class HttpClientGot {
     }
     get(url, options = { responseType: 'json' }) {
         return __awaiter(this, void 0, void 0, function* () {
-            const response = yield got.get(url, this.injectHeaders(options));
+            const response = yield got.get(url, this.injectHeadersAndCookies(url, options));
             return response.body;
         });
     }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "askui",
-    "version": "0.3.1",
+    "version": "0.4.0",
     "license": "MIT",
     "author": "askui GmbH <info@askui.com> (http://www.askui.com/)",
     "description": "Reliable, automated end-to-end-testing that depends on what is shown on your screen instead of the technology you are running on",
@@ -58,6 +58,7 @@
         "pino": "7.8.1",
         "pino-pretty": "7.5.3",
         "sharp": "0.30.6",
+        "tough-cookie": "4.1.2",
         "url-join": "4.0.1",
         "wait-port": "0.2.9",
         "ws": "7.4.4",