askui 0.11.6 → 0.12.0

package/dist/esm/core/model/annotation-result/detected-element.js

@@ -8,13 +8,14 @@ import { BoundingBox } from './boundary-box';
 *
 */
 export class DetectedElement {
-    constructor(name, text, bndbox, colors) {
+    constructor(name, text, bndbox, colors, similarityScore) {
         this.name = name;
         this.text = text;
         this.bndbox = bndbox;
         this.colors = colors;
+        this.similarityScore = similarityScore;
     }
     static fromJson(detectedElement, resizeRatio = 1) {
-        return new DetectedElement(detectedElement.name, detectedElement.text, BoundingBox.fromJson(detectedElement.bndbox, resizeRatio), detectedElement.colors ? detectedElement.colors : undefined);
+        return new DetectedElement(detectedElement.name, detectedElement.text, BoundingBox.fromJson(detectedElement.bndbox, resizeRatio), detectedElement.colors || undefined, detectedElement.similarityScore || 0);
     }
 }

package/dist/esm/core/ui-control-commands/action-class-tdk-v1.d.ts

@@ -0,0 +1,19 @@
+export declare enum ActionClassIdTdkV1 {
+    ActionClassID_Undefined = "ActionClassID_Undefined",
+    ActionClassID_Wait = "ActionClassID_Wait",
+    ActionClassID_MouseButton_Press = "ActionClassID_MouseButton_Press",
+    ActionClassID_MouseButton_Release = "ActionClassID_MouseButton_Release",
+    ActionClassID_MouseButton_PressAndRelease = "ActionClassID_MouseButton_PressAndRelease",
+    ActionClassID_MouseWheelScroll = "ActionClassID_MouseWheelScroll",
+    ActionClassID_MouseMove = "ActionClassID_MouseMove",
+    ActionClassID_MouseMove_Delta = "ActionClassID_MouseMove_Delta",
+    ActionClassID_KeyboardKey_Press = "ActionClassID_KeyboardKey_Press",
+    ActionClassID_KeyboardKey_Release = "ActionClassID_KeyboardKey_Release",
+    ActionClassID_KeyboardKey_PressAndRelease = "ActionClassID_KeyboardKey_PressAndRelease",
+    ActionClassID_KeyboardKeys_Press = "ActionClassID_KeyboardKeys_Press",
+    ActionClassID_KeyboardKeys_Release = "ActionClassID_KeyboardKeys_Release",
+    ActionClassID_KeyboardKeys_PressAndRelease = "ActionClassID_KeyboardKeys_PressAndRelease",
+    ActionClassID_KeyboardType_Text = "ActionClassID_KeyboardType_Text",
+    ActionClassID_KeyboardType_UnicodeText = "ActionClassID_KeyboardType_UnicodeText",
+    ActionClassID_RunCommand = "ActionClassID_RunCommand"
+}

package/dist/esm/core/ui-control-commands/action-class-tdk-v1.js

@@ -0,0 +1,20 @@
+export var ActionClassIdTdkV1;
+(function (ActionClassIdTdkV1) {
+    ActionClassIdTdkV1["ActionClassID_Undefined"] = "ActionClassID_Undefined";
+    ActionClassIdTdkV1["ActionClassID_Wait"] = "ActionClassID_Wait";
+    ActionClassIdTdkV1["ActionClassID_MouseButton_Press"] = "ActionClassID_MouseButton_Press";
+    ActionClassIdTdkV1["ActionClassID_MouseButton_Release"] = "ActionClassID_MouseButton_Release";
+    ActionClassIdTdkV1["ActionClassID_MouseButton_PressAndRelease"] = "ActionClassID_MouseButton_PressAndRelease";
+    ActionClassIdTdkV1["ActionClassID_MouseWheelScroll"] = "ActionClassID_MouseWheelScroll";
+    ActionClassIdTdkV1["ActionClassID_MouseMove"] = "ActionClassID_MouseMove";
+    ActionClassIdTdkV1["ActionClassID_MouseMove_Delta"] = "ActionClassID_MouseMove_Delta";
+    ActionClassIdTdkV1["ActionClassID_KeyboardKey_Press"] = "ActionClassID_KeyboardKey_Press";
+    ActionClassIdTdkV1["ActionClassID_KeyboardKey_Release"] = "ActionClassID_KeyboardKey_Release";
+    ActionClassIdTdkV1["ActionClassID_KeyboardKey_PressAndRelease"] = "ActionClassID_KeyboardKey_PressAndRelease";
+    ActionClassIdTdkV1["ActionClassID_KeyboardKeys_Press"] = "ActionClassID_KeyboardKeys_Press";
+    ActionClassIdTdkV1["ActionClassID_KeyboardKeys_Release"] = "ActionClassID_KeyboardKeys_Release";
+    ActionClassIdTdkV1["ActionClassID_KeyboardKeys_PressAndRelease"] = "ActionClassID_KeyboardKeys_PressAndRelease";
+    ActionClassIdTdkV1["ActionClassID_KeyboardType_Text"] = "ActionClassID_KeyboardType_Text";
+    ActionClassIdTdkV1["ActionClassID_KeyboardType_UnicodeText"] = "ActionClassID_KeyboardType_UnicodeText";
+    ActionClassIdTdkV1["ActionClassID_RunCommand"] = "ActionClassID_RunCommand";
+})(ActionClassIdTdkV1 || (ActionClassIdTdkV1 = {}));

package/dist/esm/core/ui-control-commands/action-tdk-v1.d.ts

@@ -0,0 +1,10 @@
+import { ActionParameters } from '../../proto/Askui/API/TDKv1/ActionParameters';
+import { ActionClassIdTdkV1 } from './action-class-tdk-v1';
+import { Action } from './action';
+export declare class ActionTdkV1 {
+    actionClassID: ActionClassIdTdkV1;
+    actionParameters: ActionParameters;
+    constructor(actionClassID: ActionClassIdTdkV1, actionParameters: ActionParameters);
+    static fromJson(json: unknown, _resizeRatio?: number): ActionTdkV1;
+    static fromAction(action: Action, _resizeRatio?: number): ActionTdkV1;
+}

package/dist/esm/core/ui-control-commands/action-tdk-v1.js

@@ -0,0 +1,96 @@
+import { ActionClassIdTdkV1 } from './action-class-tdk-v1';
+const actionMapper = {
+    NO_COMMAND: (_action) => [
+        ActionClassIdTdkV1.ActionClassID_Undefined,
+        {},
+    ],
+    MOUSE_CLICK_LEFT: (_action) => [
+        ActionClassIdTdkV1.ActionClassID_MouseButton_PressAndRelease,
+        { mouseButtonPressAndRelease: { count: 1, mouseButton: 'MouseButton_Left' } },
+    ],
+    MOUSE_CLICK_RIGHT: (_action) => [
+        ActionClassIdTdkV1.ActionClassID_MouseButton_PressAndRelease,
+        { mouseButtonPressAndRelease: { count: 1, mouseButton: 'MouseButton_Right' } },
+    ],
+    MOUSE_CLICK_MIDDLE: (_action) => [
+        ActionClassIdTdkV1.ActionClassID_MouseButton_PressAndRelease,
+        { mouseButtonPressAndRelease: { count: 1, mouseButton: 'MouseButton_Middle' } },
+    ],
+    MOUSE_CLICK_DOUBLE_LEFT: (_action) => [
+        ActionClassIdTdkV1.ActionClassID_MouseButton_PressAndRelease,
+        { mouseButtonPressAndRelease: { count: 2, mouseButton: 'MouseButton_Left' } },
+    ],
+    MOUSE_CLICK_DOUBLE_RIGHT: (_action) => [
+        ActionClassIdTdkV1.ActionClassID_MouseButton_PressAndRelease,
+        { mouseButtonPressAndRelease: { count: 2, mouseButton: 'MouseButton_Right' } },
+    ],
+    MOUSE_CLICK_DOUBLE_MIDDLE: (_action) => [
+        ActionClassIdTdkV1.ActionClassID_MouseButton_PressAndRelease,
+        { mouseButtonPressAndRelease: { count: 2, mouseButton: 'MouseButton_Middle' } },
+    ],
+    MOUSE_SCROLL: (action) => [
+        ActionClassIdTdkV1.ActionClassID_MouseWheelScroll,
+        {
+            mouseWheelScroll: {
+                direction: action.position.x !== 0
+                    ? 'MouseWheelScrollDirection_Horizontal'
+                    : 'MouseWheelScrollDirection_Vertical',
+                delta: action.position.x !== 0 ? action.position.x : action.position.y,
+            },
+        },
+    ],
+    TYPE_TEXT: (_action) => [
+        ActionClassIdTdkV1.ActionClassID_Undefined,
+        {},
+    ],
+    TYPE: (action) => [
+        ActionClassIdTdkV1.ActionClassID_KeyboardType_UnicodeText,
+        { keyboardTypeUnicodeText: { text: action.text } },
+    ],
+    PRESS_KEY_SEQUENCE: (_action) => [
+        ActionClassIdTdkV1.ActionClassID_Undefined,
+        {},
+    ],
+    PRESS_ANDROID_KEY_SEQUENCE: (_action) => [
+        ActionClassIdTdkV1.ActionClassID_Undefined,
+        {},
+    ],
+    PRESS_ANDROID_SINGLE_KEY: (_action) => [
+        ActionClassIdTdkV1.ActionClassID_Undefined,
+        {},
+    ],
+    MOUSE_MOVE_RELATIVELY: (action) => [
+        ActionClassIdTdkV1.ActionClassID_MouseMove_Delta,
+        { mouseMoveDelta: { delta: { x: action.position.x, y: action.position.y } } },
+    ],
+    MOUSE_MOVE: (action) => [
+        ActionClassIdTdkV1.ActionClassID_MouseMove,
+        { mouseMove: { position: { x: action.position.x, y: action.position.y } } },
+    ],
+    MOUSE_DOWN: (_action) => [
+        ActionClassIdTdkV1.ActionClassID_MouseButton_Press,
+        { mouseButtonPress: { mouseButton: 'MouseButton_Left' } },
+    ],
+    MOUSE_UP: (_action) => [
+        ActionClassIdTdkV1.ActionClassID_MouseButton_Release,
+        { mouseButtonPress: { mouseButton: 'MouseButton_Left' } },
+    ],
+    EXECUTE_COMMAND: (_action) => [
+        ActionClassIdTdkV1.ActionClassID_Undefined,
+        {},
+    ],
+};
+export class ActionTdkV1 {
+    constructor(actionClassID, actionParameters) {
+        this.actionClassID = actionClassID;
+        this.actionParameters = actionParameters;
+    }
+    static fromJson(json, _resizeRatio = 1) {
+        const actionTdk = json;
+        return new ActionTdkV1(ActionClassIdTdkV1[actionTdk.actionClassID], actionTdk.actionParameters);
+    }
+    static fromAction(action, _resizeRatio = 1) {
+        const [classId, parameters] = actionMapper[action.inputEvent](action);
+        return new ActionTdkV1(classId, parameters);
+    }
+}

package/dist/esm/execution/dsl.d.ts

@@ -242,7 +242,7 @@ export declare class FluentFilters extends FluentBase {
      *
      * @return {FluentFiltersOrRelations}
      */
-    withText(text: string): FluentFiltersOrRelations;
+    withText(text: string, similarityScore?: number): FluentFiltersOrRelations;
     /**
      * Filters for texts, which match the regex pattern.
      *
@@ -798,7 +798,7 @@ export declare class FluentFiltersCondition extends FluentBase {
      *
      * @return {FluentFiltersOrRelationsCondition}
      */
-    withText(text: string): FluentFiltersOrRelationsCondition;
+    withText(text: string, similarityScore?: number): FluentFiltersOrRelationsCondition;
     /**
      * Filters for texts, which match the regex pattern.
      *
@@ -1799,7 +1799,7 @@ export declare class FluentFiltersGetter extends FluentBase {
      *
      * @return {FluentFiltersOrRelationsGetter}
      */
-    withText(text: string): FluentFiltersOrRelationsGetter;
+    withText(text: string, similarityScore?: number): FluentFiltersOrRelationsGetter;
     /**
      * Filters for texts, which match the regex pattern.
      *

package/dist/esm/execution/dsl.js

@@ -351,11 +351,14 @@ export class FluentFilters extends FluentBase {
      *
      * @return {FluentFiltersOrRelations}
      */
-    withText(text) {
+    withText(text, similarityScore = 70) {
         this._textStr = '';
         this._textStr += 'with';
         this._textStr += ' text';
         this._textStr += ` ${Separators.STRING}${text}${Separators.STRING}`;
+        if (similarityScore !== undefined) {
+            this._textStr += ` that matches to ${similarityScore} %`;
+        }
         return new FluentFiltersOrRelations(this);
     }
     /**
@@ -1069,11 +1072,14 @@ export class FluentFiltersCondition extends FluentBase {
      *
      * @return {FluentFiltersOrRelationsCondition}
      */
-    withText(text) {
+    withText(text, similarityScore = 70) {
         this._textStr = '';
         this._textStr += 'with';
         this._textStr += ' text';
         this._textStr += ` ${Separators.STRING}${text}${Separators.STRING}`;
+        if (similarityScore !== undefined) {
+            this._textStr += ` that matches to ${similarityScore} %`;
+        }
         return new FluentFiltersOrRelationsCondition(this);
     }
     /**
@@ -2408,11 +2414,14 @@ export class FluentFiltersGetter extends FluentBase {
      *
      * @return {FluentFiltersOrRelationsGetter}
      */
-    withText(text) {
+    withText(text, similarityScore = 70) {
         this._textStr = '';
         this._textStr += 'with';
         this._textStr += ' text';
         this._textStr += ` ${Separators.STRING}${text}${Separators.STRING}`;
+        if (similarityScore !== undefined) {
+            this._textStr += ` that matches to ${similarityScore} %`;
+        }
         return new FluentFiltersOrRelationsGetter(this);
     }
     /**

package/dist/esm/execution/ui-control-client-error.d.ts

@@ -1,3 +1,2 @@
 export declare class UiControlClientError extends Error {
-    constructor(message: string);
 }

package/dist/esm/execution/ui-control-client-error.js

@@ -1,6 +1,2 @@
 export class UiControlClientError extends Error {
-    constructor(message) {
-        super(message);
-        this.name = 'UiControlClientError';
-    }
 }

package/dist/esm/execution/ui-controller-client-facade.d.ts

@@ -0,0 +1,15 @@
+import { CaptureScreenshotResponse, GetProcessPidResponse, StartRecordingResponse, StopRecordingResponse, ReadRecordingPartResponse, InteractiveAnnotationResponse, ControlResponse, ControlFailureResponse } from '../core/runner-protocol';
+import { DetectedElement } from '../core/model/annotation-result/detected-element';
+import { ControlCommand } from '../core/ui-control-commands/control-command';
+import { UiControllerClientConnectionState } from './ui-controller-client-connection-state';
+export interface UiControllerFacade {
+    connect(): Promise<UiControllerClientConnectionState>;
+    disconnect(): Promise<void>;
+    requestScreenshot(): Promise<CaptureScreenshotResponse>;
+    getServerPid(): Promise<GetProcessPidResponse>;
+    startVideoRecording(): Promise<StartRecordingResponse>;
+    stopVideoRecording(): Promise<StopRecordingResponse>;
+    readVideoRecording(): Promise<ReadRecordingPartResponse>;
+    annotateInteractively(boundingBoxes: DetectedElement[], imageString: string): Promise<InteractiveAnnotationResponse>;
+    requestControl(controlCommand: ControlCommand): Promise<ControlResponse | ControlFailureResponse>;
+}

package/dist/esm/execution/ui-controller-client-grpc.d.ts

@@ -0,0 +1,24 @@
+import { DetectedElement } from '../core/model/annotation-result/detected-element';
+import { CaptureScreenshotResponse, GetProcessPidResponse, StartRecordingResponse, StopRecordingResponse, ReadRecordingPartResponse, InteractiveAnnotationResponse, ControlResponse } from '../core/runner-protocol';
+import { ControlCommand } from '../core/ui-control-commands';
+import { UiControllerClientConnectionState } from './ui-controller-client-connection-state';
+import { UiControllerFacade } from './ui-controller-client-facade';
+import { SessionInfo } from '../proto/Askui/API/TDKv1/SessionInfo';
+export declare class UiControllerClientGrpc implements UiControllerFacade {
+    url: string;
+    private grpcClient;
+    private sessionGUID;
+    sessionInfo: SessionInfo | null;
+    constructor(url: string);
+    connect(): Promise<UiControllerClientConnectionState>;
+    disconnect(): Promise<void>;
+    requestScreenshot(): Promise<CaptureScreenshotResponse>;
+    getServerPid(): Promise<GetProcessPidResponse>;
+    startVideoRecording(): Promise<StartRecordingResponse>;
+    stopVideoRecording(): Promise<StopRecordingResponse>;
+    readVideoRecording(): Promise<ReadRecordingPartResponse>;
+    annotateInteractively(_boundingBoxes: DetectedElement[], _imageString: string): Promise<InteractiveAnnotationResponse>;
+    requestControl(controlCommand: ControlCommand): Promise<ControlResponse>;
+    private runRecordedFunction;
+    private pollActions;
+}

package/dist/esm/execution/ui-controller-client-grpc.js

@@ -0,0 +1,133 @@
+/* eslint-disable */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import * as grpc from '@grpc/grpc-js';
+import * as protoLoader from '@grpc/proto-loader';
+import { v4 as uuidv4 } from 'uuid';
+import Jimp from 'jimp';
+import { UiControllerClientConnectionState } from './ui-controller-client-connection-state';
+import path from 'path';
+export class UiControllerClientGrpc {
+    constructor(url) {
+        this.url = url;
+        const packageDefinition = protoLoader.loadSync(path.join(__dirname, '../proto/tdk-v1.proto'));
+        const proto = grpc.loadPackageDefinition(packageDefinition);
+        this.grpcClient = new proto.Askui.API.TDKv1.ControllerAPI("127.0.0.1:23000", grpc.credentials.createInsecure());
+        this.sessionGUID = uuidv4();
+        this.sessionInfo = null;
+    }
+    connect() {
+        return new Promise((resolve, reject) => {
+            this.grpcClient.StartSession({
+                sessionGUID: this.sessionGUID,
+                immediateExecution: true
+            }, (error, response) => {
+                if (!error && response != undefined) {
+                    this.sessionInfo = response.sessionInfo;
+                    resolve(UiControllerClientConnectionState.CONNECTED);
+                }
+                reject(UiControllerClientConnectionState.ERROR);
+            });
+        });
+    }
+    disconnect() {
+        return new Promise((resolve, reject) => {
+            this.grpcClient.EndSession({
+                sessionInfo: this.sessionInfo,
+            }, (error, _response) => {
+                if (!error) {
+                    resolve();
+                }
+                reject();
+            });
+        });
+    }
+    requestScreenshot() {
+        return new Promise((resolve, reject) => {
+            this.grpcClient.CaptureScreen({ sessionInfo: this.sessionInfo, captureArea: { displayID: 0 } }, (error, response) => __awaiter(this, void 0, void 0, function* () {
+                if (!error && (response === null || response === void 0 ? void 0 : response.bitmap) !== undefined && (response === null || response === void 0 ? void 0 : response.bitmap) !== null) {
+                    const image = new Jimp(response.bitmap.width, response.bitmap.height);
+                    let red;
+                    let green;
+                    let blue;
+                    //
+                    response.bitmap.data.forEach((byte, i) => {
+                        switch (i % 4) {
+                            case 0: return blue = byte;
+                            case 1: return green = byte;
+                            case 2: return red = byte;
+                            default:
+                                image.bitmap.data[i - 3] = red;
+                                image.bitmap.data[i - 2] = green;
+                                image.bitmap.data[i - 1] = blue;
+                                image.bitmap.data[i] = 255;
+                                return;
+                        }
+                    });
+                    resolve({
+                        data: { image: yield image.getBase64Async('image/png') },
+                        msgName: 'CAPTURE_SCREENSHOT_RESPONSE',
+                    });
+                }
+                reject({
+                    data: { error: 'Error screenshot' },
+                    msgName: 'CAPTURE_SCREENSHOT_RESPONSE',
+                });
+            }));
+        });
+    }
+    getServerPid() {
+        throw new Error('Method not implemented.');
+    }
+    startVideoRecording() {
+        throw new Error('Method not implemented.');
+    }
+    stopVideoRecording() {
+        throw new Error('Method not implemented.');
+    }
+    readVideoRecording() {
+        throw new Error('Method not implemented.');
+    }
+    annotateInteractively(_boundingBoxes, _imageString) {
+        throw new Error('Method not implemented.');
+    }
+    requestControl(controlCommand) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const actionId = yield this.runRecordedFunction(controlCommand);
+            console.log(yield this.pollActions(actionId));
+            return Promise.resolve({
+                msgName: 'CONTROL_RESPONSE',
+                data: { msg: 'e' },
+            });
+        });
+    }
+    runRecordedFunction(_controlCommand) {
+        return new Promise((resolve, reject) => {
+            this.grpcClient.RunRecordedAction({ actionClassID: 12, sessionInfo: this.sessionInfo, actionParameters: { mouseMove: { position: { x: 23, y: 23 } } } }, (error, response) => {
+                if (!error && response != undefined) {
+                    resolve(response.actionID);
+                }
+                reject(error);
+            });
+        });
+    }
+    pollActions(actionID) {
+        return new Promise((resolve, reject) => {
+            this.grpcClient.Poll({ sessionInfo: this.sessionInfo, pollEventID: 'PollEventID_ActionFinished' }, (error, response) => {
+                var _a, _b, _c, _d;
+                if (!error && response != undefined) {
+                    console.log((_b = (_a = response.pollEventParameters) === null || _a === void 0 ? void 0 : _a.actionFinished) === null || _b === void 0 ? void 0 : _b.actionID);
+                    resolve(((_d = (_c = response.pollEventParameters) === null || _c === void 0 ? void 0 : _c.actionFinished) === null || _d === void 0 ? void 0 : _d.actionID) == actionID);
+                }
+                reject();
+            });
+        });
+    }
+}

package/dist/esm/execution/ui-controller-client-tdk-v1.d.ts

@@ -0,0 +1,26 @@
+import { DetectedElement } from '../core/model/annotation-result/detected-element';
+import { CaptureScreenshotResponse, GetProcessPidResponse, StartRecordingResponse, StopRecordingResponse, ReadRecordingPartResponse, InteractiveAnnotationResponse, ControlResponse, ControlFailureResponse } from '../core/runner-protocol';
+import { ControlCommand } from '../core/ui-control-commands';
+import { UiControllerClientConnectionState } from './ui-controller-client-connection-state';
+import { UiControllerFacade } from './ui-controller-client-facade';
+import { SessionInfo } from '../proto/Askui/API/TDKv1/SessionInfo';
+export declare class UiControllerClientTdkV1 implements UiControllerFacade {
+    url: string;
+    private grpcClient;
+    private sessionGUID;
+    sessionInfo: SessionInfo | null;
+    constructor(url: string);
+    connect(): Promise<UiControllerClientConnectionState>;
+    disconnect(): Promise<void>;
+    requestScreenshot(): Promise<CaptureScreenshotResponse>;
+    getServerPid(): Promise<GetProcessPidResponse>;
+    startVideoRecording(): Promise<StartRecordingResponse>;
+    stopVideoRecording(): Promise<StopRecordingResponse>;
+    readVideoRecording(): Promise<ReadRecordingPartResponse>;
+    annotateInteractively(_boundingBoxes: DetectedElement[], _imageString: string): Promise<InteractiveAnnotationResponse>;
+    requestControl(controlCommand: ControlCommand): Promise<ControlResponse | ControlFailureResponse>;
+    private runSingleAction;
+    private runRecordedFunction;
+    private pollActions;
+    private pollWithRetry;
+}

package/dist/esm/execution/ui-controller-client-tdk-v1.js

@@ -0,0 +1,195 @@
+/* eslint-disable */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import * as grpc from '@grpc/grpc-js';
+import * as protoLoader from '@grpc/proto-loader';
+import { v4 as uuidv4 } from 'uuid';
+import Jimp from 'jimp';
+import { UiControllerClientConnectionState } from './ui-controller-client-connection-state';
+import path from 'path';
+import { PollEventID } from '../proto/Askui/API/TDKv1/PollEventID';
+import { ActionTdkV1 } from '../core/ui-control-commands/action-tdk-v1';
+export class UiControllerClientTdkV1 {
+    constructor(url) {
+        this.url = url;
+        const packageDefinition = protoLoader.loadSync(path.join(__dirname, '../proto/tdk-v1.proto'));
+        const proto = grpc.loadPackageDefinition(packageDefinition);
+        this.grpcClient = new proto.Askui.API.TDKv1.ControllerAPI("127.0.0.1:23000", grpc.credentials.createInsecure());
+        this.sessionGUID = uuidv4();
+        this.sessionInfo = null;
+    }
+    connect() {
+        return new Promise((resolve, reject) => {
+            this.grpcClient.StartSession({
+                sessionGUID: this.sessionGUID,
+                immediateExecution: true
+            }, (error, response) => {
+                if (!error && response != undefined) {
+                    this.sessionInfo = response.sessionInfo;
+                    resolve(UiControllerClientConnectionState.CONNECTED);
+                }
+                reject(UiControllerClientConnectionState.ERROR);
+            });
+        });
+    }
+    disconnect() {
+        return new Promise((resolve, reject) => {
+            this.grpcClient.EndSession({
+                sessionInfo: this.sessionInfo,
+            }, (error, _response) => {
+                if (!error) {
+                    resolve();
+                    this.grpcClient.close();
+                }
+                reject();
+            });
+        });
+    }
+    requestScreenshot() {
+        return new Promise((resolve, reject) => {
+            this.grpcClient.CaptureScreen({ sessionInfo: this.sessionInfo, captureArea: { displayID: 1 } }, (error, response) => __awaiter(this, void 0, void 0, function* () {
+                if (!error && (response === null || response === void 0 ? void 0 : response.bitmap) !== undefined && (response === null || response === void 0 ? void 0 : response.bitmap) !== null) {
+                    const image = new Jimp(response.bitmap.width, response.bitmap.height);
+                    let red;
+                    let green;
+                    let blue;
+                    //
+                    response.bitmap.data.forEach((byte, i) => {
+                        switch (i % 4) {
+                            case 0: return blue = byte;
+                            case 1: return green = byte;
+                            case 2: return red = byte;
+                            default:
+                                image.bitmap.data[i - 3] = red;
+                                image.bitmap.data[i - 2] = green;
+                                image.bitmap.data[i - 1] = blue;
+                                image.bitmap.data[i] = 255;
+                                return;
+                        }
+                    });
+                    resolve({
+                        data: { image: yield image.getBase64Async('image/png') },
+                        msgName: 'CAPTURE_SCREENSHOT_RESPONSE',
+                    });
+                }
+                reject({
+                    data: { error: `${error}` },
+                    msgName: 'CAPTURE_SCREENSHOT_RESPONSE',
+                });
+            }));
+        });
+    }
+    getServerPid() {
+        throw new Error('Method not implemented.');
+    }
+    startVideoRecording() {
+        throw new Error('Method not implemented.');
+    }
+    stopVideoRecording() {
+        throw new Error('Method not implemented.');
+    }
+    readVideoRecording() {
+        throw new Error('Method not implemented.');
+    }
+    annotateInteractively(_boundingBoxes, _imageString) {
+        throw new Error('Method not implemented.');
+    }
+    requestControl(controlCommand) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                for (const action of controlCommand.actions) {
+                    yield this.runSingleAction(ActionTdkV1.fromAction(action));
+                }
+                return Promise.resolve({
+                    msgName: 'CONTROL_RESPONSE',
+                    data: { msg: 'e' },
+                });
+            }
+            catch (error) {
+                return Promise.resolve({
+                    msgName: 'CONTROL_FAILURE_RESPONSE',
+                    data: { error: `${error}` },
+                });
+            }
+        });
+    }
+    runSingleAction(request) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const actionId = yield this.runRecordedFunction(request);
+                return Promise.resolve(this.pollActions(actionId));
+            }
+            catch (error) {
+                Promise.reject(error);
+            }
+        });
+    }
+    runRecordedFunction(request) {
+        return new Promise((resolve, reject) => {
+            this.grpcClient.RunRecordedAction(request, (error, response) => {
+                if (!error && response != undefined) {
+                    resolve(response.actionID);
+                }
+                reject(error);
+            });
+        });
+    }
+    pollActions(actionID) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const pollRequest = {
+                sessionInfo: this.sessionInfo,
+                pollEventID: PollEventID.PollEventID_ActionFinished
+            };
+            try {
+                return Promise.race([
+                    this.pollWithRetry(pollRequest, actionID),
+                    new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout')), 30000)) // 30-second timeout
+                ]);
+            }
+            catch (error) {
+                return Promise.reject(error);
+            }
+        });
+    }
+    pollWithRetry(request, actionID) {
+        var _a, _b;
+        return __awaiter(this, void 0, void 0, function* () {
+            const maxRetries = 100;
+            for (let retryCount = 0; retryCount <= maxRetries; retryCount++) {
+                try {
+                    const response = yield new Promise((resolve, reject) => {
+                        this.grpcClient.Poll(request, (error, response) => {
+                            if (error || response === undefined) {
+                                reject(error);
+                            }
+                            else {
+                                resolve(response);
+                            }
+                        });
+                    });
+                    if (((_b = (_a = response.pollEventParameters) === null || _a === void 0 ? void 0 : _a.actionFinished) === null || _b === void 0 ? void 0 : _b.actionID) === actionID) {
+                        return Promise.resolve();
+                    }
+                    else {
+                        throw new Error('Invalid response');
+                    }
+                }
+                catch (error) {
+                    if (retryCount === maxRetries) {
+                        throw error;
+                    }
+                    yield new Promise(resolve => setTimeout(resolve, 250)); // Wait for 1 second before retrying
+                    console.log("Retry ?!");
+                }
+            }
+            throw new Error('Max retries exceeded');
+        });
+    }
+}