askui 0.11.1 → 0.11.3

package/dist/cjs/core/ui-control-commands/action-class-tdk-v1.d.ts

@@ -0,0 +1,19 @@
+export declare enum ActionClassIdTdkV1 {
+    ActionClassID_Undefined = "ActionClassID_Undefined",
+    ActionClassID_Wait = "ActionClassID_Wait",
+    ActionClassID_MouseButton_Press = "ActionClassID_MouseButton_Press",
+    ActionClassID_MouseButton_Release = "ActionClassID_MouseButton_Release",
+    ActionClassID_MouseButton_PressAndRelease = "ActionClassID_MouseButton_PressAndRelease",
+    ActionClassID_MouseWheelScroll = "ActionClassID_MouseWheelScroll",
+    ActionClassID_MouseMove = "ActionClassID_MouseMove",
+    ActionClassID_MouseMove_Delta = "ActionClassID_MouseMove_Delta",
+    ActionClassID_KeyboardKey_Press = "ActionClassID_KeyboardKey_Press",
+    ActionClassID_KeyboardKey_Release = "ActionClassID_KeyboardKey_Release",
+    ActionClassID_KeyboardKey_PressAndRelease = "ActionClassID_KeyboardKey_PressAndRelease",
+    ActionClassID_KeyboardKeys_Press = "ActionClassID_KeyboardKeys_Press",
+    ActionClassID_KeyboardKeys_Release = "ActionClassID_KeyboardKeys_Release",
+    ActionClassID_KeyboardKeys_PressAndRelease = "ActionClassID_KeyboardKeys_PressAndRelease",
+    ActionClassID_KeyboardType_Text = "ActionClassID_KeyboardType_Text",
+    ActionClassID_KeyboardType_UnicodeText = "ActionClassID_KeyboardType_UnicodeText",
+    ActionClassID_RunCommand = "ActionClassID_RunCommand"
+}

package/dist/cjs/core/ui-control-commands/action-class-tdk-v1.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ActionClassIdTdkV1 = void 0;
+var ActionClassIdTdkV1;
+(function (ActionClassIdTdkV1) {
+    ActionClassIdTdkV1["ActionClassID_Undefined"] = "ActionClassID_Undefined";
+    ActionClassIdTdkV1["ActionClassID_Wait"] = "ActionClassID_Wait";
+    ActionClassIdTdkV1["ActionClassID_MouseButton_Press"] = "ActionClassID_MouseButton_Press";
+    ActionClassIdTdkV1["ActionClassID_MouseButton_Release"] = "ActionClassID_MouseButton_Release";
+    ActionClassIdTdkV1["ActionClassID_MouseButton_PressAndRelease"] = "ActionClassID_MouseButton_PressAndRelease";
+    ActionClassIdTdkV1["ActionClassID_MouseWheelScroll"] = "ActionClassID_MouseWheelScroll";
+    ActionClassIdTdkV1["ActionClassID_MouseMove"] = "ActionClassID_MouseMove";
+    ActionClassIdTdkV1["ActionClassID_MouseMove_Delta"] = "ActionClassID_MouseMove_Delta";
+    ActionClassIdTdkV1["ActionClassID_KeyboardKey_Press"] = "ActionClassID_KeyboardKey_Press";
+    ActionClassIdTdkV1["ActionClassID_KeyboardKey_Release"] = "ActionClassID_KeyboardKey_Release";
+    ActionClassIdTdkV1["ActionClassID_KeyboardKey_PressAndRelease"] = "ActionClassID_KeyboardKey_PressAndRelease";
+    ActionClassIdTdkV1["ActionClassID_KeyboardKeys_Press"] = "ActionClassID_KeyboardKeys_Press";
+    ActionClassIdTdkV1["ActionClassID_KeyboardKeys_Release"] = "ActionClassID_KeyboardKeys_Release";
+    ActionClassIdTdkV1["ActionClassID_KeyboardKeys_PressAndRelease"] = "ActionClassID_KeyboardKeys_PressAndRelease";
+    ActionClassIdTdkV1["ActionClassID_KeyboardType_Text"] = "ActionClassID_KeyboardType_Text";
+    ActionClassIdTdkV1["ActionClassID_KeyboardType_UnicodeText"] = "ActionClassID_KeyboardType_UnicodeText";
+    ActionClassIdTdkV1["ActionClassID_RunCommand"] = "ActionClassID_RunCommand";
+})(ActionClassIdTdkV1 = exports.ActionClassIdTdkV1 || (exports.ActionClassIdTdkV1 = {}));

package/dist/cjs/core/ui-control-commands/action-tdk-v1.d.ts

@@ -0,0 +1,10 @@
+import { ActionParameters } from '../../proto/Askui/API/TDKv1/ActionParameters';
+import { ActionClassIdTdkV1 } from './action-class-tdk-v1';
+import { Action } from './action';
+export declare class ActionTdkV1 {
+    actionClassID: ActionClassIdTdkV1;
+    actionParameters: ActionParameters;
+    constructor(actionClassID: ActionClassIdTdkV1, actionParameters: ActionParameters);
+    static fromJson(json: unknown, _resizeRatio?: number): ActionTdkV1;
+    static fromAction(action: Action, _resizeRatio?: number): ActionTdkV1;
+}

package/dist/cjs/core/ui-control-commands/action-tdk-v1.js

@@ -0,0 +1,100 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ActionTdkV1 = void 0;
+const action_class_tdk_v1_1 = require("./action-class-tdk-v1");
+const actionMapper = {
+    NO_COMMAND: (_action) => [
+        action_class_tdk_v1_1.ActionClassIdTdkV1.ActionClassID_Undefined,
+        {},
+    ],
+    MOUSE_CLICK_LEFT: (_action) => [
+        action_class_tdk_v1_1.ActionClassIdTdkV1.ActionClassID_MouseButton_PressAndRelease,
+        { mouseButtonPressAndRelease: { count: 1, mouseButton: 'MouseButton_Left' } },
+    ],
+    MOUSE_CLICK_RIGHT: (_action) => [
+        action_class_tdk_v1_1.ActionClassIdTdkV1.ActionClassID_MouseButton_PressAndRelease,
+        { mouseButtonPressAndRelease: { count: 1, mouseButton: 'MouseButton_Right' } },
+    ],
+    MOUSE_CLICK_MIDDLE: (_action) => [
+        action_class_tdk_v1_1.ActionClassIdTdkV1.ActionClassID_MouseButton_PressAndRelease,
+        { mouseButtonPressAndRelease: { count: 1, mouseButton: 'MouseButton_Middle' } },
+    ],
+    MOUSE_CLICK_DOUBLE_LEFT: (_action) => [
+        action_class_tdk_v1_1.ActionClassIdTdkV1.ActionClassID_MouseButton_PressAndRelease,
+        { mouseButtonPressAndRelease: { count: 2, mouseButton: 'MouseButton_Left' } },
+    ],
+    MOUSE_CLICK_DOUBLE_RIGHT: (_action) => [
+        action_class_tdk_v1_1.ActionClassIdTdkV1.ActionClassID_MouseButton_PressAndRelease,
+        { mouseButtonPressAndRelease: { count: 2, mouseButton: 'MouseButton_Right' } },
+    ],
+    MOUSE_CLICK_DOUBLE_MIDDLE: (_action) => [
+        action_class_tdk_v1_1.ActionClassIdTdkV1.ActionClassID_MouseButton_PressAndRelease,
+        { mouseButtonPressAndRelease: { count: 2, mouseButton: 'MouseButton_Middle' } },
+    ],
+    MOUSE_SCROLL: (action) => [
+        action_class_tdk_v1_1.ActionClassIdTdkV1.ActionClassID_MouseWheelScroll,
+        {
+            mouseWheelScroll: {
+                direction: action.position.x !== 0
+                    ? 'MouseWheelScrollDirection_Horizontal'
+                    : 'MouseWheelScrollDirection_Vertical',
+                delta: action.position.x !== 0 ? action.position.x : action.position.y,
+            },
+        },
+    ],
+    TYPE_TEXT: (_action) => [
+        action_class_tdk_v1_1.ActionClassIdTdkV1.ActionClassID_Undefined,
+        {},
+    ],
+    TYPE: (action) => [
+        action_class_tdk_v1_1.ActionClassIdTdkV1.ActionClassID_KeyboardType_UnicodeText,
+        { keyboardTypeUnicodeText: { text: action.text } },
+    ],
+    PRESS_KEY_SEQUENCE: (_action) => [
+        action_class_tdk_v1_1.ActionClassIdTdkV1.ActionClassID_Undefined,
+        {},
+    ],
+    PRESS_ANDROID_KEY_SEQUENCE: (_action) => [
+        action_class_tdk_v1_1.ActionClassIdTdkV1.ActionClassID_Undefined,
+        {},
+    ],
+    PRESS_ANDROID_SINGLE_KEY: (_action) => [
+        action_class_tdk_v1_1.ActionClassIdTdkV1.ActionClassID_Undefined,
+        {},
+    ],
+    MOUSE_MOVE_RELATIVELY: (action) => [
+        action_class_tdk_v1_1.ActionClassIdTdkV1.ActionClassID_MouseMove_Delta,
+        { mouseMoveDelta: { delta: { x: action.position.x, y: action.position.y } } },
+    ],
+    MOUSE_MOVE: (action) => [
+        action_class_tdk_v1_1.ActionClassIdTdkV1.ActionClassID_MouseMove,
+        { mouseMove: { position: { x: action.position.x, y: action.position.y } } },
+    ],
+    MOUSE_DOWN: (_action) => [
+        action_class_tdk_v1_1.ActionClassIdTdkV1.ActionClassID_MouseButton_Press,
+        { mouseButtonPress: { mouseButton: 'MouseButton_Left' } },
+    ],
+    MOUSE_UP: (_action) => [
+        action_class_tdk_v1_1.ActionClassIdTdkV1.ActionClassID_MouseButton_Release,
+        { mouseButtonPress: { mouseButton: 'MouseButton_Left' } },
+    ],
+    EXECUTE_COMMAND: (_action) => [
+        action_class_tdk_v1_1.ActionClassIdTdkV1.ActionClassID_Undefined,
+        {},
+    ],
+};
+class ActionTdkV1 {
+    constructor(actionClassID, actionParameters) {
+        this.actionClassID = actionClassID;
+        this.actionParameters = actionParameters;
+    }
+    static fromJson(json, _resizeRatio = 1) {
+        const actionTdk = json;
+        return new ActionTdkV1(action_class_tdk_v1_1.ActionClassIdTdkV1[actionTdk.actionClassID], actionTdk.actionParameters);
+    }
+    static fromAction(action, _resizeRatio = 1) {
+        const [classId, parameters] = actionMapper[action.inputEvent](action);
+        return new ActionTdkV1(classId, parameters);
+    }
+}
+exports.ActionTdkV1 = ActionTdkV1;

package/dist/cjs/execution/ui-controller-client-facade.d.ts

@@ -0,0 +1,15 @@
+import { CaptureScreenshotResponse, GetProcessPidResponse, StartRecordingResponse, StopRecordingResponse, ReadRecordingPartResponse, InteractiveAnnotationResponse, ControlResponse, ControlFailureResponse } from '../core/runner-protocol';
+import { DetectedElement } from '../core/model/annotation-result/detected-element';
+import { ControlCommand } from '../core/ui-control-commands/control-command';
+import { UiControllerClientConnectionState } from './ui-controller-client-connection-state';
+export interface UiControllerFacade {
+    connect(): Promise<UiControllerClientConnectionState>;
+    disconnect(): Promise<void>;
+    requestScreenshot(): Promise<CaptureScreenshotResponse>;
+    getServerPid(): Promise<GetProcessPidResponse>;
+    startVideoRecording(): Promise<StartRecordingResponse>;
+    stopVideoRecording(): Promise<StopRecordingResponse>;
+    readVideoRecording(): Promise<ReadRecordingPartResponse>;
+    annotateInteractively(boundingBoxes: DetectedElement[], imageString: string): Promise<InteractiveAnnotationResponse>;
+    requestControl(controlCommand: ControlCommand): Promise<ControlResponse | ControlFailureResponse>;
+}

package/dist/cjs/execution/ui-controller-client-grpc.d.ts

@@ -0,0 +1,24 @@
+import { DetectedElement } from '../core/model/annotation-result/detected-element';
+import { CaptureScreenshotResponse, GetProcessPidResponse, StartRecordingResponse, StopRecordingResponse, ReadRecordingPartResponse, InteractiveAnnotationResponse, ControlResponse } from '../core/runner-protocol';
+import { ControlCommand } from '../core/ui-control-commands';
+import { UiControllerClientConnectionState } from './ui-controller-client-connection-state';
+import { UiControllerFacade } from './ui-controller-client-facade';
+import { SessionInfo } from '../proto/Askui/API/TDKv1/SessionInfo';
+export declare class UiControllerClientGrpc implements UiControllerFacade {
+    url: string;
+    private grpcClient;
+    private sessionGUID;
+    sessionInfo: SessionInfo | null;
+    constructor(url: string);
+    connect(): Promise<UiControllerClientConnectionState>;
+    disconnect(): Promise<void>;
+    requestScreenshot(): Promise<CaptureScreenshotResponse>;
+    getServerPid(): Promise<GetProcessPidResponse>;
+    startVideoRecording(): Promise<StartRecordingResponse>;
+    stopVideoRecording(): Promise<StopRecordingResponse>;
+    readVideoRecording(): Promise<ReadRecordingPartResponse>;
+    annotateInteractively(_boundingBoxes: DetectedElement[], _imageString: string): Promise<InteractiveAnnotationResponse>;
+    requestControl(controlCommand: ControlCommand): Promise<ControlResponse>;
+    private runRecordedFunction;
+    private pollActions;
+}

package/dist/cjs/execution/ui-controller-client-grpc.js

@@ -0,0 +1,159 @@
+"use strict";
+/* eslint-disable */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.UiControllerClientGrpc = void 0;
+const grpc = __importStar(require("@grpc/grpc-js"));
+const protoLoader = __importStar(require("@grpc/proto-loader"));
+const uuid_1 = require("uuid");
+const jimp_1 = __importDefault(require("jimp"));
+const ui_controller_client_connection_state_1 = require("./ui-controller-client-connection-state");
+const path_1 = __importDefault(require("path"));
+class UiControllerClientGrpc {
+    constructor(url) {
+        this.url = url;
+        const packageDefinition = protoLoader.loadSync(path_1.default.join(__dirname, '../proto/tdk-v1.proto'));
+        const proto = grpc.loadPackageDefinition(packageDefinition);
+        this.grpcClient = new proto.Askui.API.TDKv1.ControllerAPI("127.0.0.1:23000", grpc.credentials.createInsecure());
+        this.sessionGUID = (0, uuid_1.v4)();
+        this.sessionInfo = null;
+    }
+    connect() {
+        return new Promise((resolve, reject) => {
+            this.grpcClient.StartSession({
+                sessionGUID: this.sessionGUID,
+                immediateExecution: true
+            }, (error, response) => {
+                if (!error && response != undefined) {
+                    this.sessionInfo = response.sessionInfo;
+                    resolve(ui_controller_client_connection_state_1.UiControllerClientConnectionState.CONNECTED);
+                }
+                reject(ui_controller_client_connection_state_1.UiControllerClientConnectionState.ERROR);
+            });
+        });
+    }
+    disconnect() {
+        return new Promise((resolve, reject) => {
+            this.grpcClient.EndSession({
+                sessionInfo: this.sessionInfo,
+            }, (error, _response) => {
+                if (!error) {
+                    resolve();
+                }
+                reject();
+            });
+        });
+    }
+    requestScreenshot() {
+        return new Promise((resolve, reject) => {
+            this.grpcClient.CaptureScreen({ sessionInfo: this.sessionInfo, captureArea: { displayID: 0 } }, (error, response) => __awaiter(this, void 0, void 0, function* () {
+                if (!error && (response === null || response === void 0 ? void 0 : response.bitmap) !== undefined && (response === null || response === void 0 ? void 0 : response.bitmap) !== null) {
+                    const image = new jimp_1.default(response.bitmap.width, response.bitmap.height);
+                    let red;
+                    let green;
+                    let blue;
+                    //
+                    response.bitmap.data.forEach((byte, i) => {
+                        switch (i % 4) {
+                            case 0: return blue = byte;
+                            case 1: return green = byte;
+                            case 2: return red = byte;
+                            default:
+                                image.bitmap.data[i - 3] = red;
+                                image.bitmap.data[i - 2] = green;
+                                image.bitmap.data[i - 1] = blue;
+                                image.bitmap.data[i] = 255;
+                                return;
+                        }
+                    });
+                    resolve({
+                        data: { image: yield image.getBase64Async('image/png') },
+                        msgName: 'CAPTURE_SCREENSHOT_RESPONSE',
+                    });
+                }
+                reject({
+                    data: { error: 'Error screenshot' },
+                    msgName: 'CAPTURE_SCREENSHOT_RESPONSE',
+                });
+            }));
+        });
+    }
+    getServerPid() {
+        throw new Error('Method not implemented.');
+    }
+    startVideoRecording() {
+        throw new Error('Method not implemented.');
+    }
+    stopVideoRecording() {
+        throw new Error('Method not implemented.');
+    }
+    readVideoRecording() {
+        throw new Error('Method not implemented.');
+    }
+    annotateInteractively(_boundingBoxes, _imageString) {
+        throw new Error('Method not implemented.');
+    }
+    requestControl(controlCommand) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const actionId = yield this.runRecordedFunction(controlCommand);
+            console.log(yield this.pollActions(actionId));
+            return Promise.resolve({
+                msgName: 'CONTROL_RESPONSE',
+                data: { msg: 'e' },
+            });
+        });
+    }
+    runRecordedFunction(_controlCommand) {
+        return new Promise((resolve, reject) => {
+            this.grpcClient.RunRecordedAction({ actionClassID: 12, sessionInfo: this.sessionInfo, actionParameters: { mouseMove: { position: { x: 23, y: 23 } } } }, (error, response) => {
+                if (!error && response != undefined) {
+                    resolve(response.actionID);
+                }
+                reject(error);
+            });
+        });
+    }
+    pollActions(actionID) {
+        return new Promise((resolve, reject) => {
+            this.grpcClient.Poll({ sessionInfo: this.sessionInfo, pollEventID: 'PollEventID_ActionFinished' }, (error, response) => {
+                var _a, _b, _c, _d;
+                if (!error && response != undefined) {
+                    console.log((_b = (_a = response.pollEventParameters) === null || _a === void 0 ? void 0 : _a.actionFinished) === null || _b === void 0 ? void 0 : _b.actionID);
+                    resolve(((_d = (_c = response.pollEventParameters) === null || _c === void 0 ? void 0 : _c.actionFinished) === null || _d === void 0 ? void 0 : _d.actionID) == actionID);
+                }
+                reject();
+            });
+        });
+    }
+}
+exports.UiControllerClientGrpc = UiControllerClientGrpc;

package/dist/cjs/execution/ui-controller-client-tdk-v1.d.ts

@@ -0,0 +1,26 @@
+import { DetectedElement } from '../core/model/annotation-result/detected-element';
+import { CaptureScreenshotResponse, GetProcessPidResponse, StartRecordingResponse, StopRecordingResponse, ReadRecordingPartResponse, InteractiveAnnotationResponse, ControlResponse, ControlFailureResponse } from '../core/runner-protocol';
+import { ControlCommand } from '../core/ui-control-commands';
+import { UiControllerClientConnectionState } from './ui-controller-client-connection-state';
+import { UiControllerFacade } from './ui-controller-client-facade';
+import { SessionInfo } from '../proto/Askui/API/TDKv1/SessionInfo';
+export declare class UiControllerClientTdkV1 implements UiControllerFacade {
+    url: string;
+    private grpcClient;
+    private sessionGUID;
+    sessionInfo: SessionInfo | null;
+    constructor(url: string);
+    connect(): Promise<UiControllerClientConnectionState>;
+    disconnect(): Promise<void>;
+    requestScreenshot(): Promise<CaptureScreenshotResponse>;
+    getServerPid(): Promise<GetProcessPidResponse>;
+    startVideoRecording(): Promise<StartRecordingResponse>;
+    stopVideoRecording(): Promise<StopRecordingResponse>;
+    readVideoRecording(): Promise<ReadRecordingPartResponse>;
+    annotateInteractively(_boundingBoxes: DetectedElement[], _imageString: string): Promise<InteractiveAnnotationResponse>;
+    requestControl(controlCommand: ControlCommand): Promise<ControlResponse | ControlFailureResponse>;
+    private runSingleAction;
+    private runRecordedFunction;
+    private pollActions;
+    private pollWithRetry;
+}

package/dist/cjs/execution/ui-controller-client-tdk-v1.js

@@ -0,0 +1,221 @@
+"use strict";
+/* eslint-disable */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.UiControllerClientTdkV1 = void 0;
+const grpc = __importStar(require("@grpc/grpc-js"));
+const protoLoader = __importStar(require("@grpc/proto-loader"));
+const uuid_1 = require("uuid");
+const jimp_1 = __importDefault(require("jimp"));
+const ui_controller_client_connection_state_1 = require("./ui-controller-client-connection-state");
+const path_1 = __importDefault(require("path"));
+const PollEventID_1 = require("../proto/Askui/API/TDKv1/PollEventID");
+const action_tdk_v1_1 = require("../core/ui-control-commands/action-tdk-v1");
+class UiControllerClientTdkV1 {
+    constructor(url) {
+        this.url = url;
+        const packageDefinition = protoLoader.loadSync(path_1.default.join(__dirname, '../proto/tdk-v1.proto'));
+        const proto = grpc.loadPackageDefinition(packageDefinition);
+        this.grpcClient = new proto.Askui.API.TDKv1.ControllerAPI("127.0.0.1:23000", grpc.credentials.createInsecure());
+        this.sessionGUID = (0, uuid_1.v4)();
+        this.sessionInfo = null;
+    }
+    connect() {
+        return new Promise((resolve, reject) => {
+            this.grpcClient.StartSession({
+                sessionGUID: this.sessionGUID,
+                immediateExecution: true
+            }, (error, response) => {
+                if (!error && response != undefined) {
+                    this.sessionInfo = response.sessionInfo;
+                    resolve(ui_controller_client_connection_state_1.UiControllerClientConnectionState.CONNECTED);
+                }
+                reject(ui_controller_client_connection_state_1.UiControllerClientConnectionState.ERROR);
+            });
+        });
+    }
+    disconnect() {
+        return new Promise((resolve, reject) => {
+            this.grpcClient.EndSession({
+                sessionInfo: this.sessionInfo,
+            }, (error, _response) => {
+                if (!error) {
+                    resolve();
+                    this.grpcClient.close();
+                }
+                reject();
+            });
+        });
+    }
+    requestScreenshot() {
+        return new Promise((resolve, reject) => {
+            this.grpcClient.CaptureScreen({ sessionInfo: this.sessionInfo, captureArea: { displayID: 1 } }, (error, response) => __awaiter(this, void 0, void 0, function* () {
+                if (!error && (response === null || response === void 0 ? void 0 : response.bitmap) !== undefined && (response === null || response === void 0 ? void 0 : response.bitmap) !== null) {
+                    const image = new jimp_1.default(response.bitmap.width, response.bitmap.height);
+                    let red;
+                    let green;
+                    let blue;
+                    //
+                    response.bitmap.data.forEach((byte, i) => {
+                        switch (i % 4) {
+                            case 0: return blue = byte;
+                            case 1: return green = byte;
+                            case 2: return red = byte;
+                            default:
+                                image.bitmap.data[i - 3] = red;
+                                image.bitmap.data[i - 2] = green;
+                                image.bitmap.data[i - 1] = blue;
+                                image.bitmap.data[i] = 255;
+                                return;
+                        }
+                    });
+                    resolve({
+                        data: { image: yield image.getBase64Async('image/png') },
+                        msgName: 'CAPTURE_SCREENSHOT_RESPONSE',
+                    });
+                }
+                reject({
+                    data: { error: `${error}` },
+                    msgName: 'CAPTURE_SCREENSHOT_RESPONSE',
+                });
+            }));
+        });
+    }
+    getServerPid() {
+        throw new Error('Method not implemented.');
+    }
+    startVideoRecording() {
+        throw new Error('Method not implemented.');
+    }
+    stopVideoRecording() {
+        throw new Error('Method not implemented.');
+    }
+    readVideoRecording() {
+        throw new Error('Method not implemented.');
+    }
+    annotateInteractively(_boundingBoxes, _imageString) {
+        throw new Error('Method not implemented.');
+    }
+    requestControl(controlCommand) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                for (const action of controlCommand.actions) {
+                    yield this.runSingleAction(action_tdk_v1_1.ActionTdkV1.fromAction(action));
+                }
+                return Promise.resolve({
+                    msgName: 'CONTROL_RESPONSE',
+                    data: { msg: 'e' },
+                });
+            }
+            catch (error) {
+                return Promise.resolve({
+                    msgName: 'CONTROL_FAILURE_RESPONSE',
+                    data: { error: `${error}` },
+                });
+            }
+        });
+    }
+    runSingleAction(request) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const actionId = yield this.runRecordedFunction(request);
+                return Promise.resolve(this.pollActions(actionId));
+            }
+            catch (error) {
+                Promise.reject(error);
+            }
+        });
+    }
+    runRecordedFunction(request) {
+        return new Promise((resolve, reject) => {
+            this.grpcClient.RunRecordedAction(request, (error, response) => {
+                if (!error && response != undefined) {
+                    resolve(response.actionID);
+                }
+                reject(error);
+            });
+        });
+    }
+    pollActions(actionID) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const pollRequest = {
+                sessionInfo: this.sessionInfo,
+                pollEventID: PollEventID_1.PollEventID.PollEventID_ActionFinished
+            };
+            try {
+                return Promise.race([
+                    this.pollWithRetry(pollRequest, actionID),
+                    new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout')), 30000)) // 30-second timeout
+                ]);
+            }
+            catch (error) {
+                return Promise.reject(error);
+            }
+        });
+    }
+    pollWithRetry(request, actionID) {
+        var _a, _b;
+        return __awaiter(this, void 0, void 0, function* () {
+            const maxRetries = 100;
+            for (let retryCount = 0; retryCount <= maxRetries; retryCount++) {
+                try {
+                    const response = yield new Promise((resolve, reject) => {
+                        this.grpcClient.Poll(request, (error, response) => {
+                            if (error || response === undefined) {
+                                reject(error);
+                            }
+                            else {
+                                resolve(response);
+                            }
+                        });
+                    });
+                    if (((_b = (_a = response.pollEventParameters) === null || _a === void 0 ? void 0 : _a.actionFinished) === null || _b === void 0 ? void 0 : _b.actionID) === actionID) {
+                        return Promise.resolve();
+                    }
+                    else {
+                        throw new Error('Invalid response');
+                    }
+                }
+                catch (error) {
+                    if (retryCount === maxRetries) {
+                        throw error;
+                    }
+                    yield new Promise(resolve => setTimeout(resolve, 250)); // Wait for 1 second before retrying
+                    console.log("Retry ?!");
+                }
+            }
+            throw new Error('Max retries exceeded');
+        });
+    }
+}
+exports.UiControllerClientTdkV1 = UiControllerClientTdkV1;

package/dist/cjs/proto/Askui/API/TDKv1/ActionClassID.d.ts

@@ -0,0 +1,21 @@
+export declare const ActionClassID: {
+    readonly ActionClassID_Undefined: "ActionClassID_Undefined";
+    readonly ActionClassID_Wait: "ActionClassID_Wait";
+    readonly ActionClassID_MouseButton_Press: "ActionClassID_MouseButton_Press";
+    readonly ActionClassID_MouseButton_Release: "ActionClassID_MouseButton_Release";
+    readonly ActionClassID_MouseButton_PressAndRelease: "ActionClassID_MouseButton_PressAndRelease";
+    readonly ActionClassID_MouseWheelScroll: "ActionClassID_MouseWheelScroll";
+    readonly ActionClassID_MouseMove: "ActionClassID_MouseMove";
+    readonly ActionClassID_MouseMove_Delta: "ActionClassID_MouseMove_Delta";
+    readonly ActionClassID_KeyboardKey_Press: "ActionClassID_KeyboardKey_Press";
+    readonly ActionClassID_KeyboardKey_Release: "ActionClassID_KeyboardKey_Release";
+    readonly ActionClassID_KeyboardKey_PressAndRelease: "ActionClassID_KeyboardKey_PressAndRelease";
+    readonly ActionClassID_KeyboardKeys_Press: "ActionClassID_KeyboardKeys_Press";
+    readonly ActionClassID_KeyboardKeys_Release: "ActionClassID_KeyboardKeys_Release";
+    readonly ActionClassID_KeyboardKeys_PressAndRelease: "ActionClassID_KeyboardKeys_PressAndRelease";
+    readonly ActionClassID_KeyboardType_Text: "ActionClassID_KeyboardType_Text";
+    readonly ActionClassID_KeyboardType_UnicodeText: "ActionClassID_KeyboardType_UnicodeText";
+    readonly ActionClassID_RunCommand: "ActionClassID_RunCommand";
+};
+export declare type ActionClassID = 'ActionClassID_Undefined' | 0 | 'ActionClassID_Wait' | 1 | 'ActionClassID_MouseButton_Press' | 8 | 'ActionClassID_MouseButton_Release' | 9 | 'ActionClassID_MouseButton_PressAndRelease' | 10 | 'ActionClassID_MouseWheelScroll' | 11 | 'ActionClassID_MouseMove' | 12 | 'ActionClassID_MouseMove_Delta' | 13 | 'ActionClassID_KeyboardKey_Press' | 14 | 'ActionClassID_KeyboardKey_Release' | 15 | 'ActionClassID_KeyboardKey_PressAndRelease' | 16 | 'ActionClassID_KeyboardKeys_Press' | 17 | 'ActionClassID_KeyboardKeys_Release' | 18 | 'ActionClassID_KeyboardKeys_PressAndRelease' | 19 | 'ActionClassID_KeyboardType_Text' | 20 | 'ActionClassID_KeyboardType_UnicodeText' | 21 | 'ActionClassID_RunCommand' | 22;
+export declare type ActionClassID__Output = typeof ActionClassID[keyof typeof ActionClassID];

package/dist/cjs/proto/Askui/API/TDKv1/ActionClassID.js

@@ -0,0 +1,23 @@
+"use strict";
+// Original file: src/proto/tdk-v1.proto
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ActionClassID = void 0;
+exports.ActionClassID = {
+    ActionClassID_Undefined: 'ActionClassID_Undefined',
+    ActionClassID_Wait: 'ActionClassID_Wait',
+    ActionClassID_MouseButton_Press: 'ActionClassID_MouseButton_Press',
+    ActionClassID_MouseButton_Release: 'ActionClassID_MouseButton_Release',
+    ActionClassID_MouseButton_PressAndRelease: 'ActionClassID_MouseButton_PressAndRelease',
+    ActionClassID_MouseWheelScroll: 'ActionClassID_MouseWheelScroll',
+    ActionClassID_MouseMove: 'ActionClassID_MouseMove',
+    ActionClassID_MouseMove_Delta: 'ActionClassID_MouseMove_Delta',
+    ActionClassID_KeyboardKey_Press: 'ActionClassID_KeyboardKey_Press',
+    ActionClassID_KeyboardKey_Release: 'ActionClassID_KeyboardKey_Release',
+    ActionClassID_KeyboardKey_PressAndRelease: 'ActionClassID_KeyboardKey_PressAndRelease',
+    ActionClassID_KeyboardKeys_Press: 'ActionClassID_KeyboardKeys_Press',
+    ActionClassID_KeyboardKeys_Release: 'ActionClassID_KeyboardKeys_Release',
+    ActionClassID_KeyboardKeys_PressAndRelease: 'ActionClassID_KeyboardKeys_PressAndRelease',
+    ActionClassID_KeyboardType_Text: 'ActionClassID_KeyboardType_Text',
+    ActionClassID_KeyboardType_UnicodeText: 'ActionClassID_KeyboardType_UnicodeText',
+    ActionClassID_RunCommand: 'ActionClassID_RunCommand',
+};