@resolveio/server-lib 20.14.17 → 20.14.18

package/managers/worker-dispatcher.manager.d.ts

@@ -9,6 +9,13 @@ export interface WorkerConnection {
         taskId: string;
         weight: number;
         method?: string;
+        startedAt?: number;
+        messageId?: number;
+        userContext?: {
+            id_user?: string;
+            user?: string;
+            id_ws?: string;
+        };
     }[];
     workerIndex?: string;
     workerInstance?: string;
@@ -21,7 +28,9 @@ export declare class WorkerDispatcherManager {
     private _clientRequests;
     private _pendingTasks;
     private _debugNoWorkerTaskIds;
+    private _taskTimings;
     private MAX_CONCURRENCY;
+    private WORKER_TASK_LOG_THRESHOLD_MS;
     constructor();
     static create(websocketManager: WebSocketManager, methodManager: MethodManager): WorkerDispatcherManager;
     initialize(websocketManager: WebSocketManager, methodManager: MethodManager): void;
@@ -72,6 +81,8 @@ export declare class WorkerDispatcherManager {
     private assignTaskToWorker;
     private logNoWorkerAvailability;
     private shouldDebug;
+    private logWorkerEvent;
+    private safeStringify;
     /**
      * Handle messages coming back from a worker (like 'taskComplete').
      */

package/managers/worker-dispatcher.manager.js

@@ -1,4 +1,15 @@
 "use strict";
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
 var __values = (this && this.__values) || function(o) {
     var s = typeof Symbol === "function" && Symbol.iterator, m = s && o[s], i = 0;
     if (m) return m.call(o);
@@ -21,7 +32,9 @@ var WorkerDispatcherManager = /** @class */ (function () {
         this._clientRequests = {};
         this._pendingTasks = new Map();
         this._debugNoWorkerTaskIds = new Set();
+        this._taskTimings = new Map();
         this.MAX_CONCURRENCY = 10;
+        this.WORKER_TASK_LOG_THRESHOLD_MS = 200;
     }
     WorkerDispatcherManager.create = function (websocketManager, methodManager) {
         var workerDispatcherManager = new WorkerDispatcherManager();
@@ -100,12 +113,20 @@ var WorkerDispatcherManager = /** @class */ (function () {
      */
     WorkerDispatcherManager.prototype.sendClientTask = function (messageId, method, params, userContext) {
         var taskId = 'task-' + (0, common_1.objectIdHexString)();
+        var queuedAt = Date.now();
         this._taskQueue.push({
             type: 'task',
             taskId: taskId,
             messageId: messageId,
             method: method,
             params: params,
+            userContext: userContext,
+            queuedAt: queuedAt
+        });
+        this._taskTimings.set(taskId, {
+            queuedAt: queuedAt,
+            method: method,
+            messageId: messageId,
             userContext: userContext
         });
         if (userContext && userContext.id_ws) {
@@ -125,6 +146,7 @@ var WorkerDispatcherManager = /** @class */ (function () {
         // eslint-disable-next-line no-restricted-syntax
         return new Promise(function (resolve, reject) {
             var taskId = 'task-' + (0, common_1.objectIdHexString)();
+            var queuedAt = Date.now();
             _this._taskQueue.push({
                 type: 'task',
                 taskId: taskId,
@@ -133,7 +155,14 @@ var WorkerDispatcherManager = /** @class */ (function () {
                 messageId: 0,
                 userContext: {
                     user: 'Internal System'
-                }
+                },
+                queuedAt: queuedAt
+            });
+            _this._taskTimings.set(taskId, {
+                queuedAt: queuedAt,
+                method: method,
+                messageId: 0,
+                userContext: { user: 'Internal System' }
             });
             _this._pendingTasks.set(taskId, {
                 timeout: null,
@@ -154,6 +183,7 @@ var WorkerDispatcherManager = /** @class */ (function () {
             params = [params];
         }
         var taskId = 'task-' + (0, common_1.objectIdHexString)();
+        var queuedAt = Date.now();
         this._taskQueue.push({
             type: 'task',
             taskId: taskId,
@@ -162,7 +192,14 @@ var WorkerDispatcherManager = /** @class */ (function () {
             messageId: 0,
             userContext: {
                 user: 'Internal System'
-            }
+            },
+            queuedAt: queuedAt
+        });
+        this._taskTimings.set(taskId, {
+            queuedAt: queuedAt,
+            method: method,
+            messageId: 0,
+            userContext: { user: 'Internal System' }
         });
         if (this._methodManager.getEnableDebug()) {
             console.log(new Date(), 'Send Internal Task', this._taskQueue);
@@ -294,12 +331,23 @@ var WorkerDispatcherManager = /** @class */ (function () {
         var method = this._methodManager.getMethod(task.method);
         if (!method) {
             console.error('Failed to send task to worker - Could not find method:', task.method);
+            this._taskTimings.delete(task.taskId);
             return;
         }
+        var dispatchedAt = Date.now();
+        var timing = this._taskTimings.get(task.taskId) || { queuedAt: dispatchedAt };
+        timing.dispatchedAt = dispatchedAt;
+        timing.method = task.method;
+        timing.messageId = task.messageId;
+        timing.userContext = task.userContext;
+        this._taskTimings.set(task.taskId, timing);
         worker.activeTasks.push({
             taskId: task.taskId,
             weight: method && method.workerTaskWeight ? method.workerTaskWeight : 1,
-            method: task.method
+            method: task.method,
+            startedAt: dispatchedAt,
+            messageId: task.messageId,
+            userContext: task.userContext
         });
         this._debugNoWorkerTaskIds.delete(task.taskId);
         var payload = {
@@ -308,18 +356,38 @@ var WorkerDispatcherManager = /** @class */ (function () {
             messageId: task.messageId,
             method: task.method,
             params: task.params,
-            userContext: task.userContext || {}
+            userContext: task.userContext || {},
+            queuedAt: timing.queuedAt,
+            dispatchedAt: dispatchedAt
         };
         var timeoutHandle = setTimeout(function () {
             var pending = _this._pendingTasks.get(task.taskId);
+            var timeoutAt = Date.now();
+            var queuedAt = timing.queuedAt;
+            var queueWaitMs = queuedAt ? (0, common_1.round)(dispatchedAt - queuedAt) : null;
+            var dispatchToTimeoutMs = (0, common_1.round)(timeoutAt - dispatchedAt);
             if (pending) {
                 if (pending.promise) {
                     pending.promise.reject('Task timed out after 2m in WorkerDispatcherManager for method: ' + task.method);
                 }
                 _this._pendingTasks.delete(task.taskId);
             }
-            console.log(new Date(), 'TIMEOUT HIT', 'task', task.taskId, task.messageId, task.method, JSON.stringify(_this._workers.map(function (a) { return a.activeTasks; })));
+            _this.logWorkerEvent('workerTaskTimeout', {
+                taskId: task.taskId,
+                method: task.method,
+                messageId: task.messageId,
+                queueWaitMs: queueWaitMs,
+                dispatchToTimeoutMs: dispatchToTimeoutMs,
+                userContext: task.userContext,
+                worker: {
+                    id: worker.id,
+                    index: worker.workerIndex,
+                    instance: worker.workerInstance
+                },
+                workers: _this._workers.map(function (a) { return a.activeTasks; })
+            });
             worker.activeTasks = worker.activeTasks.filter(function (a) { return a.taskId !== task.taskId; });
+            _this._taskTimings.delete(task.taskId);
             if (task.userContext && task.userContext.id_ws) {
                 var timeoutRes = {
                     messageId: task.messageId,
@@ -409,6 +477,42 @@ var WorkerDispatcherManager = /** @class */ (function () {
     WorkerDispatcherManager.prototype.shouldDebug = function () {
         return this._methodManager.getEnableDebug() || process.env.WORKER_DISPATCH_DEBUG === 'true';
     };
+    WorkerDispatcherManager.prototype.logWorkerEvent = function (event, details) {
+        console.log(this.safeStringify(__assign({ ts: new Date().toISOString(), event: event }, details)));
+    };
+    WorkerDispatcherManager.prototype.safeStringify = function (value) {
+        try {
+            var seen_1 = new WeakSet();
+            return JSON.stringify(value, function (key, val) {
+                if (key === '__proto__') {
+                    return undefined;
+                }
+                if (typeof val === 'bigint') {
+                    return val.toString();
+                }
+                if (val instanceof Error) {
+                    return {
+                        name: val.name,
+                        message: val.message,
+                        stack: val.stack
+                    };
+                }
+                if (val && typeof val === 'object') {
+                    if (seen_1.has(val)) {
+                        return '[Circular]';
+                    }
+                    seen_1.add(val);
+                }
+                return val;
+            });
+        }
+        catch (err) {
+            return JSON.stringify({
+                error: 'safeStringifyFailed',
+                message: (err === null || err === void 0 ? void 0 : err.message) || String(err)
+            });
+        }
+    };
     /**
      * Handle messages coming back from a worker (like 'taskComplete').
      */
@@ -449,6 +553,14 @@ var WorkerDispatcherManager = /** @class */ (function () {
             }
             var taskId_1 = data.taskId, messageId = data.messageId, error = data.error, result = data.result;
             var unpackedResult = result;
+            var timing = this._taskTimings.get(taskId_1);
+            var completedAt = Date.now();
+            var dispatchedAt = timing === null || timing === void 0 ? void 0 : timing.dispatchedAt;
+            var queuedAt = timing === null || timing === void 0 ? void 0 : timing.queuedAt;
+            var dispatchToCompleteMs = dispatchedAt ? (0, common_1.round)(completedAt - dispatchedAt) : null;
+            var queueWaitMs = queuedAt && dispatchedAt ? (0, common_1.round)(dispatchedAt - queuedAt) : null;
+            var totalMs = queuedAt ? (0, common_1.round)(completedAt - queuedAt) : null;
+            var packedBytes = data['packedResult'] ? data['packedResult'].length : null;
             if (!error && unpackedResult === null && data['packedResult']) {
                 try {
                     unpackedResult = (0, msgpackr_1.unpack)(data['packedResult']);
@@ -458,9 +570,26 @@ var WorkerDispatcherManager = /** @class */ (function () {
                 }
             }
             worker.activeTasks = worker.activeTasks.filter(function (a) { return a.taskId !== taskId_1; });
+            this._taskTimings.delete(taskId_1);
             var pendingTask = this._pendingTasks.get(taskId_1);
-            if (this._methodManager.getEnableDebug()) {
-                console.log(new Date(), 'Recv from Worker Server', data);
+            if (error || (dispatchToCompleteMs !== null && dispatchToCompleteMs >= this.WORKER_TASK_LOG_THRESHOLD_MS)) {
+                this.logWorkerEvent('workerTaskComplete', {
+                    taskId: taskId_1,
+                    method: pendingTask === null || pendingTask === void 0 ? void 0 : pendingTask.method,
+                    messageId: messageId,
+                    error: error,
+                    dispatchToCompleteMs: dispatchToCompleteMs,
+                    queueWaitMs: queueWaitMs,
+                    totalMs: totalMs,
+                    packedBytes: packedBytes,
+                    userContext: timing === null || timing === void 0 ? void 0 : timing.userContext,
+                    worker: {
+                        id: worker.id,
+                        index: worker.workerIndex,
+                        instance: worker.workerInstance
+                    },
+                    queue: this.getQueueSnapshot()
+                });
             }
             if (pendingTask) {
                 if (pendingTask.promise) {

package/managers/worker-dispatcher.manager.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/managers/worker-dispatcher.manager.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAEA,yCAAgE;AAGhE,qCAAwC;AAqBxC;IAWC;QARQ,aAAQ,GAAuB,EAAE,CAAC;QAClC,eAAU,GAAkB,EAAE,CAAC;QAC/B,oBAAe,GAAiC,EAAE,CAAC;QACnD,kBAAa,GAA6B,IAAI,GAAG,EAAE,CAAC;QACpD,0BAAqB,GAAgB,IAAI,GAAG,EAAE,CAAC;QAE/C,oBAAe,GAAG,EAAE,CAAC;IAEd,CAAC;IAET,8BAAM,GAAb,UAAc,gBAAkC,EAAE,aAA4B;QAC7E,IAAM,uBAAuB,GAAG,IAAI,uBAAuB,EAAE,CAAC;QAC9D,uBAAuB,CAAC,UAAU,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAC;QACpE,OAAO,uBAAuB,CAAC;IAChC,CAAC;IAEM,4CAAU,GAAjB,UAAkB,gBAAkC,EAAE,aAA4B;QACjF,IAAI,CAAC,iBAAiB,GAAG,gBAAgB,CAAC;QAC1C,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;IACrC,CAAC;IAEM,gDAAc,GAArB;QACC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;IAC5D,CAAC;IAEM,2CAAS,GAAhB,UAAiB,EAAuB;QAAxC,iBAuBC;QAtBA,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAClB,EAAE,EAAE,EAAE,CAAC,WAAW,CAAC;YACnB,EAAE,EAAE,EAAE;YACN,WAAW,EAAE,EAAE;YACf,WAAW,EAAE,EAAE,CAAC,aAAa,CAAC;YAC9B,cAAc,EAAE,EAAE,CAAC,gBAAgB,CAAC;SACpC,CAAC,CAAC;QAEH,WAAW,CAAC;YACX,IAAI,KAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;gBAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,KAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAA,CAAC;oBAC7C,OAAO;wBACN,EAAE,EAAE,CAAC,CAAC,EAAE;wBACR,WAAW,EAAE,CAAC,CAAC,WAAW;qBAC1B,CAAA;gBACF,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACf,CAAC;QACF,CAAC,EAAE,IAAI,CAAC,CAAC;QAET,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YAC5B,IAAI,CAAC,aAAa,EAAE,CAAC;QACtB,CAAC;IACF,CAAC;IAEM,kDAAgB,GAAvB,UAAwB,QAAgB;QACvC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,CAAC,EAAE,KAAK,QAAQ,EAAjB,CAAiB,CAAC,CAAC;IAC9D,CAAC;IAEM,4CAAU,GAAjB;QACC,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;IACjC,CAAC;IAEM,kDAAgB,GAAvB;QACC,IAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAC,KAAK,EAAE,MAAM,IAAK,OAAA,KAAK,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAA5D,CAA4D,EAAE,CAAC,CAAC,CAAC;QAC7H,OAAO;YACN,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM;YAClC,YAAY,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI;YACrC,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;YACjC,WAAW,aAAA;YACX,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAA,MAAM,IAAI,OAAA,CAAC;gBACrC,EAAE,EAAE,MAAM,CAAC,EAAE;gBACb,KAAK,EAAE,MAAM,CAAC,WAAW;gBACzB,QAAQ,EAAE,MAAM,CAAC,cAAc;gBAC/B,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;aAC/D,CAAC,EALmC,CAKnC,CAAC;SACH,CAAC;IACH,CAAC;IAEM,qDAAmB,GAA1B,UAA2B,UAAkB;QAA7C,iBAiBC;QAhBA,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC3B,OAAO,KAAK,CAAC;QACd,CAAC;QAED,IAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACzD,IAAM,iBAAiB,GAAG,IAAI,CAAC,oBAAoB,CAAC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,iBAAiB,CAAC,CAAC;QAC/E,IAAM,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,oBAAoB,CAAC,CAAC;QAErF,IAAI,CAAC,iBAAiB,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACjD,OAAO,IAAI,CAAC;QACb,CAAC;QAED,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAA,MAAM;YAC/B,OAAA,CAAC,CAAC,iBAAiB,IAAI,KAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,iBAAiB,CAAC;gBAC3F,CAAC,CAAC,oBAAoB,IAAI,KAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,cAAc,CAAC,KAAK,oBAAoB,CAAC;QADpG,CACoG,CACpG,CAAC;IACH,CAAC;IAED;;OAEG;IACI,gDAAc,GAArB,UAAsB,SAAiB,EAAE,MAAc,EAAE,MAAa,EAAE,WAAiE;QACxI,IAAI,MAAM,GAAG,OAAO,GAAG,IAAA,0BAAiB,GAAE,CAAC;QAE3C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACpB,IAAI,EAAE,MAAM;YACZ,MAAM,QAAA;YACN,SAAS,WAAA;YACT,MAAM,QAAA;YACN,MAAM,QAAA;YACN,WAAW,aAAA;SACX,CAAC,CAAC;QAEH,IAAI,WAAW,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;YACtC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC;QAClD,CAAC;QAED,IAAI,CAAC,aAAa,EAAE,CAAC;IACtB,CAAC;IAED;;OAEG;IACI,qDAAmB,GAA1B,UAA2B,MAAc,EAAE,MAAkB;QAA7D,iBA+BC;QA/B0C,uBAAA,EAAA,WAAkB;QAC5D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC;QACnB,CAAC;QAED,gDAAgD;QAChD,OAAO,IAAI,OAAO,CAAC,UAAC,OAAO,EAAE,MAAM;YAClC,IAAI,MAAM,GAAG,OAAO,GAAG,IAAA,0BAAiB,GAAE,CAAC;YAE3C,KAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACpB,IAAI,EAAE,MAAM;gBACZ,MAAM,QAAA;gBACN,MAAM,QAAA;gBACN,MAAM,QAAA;gBACN,SAAS,EAAE,CAAC;gBACZ,WAAW,EAAE;oBACZ,IAAI,EAAE,iBAAiB;iBACvB;aACD,CAAC,CAAC;YAEH,KAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,EAAE;gBAC9B,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE,EAAE,OAAO,SAAA,EAAE,MAAM,QAAA,EAAE;aAC5B,CAAC,CAAC;YAEH,IAAI,KAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;gBAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,uBAAuB,EAAE,KAAI,CAAC,UAAU,CAAC,CAAC;YACnE,CAAC;YAED,KAAI,CAAC,aAAa,EAAE,CAAC;QACtB,CAAC,CAAC,CAAC;IACJ,CAAC;IAED;;OAEG;IACI,kDAAgB,GAAvB,UAAwB,MAAc,EAAE,MAAW;QAAX,uBAAA,EAAA,WAAW;QAClD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC;QACnB,CAAC;QAED,IAAI,MAAM,GAAG,OAAO,GAAG,IAAA,0BAAiB,GAAE,CAAC;QAE3C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACpB,IAAI,EAAE,MAAM;YACZ,MAAM,QAAA;YACN,MAAM,QAAA;YACN,MAAM,QAAA;YACN,SAAS,EAAE,CAAC;YACZ,WAAW,EAAE;gBACZ,IAAI,EAAE,iBAAiB;aACvB;SACD,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;YAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,oBAAoB,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAChE,CAAC;QAED,IAAI,CAAC,aAAa,EAAE,CAAC;IACtB,CAAC;IAED;;OAEG;IACK,+CAAa,GAArB;QAAA,iBAiCC;QAhCA,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YAC7B,OAAO;QACR,CAAC;QAED,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,IAAI,QAAQ,GAAG,KAAK,CAAC;YAErB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACjD,IAAI,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;gBAC9B,IAAI,YAAY,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;gBAEhD,IAAI,YAAY,EAAE,CAAC;oBAClB,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;oBAC7B,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;oBAC5C,QAAQ,GAAG,IAAI,CAAC;oBAChB,MAAM;gBACP,CAAC;YACF,CAAC;YAED,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACf,IAAI,CAAC,uBAAuB,EAAE,CAAC;gBAC/B,UAAU,CAAC;oBACV,KAAI,CAAC,aAAa,EAAE,CAAC;gBACtB,CAAC,EAAE,EAAE,CAAC,CAAC;gBAEP,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;oBAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,qBAAqB,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;gBACxF,CAAC;gBAED,OAAO,CAAC,qDAAqD;YAC9D,CAAC;QACF,CAAC;IACF,CAAC;IAED;;OAEG;IACK,mDAAiB,GAAzB,UAA0B,IAAiB;QAA3C,iBAmFC;QAlFA,IAAI,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxD,IAAI,WAAW,GAAG,MAAM,IAAI,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9G,IAAI,sBAAsB,GAAG,MAAM,IAAI,MAAM,CAAC,yBAAyB,IAAI,MAAM,CAAC,yBAAyB,GAAG,CAAC;YAC9G,CAAC,CAAC,MAAM,CAAC,yBAAyB;YAClC,CAAC,CAAC,IAAI,CAAC;QACR,IAAI,iBAAiB,GAAG,IAAI,CAAC,oBAAoB,CAAC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,iBAAiB,CAAC,CAAC;QAC7E,IAAI,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,oBAAoB,CAAC,CAAC;QACnF,IAAM,aAAa,GAAG,CAAC,iBAAiB,CAAC;QAEzC,IAAI,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,KAAI,CAAC,eAAe,EAA3C,CAA2C,CAAC,CAAC;QACxF,IAAI,iBAAiB,EAAE,CAAC;YACvB,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,UAAA,MAAM,IAAI,OAAA,KAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,iBAAiB,EAAnE,CAAmE,CAAC,CAAC;QAC/G,CAAC;QACD,IAAI,oBAAoB,EAAE,CAAC;YAC1B,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,UAAA,MAAM,IAAI,OAAA,KAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,cAAc,CAAC,KAAK,oBAAoB,EAAzE,CAAyE,CAAC,CAAC;QACrH,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YACxB,OAAO,IAAI,CAAC;QACb,CAAC;QAED,IAAI,QAAQ,GAAG,UAAU,CAAC,MAAM,CAAC,UAAC,MAAM;YACvC,IAAI,CAAC,WAAW,EAAE,CAAC;gBAClB,IAAI,CAAC,sBAAsB,EAAE,CAAC;oBAC7B,OAAO,IAAI,CAAC;gBACb,CAAC;YACF,CAAC;YAED,IAAI,sBAAsB,EAAE,CAAC;gBAC5B,IAAM,eAAe,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAxB,CAAwB,CAAC,CAAC,MAAM,CAAC;gBACxF,IAAI,eAAe,IAAI,sBAAsB,EAAE,CAAC;oBAC/C,OAAO,KAAK,CAAC;gBACd,CAAC;YACF,CAAC;YAED,IAAI,CAAC,WAAW,EAAE,CAAC;gBAClB,OAAO,IAAI,CAAC;YACb,CAAC;YAED,IAAI,GAAG,GAAG,MAAM,CAAC,WAAW,IAAI,SAAS,CAAC;YAC1C,IAAI,OAAO,GAAG,CAAC,CAAC;YAEhB,KAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAC,CAAC;gBACvB,IAAI,CAAC,CAAC,CAAC,WAAW,IAAI,SAAS,CAAC,KAAK,GAAG,EAAE,CAAC;oBAC1C,OAAO,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAxB,CAAwB,CAAC,CAAC,MAAM,CAAC;gBACvE,CAAC;YACF,CAAC,CAAC,CAAC;YAEH,OAAO,OAAO,GAAG,WAAW,CAAC;QAC9B,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QACb,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC,UAAC,CAAC,EAAE,CAAC;YAClB,IAAM,MAAM,GAAG,KAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;YACrC,IAAM,MAAM,GAAG,KAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;YACrC,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;gBACvB,OAAO,MAAM,GAAG,MAAM,CAAC;YACxB,CAAC;YAED,IAAI,aAAa,EAAE,CAAC;gBACnB,IAAM,KAAK,GAAG,KAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3C,IAAM,KAAK,GAAG,KAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3C,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC;oBACrB,OAAO,KAAK,GAAG,KAAK,CAAC;gBACtB,CAAC;YACF,CAAC;YAED,IAAM,IAAI,GAAG,KAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;YAC5D,IAAM,IAAI,GAAG,KAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;YAC5D,IAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;YAC/F,IAAI,UAAU,KAAK,CAAC,EAAE,CAAC;gBACtB,OAAO,UAAU,CAAC;YACnB,CAAC;YAED,IAAM,KAAK,GAAG,KAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAChE,IAAM,KAAK,GAAG,KAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAChE,OAAO,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;QACtF,CAAC,CAAC,CAAC;QAEH,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAEO,sDAAoB,GAA5B,UAA6B,KAA8B;QAC1D,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3C,OAAO,IAAI,CAAC;QACb,CAAC;QAED,IAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;QACxC,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9C,CAAC;IAEO,+CAAa,GAArB,UAAsB,MAAwB;QAC7C,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,UAAC,GAAG,EAAE,IAAI,IAAK,OAAA,GAAG,GAAG,IAAI,CAAC,MAAM,EAAjB,CAAiB,EAAE,CAAC,CAAC,CAAC;IACvE,CAAC;IAEO,8CAAY,GAApB,UAAqB,MAAwB;QAC5C,OAAO,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,GAAG,CAAC;IAC9D,CAAC;IAEO,oDAAkB,GAA1B,UAA2B,MAAwB,EAAE,IAAiB;QAAtE,iBA+EC;QA9EA,IAAI,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAExD,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,wDAAwD,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACrF,OAAO;QACR,CAAC;QAED,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,MAAM,IAAI,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;YACvE,MAAM,EAAE,IAAI,CAAC,MAAM;SACnB,CAAC,CAAC;QACH,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE/C,IAAI,OAAO,GAAgB;YAC1B,IAAI,EAAE,MAAM;YACZ,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,EAAE;SACnC,CAAC;QAEF,IAAI,aAAa,GAAG,UAAU,CAAC;YAC9B,IAAI,OAAO,GAAG,KAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAElD,IAAI,OAAO,EAAE,CAAC;gBACb,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;oBACrB,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,iEAAiE,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;gBACzG,CAAC;gBAED,KAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACxC,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,KAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,CAAC,WAAW,EAAb,CAAa,CAAC,CAAC,CAAC,CAAC;YAEhJ,MAAM,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAxB,CAAwB,CAAC,CAAC;YAE9E,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;gBAChD,IAAI,UAAU,GAAwB;oBACrC,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,QAAQ,EAAE,IAAI;oBACd,IAAI,EAAE,iEAAiE,GAAG,IAAI,CAAC,MAAM;iBACrF,CAAC;gBAEF,IAAI,QAAQ,GAAG,KAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;gBAE3E,IAAI,QAAQ,EAAE,CAAC;oBACd,KAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;gBACnD,CAAC;YACF,CAAC;QACF,CAAC,EAAE,MAAM,CAAC,eAAe,IAAI,CAAC,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAE9C,+FAA+F;QAC/F,IAAI,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,QAAQ,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;QACnD,CAAC;aACI,CAAC;YACL,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC/B,CAAC;QAED,QAAQ,CAAC,OAAO,GAAG,aAAa,CAAC;QACjC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAE9C,IAAI,CAAC;YACJ,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,GAAG,EAAE,CAAC;YACZ,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;YAErD,YAAY,CAAC,aAAa,CAAC,CAAC;YAC5B,MAAM,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAxB,CAAwB,CAAC,CAAC;YAE9E,yBAAyB;YACzB,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC9B,IAAI,CAAC,aAAa,EAAE,CAAC;QACtB,CAAC;IACF,CAAC;IAEO,yDAAuB,GAA/B;;QAAA,iBAyCC;QAxCA,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACzB,OAAO;QACR,CAAC;QAED,IAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAA,MAAM,IAAI,OAAA,CAAC;YAC5C,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,KAAK,EAAE,KAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,WAAW,CAAC;YACpD,QAAQ,EAAE,KAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,cAAc,CAAC;YAC1D,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM;SACtC,CAAC,EAL0C,CAK1C,CAAC,CAAC;gCAEO,IAAI;YACd,IAAM,MAAM,GAAG,OAAK,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1D,IAAM,iBAAiB,GAAG,OAAK,oBAAoB,CAAC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,iBAAiB,CAAC,CAAC;YAC/E,IAAM,oBAAoB,GAAG,OAAK,oBAAoB,CAAC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,oBAAoB,CAAC,CAAC;YAErF,IAAI,CAAC,iBAAiB,IAAI,CAAC,oBAAoB,EAAE,CAAC;;YAElD,CAAC;YAED,IAAI,OAAK,qBAAqB,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;;YAElD,CAAC;YAED,IAAM,QAAQ,GAAG,OAAK,QAAQ,CAAC,IAAI,CAAC,UAAA,MAAM;gBACzC,OAAA,CAAC,CAAC,iBAAiB,IAAI,KAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,iBAAiB,CAAC;oBAC3F,CAAC,CAAC,oBAAoB,IAAI,KAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,cAAc,CAAC,KAAK,oBAAoB,CAAC;YADpG,CACoG,CACpG,CAAC;YAEF,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACf,OAAK,qBAAqB,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAC5C,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,EAAE,gDAAgD,EAAE;oBAC1E,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,iBAAiB,mBAAA;oBACjB,oBAAoB,sBAAA;oBACpB,OAAO,SAAA;iBACP,CAAC,CAAC;YACJ,CAAC;;;;YA3BF,KAAmB,IAAA,KAAA,SAAA,IAAI,CAAC,UAAU,CAAA,gBAAA;gBAA7B,IAAM,IAAI,WAAA;wBAAJ,IAAI;aA4Bd;;;;;;;;;IACF,CAAC;IAEO,6CAAW,GAAnB;QACC,OAAO,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,IAAI,OAAO,CAAC,GAAG,CAAC,qBAAqB,KAAK,MAAM,CAAC;IAC7F,CAAC;IAED;;OAEG;IACI,qDAAmB,GAA1B,UAA2B,QAAgB,EAAE,UAA6B;QACzE,IAAI,IAAkB,CAAC;QAEvB,IAAI,CAAC;YACJ,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;gBACpC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,oBAAW,CAAC,CAAC;YAC5C,CAAC;iBACI,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBACtC,IAAI,GAAG,IAAA,iBAAM,EAAC,UAAU,CAAC,CAAC;YAC3B,CAAC;iBACI,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;gBACpC,IAAM,MAAM,GAAG,UAAkD,CAAC;gBAClE,IAAI,GAAG,IAAA,iBAAM,EAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;YACtC,CAAC;iBACI,IAAI,UAAU,YAAY,WAAW,EAAE,CAAC;gBAC5C,IAAI,GAAG,IAAA,iBAAM,EAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;YACxC,CAAC;iBACI,IAAI,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;gBACzC,IAAM,IAAI,GAAG,UAAoC,CAAC;gBAClD,IAAI,GAAG,IAAA,iBAAM,EAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;YAC3E,CAAC;iBACI,CAAC;gBACL,OAAO,CAAC,KAAK,CAAC,2CAA2C,EAAE,OAAO,UAAU,CAAC,CAAC;gBAC9E,OAAO;YACR,CAAC;QACF,CAAC;QACD,OAAO,GAAG,EAAE,CAAC;YACZ,OAAO,CAAC,KAAK,CAAC,iCAAiC,EAAE,GAAG,CAAC,CAAC;YACtD,OAAO;QACR,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAClC,IAAI,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,CAAC,EAAE,KAAK,QAAQ,EAAjB,CAAiB,CAAC,CAAC;YAExD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,kCAAkC,EAAE,QAAQ,CAAC,CAAC;gBAC5D,OAAO;YACR,CAAC;YAEK,IAAA,QAAM,GAA+B,IAAI,OAAnC,EAAE,SAAS,GAAoB,IAAI,UAAxB,EAAE,KAAK,GAAa,IAAI,MAAjB,EAAE,MAAM,GAAK,IAAI,OAAT,CAAU;YAChD,IAAI,cAAc,GAAG,MAAM,CAAC;YAE5B,IAAI,CAAC,KAAK,IAAI,cAAc,KAAK,IAAI,IAAI,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;gBAC/D,IAAI,CAAC;oBACJ,cAAc,GAAG,IAAA,iBAAM,EAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;gBAC/C,CAAC;gBACD,OAAO,GAAG,EAAE,CAAC;oBACZ,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;gBACtD,CAAC;YACF,CAAC;YACD,MAAM,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,CAAC,MAAM,KAAK,QAAM,EAAnB,CAAmB,CAAC,CAAC;YAEzE,IAAI,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAM,CAAC,CAAC;YAEjD,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;gBAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,yBAAyB,EAAE,IAAI,CAAC,CAAC;YAC1D,CAAC;YAED,IAAI,WAAW,EAAE,CAAC;gBACjB,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;oBACzB,IAAI,KAAK,EAAE,CAAC;wBACX,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;oBAC5C,CAAC;yBACI,CAAC;wBACL,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;oBAC7C,CAAC;gBACF,CAAC;gBAED,YAAY,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;gBAClC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,QAAM,CAAC,CAAC;YACnC,CAAC;YAED,oDAAoD;YACpD,IAAI,WAAW,GAAG,IAAI,CAAC,eAAe,CAAC,QAAM,CAAC,CAAC;YAE/C,IAAI,WAAW,EAAE,CAAC;gBACjB,IAAI,GAAG,GAAwB;oBAC9B,SAAS,EAAE,SAAS;oBACpB,QAAQ,EAAE,KAAK;oBACf,IAAI,EAAE,MAAM;iBACZ,CAAC;gBAEF,IAAI,KAAK,EAAE,CAAC;oBACX,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC;oBACpB,GAAG,CAAC,IAAI,GAAG,MAAM,CAAC;gBACnB,CAAC;gBAED,IAAI,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;gBAEnE,IAAI,WAAW,EAAE,CAAC;oBACjB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;wBACpC,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,CAAC,cAAc,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI,SAAS,CAAC,CAAC;oBAC7H,CAAC;yBACI,CAAC;wBACL,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;oBAC/C,CAAC;gBACF,CAAC;gBAED,IAAI,IAAI,CAAC,eAAe,CAAC,QAAM,CAAC,EAAE,CAAC;oBAClC,OAAO,IAAI,CAAC,eAAe,CAAC,QAAM,CAAC,CAAC;gBACrC,CAAC;YACF,CAAC;YAED,sCAAsC;YACtC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;gBAC5B,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,CAAC;QACF,CAAC;IACF,CAAC;IAEM,mDAAiB,GAAxB,UAAyB,EAAuB,EAAE,OAA6B;QAC9E,IAAI,CAAC,EAAE,EAAE,CAAC;YACT,OAAO;QACR,CAAC;QAED,IAAI,EAAE,CAAC,UAAU,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;YAC/B,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;gBACjC,IAAM,MAAI,GAAG,OAAO,CAAC;gBAErB,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,MAAI,CAAC,CAAC;gBAC9B,IAAI,CAAC,aAAa,EAAE,CAAC;gBAErB,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC;gBACvC,EAAE,CAAC,KAAK,EAAE,CAAC;gBAEZ,IAAI,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAI,CAAC,MAAM,CAAC,CAAC;gBAEtD,IAAI,WAAW,EAAE,CAAC;oBACjB,YAAY,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;oBAClC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,MAAI,CAAC,MAAM,CAAC,CAAC;oBAEvC,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;wBACzB,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,yBAAyB,CAAC,CAAC;oBACvD,CAAC;gBACF,CAAC;YACD,CAAC;iBACI,CAAC;gBACL,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC;gBACvC,EAAE,CAAC,KAAK,EAAE,CAAC;YACZ,CAAC;YAED,OAAO;QACR,CAAC;QAED,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YACjC,IAAI,CAAC;gBACJ,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAClB,CAAC;YACD,OAAO,GAAG,EAAE,CAAC;gBACZ,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC;gBACvC,EAAE,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;YACtD,CAAC;YAED,OAAO;QACR,CAAC;QAED,IAAM,IAAI,GAAG,OAAO,CAAC;QACrB,IAAM,aAAa,GAAG,IAAA,eAAI,EAAC,IAAI,CAAC,CAAC;QAEjC,IAAI,CAAC;YACJ,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACxB,CAAC;QACD,OAAO,GAAG,EAAE,CAAC;YACZ,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC9B,IAAI,CAAC,aAAa,EAAE,CAAC;YAErB,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;gBAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,mBAAmB,EAAE,IAAI,CAAC,CAAC;YACpD,CAAC;YAED,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC;YACvC,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,iCAAiC,EAAE,GAAG,CAAC,CAAC;YAEtD,IAAI,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEtD,IAAI,WAAW,EAAE,CAAC;gBACjB,YAAY,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;gBAClC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAEvC,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;oBACzB,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,kCAAkC,IAAG,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,QAAQ,EAAE,CAAA,CAAC,CAAC;gBAClF,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IACF,8BAAC;AAAD,CAnoBA,AAmoBC,IAAA;AAnoBY,0DAAuB","file":"worker-dispatcher.manager.js","sourcesContent":["import * as WebSocket from 'ws';\nimport { ServerResponseModel, TaskPayload, TaskResponse } from '../models/server-message.model';\nimport { dateReviver, objectIdHexString } from '../util/common';\nimport { MethodManager } from './method.manager';\nimport { WebSocketManager } from './websocket.manager';\nimport { pack, unpack } from 'msgpackr';\n\nexport interface WorkerConnection {\n\tid: string;\n\tws: WebSocket.WebSocket;\n\tactiveTasks: { taskId: string; weight: number; method?: string }[];\n\tworkerIndex?: string;\n\tworkerInstance?: string;\n}\n\ninterface PendingTask {\n\ttimeout: NodeJS.Timeout;\n\tpromise?: {\n\t\t// eslint-disable-next-line no-unused-vars\n\t\tresolve: (value: any) => void;\n\t\t// eslint-disable-next-line no-unused-vars\n\t\treject: (reason?: any) => void;\n\t};\n\tmethod?: string;\n}\n\nexport class WorkerDispatcherManager {\n\tprivate _websocketManager: WebSocketManager;\n\tprivate _methodManager: MethodManager;\n\tprivate _workers: WorkerConnection[] = [];\n\tprivate _taskQueue: TaskPayload[] = [];\n\tprivate _clientRequests: { [taskId: string]: string } = {};\n\tprivate _pendingTasks: Map<string, PendingTask> = new Map();\n\tprivate _debugNoWorkerTaskIds: Set<string> = new Set();\n\n\tprivate MAX_CONCURRENCY = 10;\n\n\tconstructor() {}\n\n\tstatic create(websocketManager: WebSocketManager, methodManager: MethodManager) {\n\t\tconst workerDispatcherManager = new WorkerDispatcherManager();\n\t\tworkerDispatcherManager.initialize(websocketManager, methodManager);\n\t\treturn workerDispatcherManager;\n\t}\n\n\tpublic initialize(websocketManager: WebSocketManager, methodManager: MethodManager) {\n\t\tthis._websocketManager = websocketManager;\n\t\tthis._methodManager = methodManager;\n\t}\n\n\tpublic isSafeShutdown() {\n\t\treturn !this._taskQueue.length && !this._pendingTasks.size;\n\t}\n\n\tpublic addWorker(ws: WebSocket.WebSocket) {\n\t\tthis._workers.push({\n\t\t\tid: ws['id_worker'],\n\t\t\tws: ws,\n\t\t\tactiveTasks: [],\n\t\t\tworkerIndex: ws['workerIndex'],\n\t\t\tworkerInstance: ws['workerInstance']\n\t\t});\n\n\t\tsetInterval(() => {\n\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\tconsole.log(JSON.stringify(this._workers.map(a => {\n\t\t\t\t\treturn {\n\t\t\t\t\t\tid: a.id,\n\t\t\t\t\t\tactiveTasks: a.activeTasks\n\t\t\t\t\t}\n\t\t\t\t}), null, 2));\n\t\t\t}\n\t\t}, 5000);\n\n\t\tif (this._taskQueue.length) {\n\t\t\tthis.dispatchQueue();\n\t\t}\n\t}\n\n\tpublic disconnectWorker(workerId: string) {\n\t\tthis._workers = this._workers.filter(w => w.id !== workerId);\n\t}\n\n\tpublic hasWorkers() {\n\t\treturn this._workers.length > 0;\n\t}\n\n\tpublic getQueueSnapshot() {\n\t\tconst activeTasks = this._workers.reduce((total, worker) => total + (worker.activeTasks ? worker.activeTasks.length : 0), 0);\n\t\treturn {\n\t\t\tqueueDepth: this._taskQueue.length,\n\t\t\tpendingTasks: this._pendingTasks.size,\n\t\t\tworkerCount: this._workers.length,\n\t\t\tactiveTasks,\n\t\t\tworkers: this._workers.map(worker => ({\n\t\t\t\tid: worker.id,\n\t\t\t\tindex: worker.workerIndex,\n\t\t\t\tinstance: worker.workerInstance,\n\t\t\t\tactiveTasks: worker.activeTasks ? worker.activeTasks.length : 0\n\t\t\t}))\n\t\t};\n\t}\n\n\tpublic hasWorkersForMethod(methodName: string) {\n\t\tif (!this._workers.length) {\n\t\t\treturn false;\n\t\t}\n\n\t\tconst method = this._methodManager.getMethod(methodName);\n\t\tconst targetWorkerIndex = this.normalizeWorkerIndex(method?.targetWorkerIndex);\n\t\tconst targetWorkerInstance = this.normalizeWorkerIndex(method?.targetWorkerInstance);\n\n\t\tif (!targetWorkerIndex && !targetWorkerInstance) {\n\t\t\treturn true;\n\t\t}\n\n\t\treturn this._workers.some(worker =>\n\t\t\t(!targetWorkerIndex || this.normalizeWorkerIndex(worker.workerIndex) === targetWorkerIndex) &&\n\t\t\t(!targetWorkerInstance || this.normalizeWorkerIndex(worker.workerInstance) === targetWorkerInstance)\n\t\t);\n\t}\n\n\t/**\n\t * Add a new task to our in-memory queue and try to dispatch.\n\t */\n\tpublic sendClientTask(messageId: number, method: string, params: any[], userContext?: { id_user?: string; user?: string; id_ws?: string }) {\n\t\tlet taskId = 'task-' + objectIdHexString();\n\n\t\tthis._taskQueue.push({\n\t\t\ttype: 'task',\n\t\t\ttaskId,\n\t\t\tmessageId,\n\t\t\tmethod,\n\t\t\tparams,\n\t\t\tuserContext\n\t\t});\n\n\t\tif (userContext && userContext.id_ws) {\n\t\t\tthis._clientRequests[taskId] = userContext.id_ws;\n\t\t}\n\n\t\tthis.dispatchQueue();\n\t}\n\n\t/**\n\t * Same as sendInternalTask but returns a Promise so you can `await` it.\n\t */\n\tpublic sendInternalPromise(method: string, params: any[] = []): Promise<any> {\n\t\tif (!Array.isArray(params)) {\n\t\t\tparams = [params];\n\t\t}\n\n\t\t// eslint-disable-next-line no-restricted-syntax\n\t\treturn new Promise((resolve, reject) => {\n\t\t\tlet taskId = 'task-' + objectIdHexString();\n\n\t\t\tthis._taskQueue.push({\n\t\t\t\ttype: 'task',\n\t\t\t\ttaskId,\n\t\t\t\tmethod,\n\t\t\t\tparams,\n\t\t\t\tmessageId: 0,\n\t\t\t\tuserContext: {\n\t\t\t\t\tuser: 'Internal System'\n\t\t\t\t}\n\t\t\t});\n\n\t\t\tthis._pendingTasks.set(taskId, {\n\t\t\t\ttimeout: null,\n\t\t\t\tpromise: { resolve, reject }\n\t\t\t});\n\n\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\tconsole.log(new Date(), 'Send Internal Promise', this._taskQueue);\n\t\t\t}\n\n\t\t\tthis.dispatchQueue();\n\t\t});\n\t}\n\n\t/**\n\t * Send a task internally without returning a promise.\n\t */\n\tpublic sendInternalTask(method: string, params = []) {\n\t\tif (!Array.isArray(params)) {\n\t\t\tparams = [params];\n\t\t}\n\n\t\tlet taskId = 'task-' + objectIdHexString();\n\n\t\tthis._taskQueue.push({\n\t\t\ttype: 'task',\n\t\t\ttaskId,\n\t\t\tmethod,\n\t\t\tparams,\n\t\t\tmessageId: 0,\n\t\t\tuserContext: {\n\t\t\t\tuser: 'Internal System'\n\t\t\t}\n\t\t});\n\n\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\tconsole.log(new Date(), 'Send Internal Task', this._taskQueue);\n\t\t}\n\n\t\tthis.dispatchQueue();\n\t}\n\n\t/**\n\t * The main loop that assigns tasks from _taskQueue to any worker that has capacity.\n\t */\n\tprivate dispatchQueue() {\n\t\tif (!this._taskQueue.length) {\n\t\t\treturn;\n\t\t}\n\n\t\twhile (this._taskQueue.length > 0) {\n\t\t\tlet assigned = false;\n\n\t\t\tfor (let i = 0; i < this._taskQueue.length; i++) {\n\t\t\t\tlet task = this._taskQueue[i];\n\t\t\t\tlet targetWorker = this.findWorkerForTask(task);\n\n\t\t\t\tif (targetWorker) {\n\t\t\t\t\tthis._taskQueue.splice(i, 1);\n\t\t\t\t\tthis.assignTaskToWorker(targetWorker, task);\n\t\t\t\t\tassigned = true;\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif (!assigned) {\n\t\t\t\tthis.logNoWorkerAvailability();\n\t\t\t\tsetTimeout(() => {\n\t\t\t\t\tthis.dispatchQueue();\n\t\t\t\t}, 25);\n\n\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\tconsole.log(new Date(), 'No Worker Available', JSON.stringify(this._workers, null, 2));\n\t\t\t\t}\n\n\t\t\t\treturn; // nothing can run right now due to per-worker limits\n\t\t\t}\n\t\t}\n\t}\n\n\t/**\n\t * Returns the worker with the fewest activeTasks that is under maxConcurrency and per-method limit.\n\t */\n\tprivate findWorkerForTask(task: TaskPayload): WorkerConnection | null {\n\t\tlet method = this._methodManager.getMethod(task.method);\n\t\tlet methodLimit = method && method.maxConcurrency && method.maxConcurrency > 0 ? method.maxConcurrency : null;\n\t\tlet methodLimitPerInstance = method && method.maxConcurrencyPerInstance && method.maxConcurrencyPerInstance > 0\n\t\t\t? method.maxConcurrencyPerInstance\n\t\t\t: null;\n\t\tlet targetWorkerIndex = this.normalizeWorkerIndex(method?.targetWorkerIndex);\n\t\tlet targetWorkerInstance = this.normalizeWorkerIndex(method?.targetWorkerInstance);\n\t\tconst preferNonZero = !targetWorkerIndex;\n\n\t\tlet candidates = this._workers.filter(x => x.activeTasks.length < this.MAX_CONCURRENCY);\n\t\tif (targetWorkerIndex) {\n\t\t\tcandidates = candidates.filter(worker => this.normalizeWorkerIndex(worker.workerIndex) === targetWorkerIndex);\n\t\t}\n\t\tif (targetWorkerInstance) {\n\t\t\tcandidates = candidates.filter(worker => this.normalizeWorkerIndex(worker.workerInstance) === targetWorkerInstance);\n\t\t}\n\t\tif (!candidates.length) {\n\t\t\treturn null;\n\t\t}\n\n\t\tlet eligible = candidates.filter((worker) => {\n\t\t\tif (!methodLimit) {\n\t\t\t\tif (!methodLimitPerInstance) {\n\t\t\t\t\treturn true;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif (methodLimitPerInstance) {\n\t\t\t\tconst currentInstance = worker.activeTasks.filter(a => a.method === task.method).length;\n\t\t\t\tif (currentInstance >= methodLimitPerInstance) {\n\t\t\t\t\treturn false;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif (!methodLimit) {\n\t\t\t\treturn true;\n\t\t\t}\n\n\t\t\tlet idx = worker.workerIndex || 'unknown';\n\t\t\tlet current = 0;\n\n\t\t\tthis._workers.forEach((w) => {\n\t\t\t\tif ((w.workerIndex || 'unknown') === idx) {\n\t\t\t\t\tcurrent += w.activeTasks.filter(a => a.method === task.method).length;\n\t\t\t\t}\n\t\t\t});\n\n\t\t\treturn current < methodLimit;\n\t\t});\n\n\t\tif (!eligible.length) {\n\t\t\treturn null;\n\t\t}\n\n\t\teligible.sort((x, y) => {\n\t\t\tconst totalX = this.getWorkerLoad(x);\n\t\t\tconst totalY = this.getWorkerLoad(y);\n\t\t\tif (totalX !== totalY) {\n\t\t\t\treturn totalX - totalY;\n\t\t\t}\n\n\t\t\tif (preferNonZero) {\n\t\t\t\tconst xZero = this.isWorkerZero(x) ? 1 : 0;\n\t\t\t\tconst yZero = this.isWorkerZero(y) ? 1 : 0;\n\t\t\t\tif (xZero !== yZero) {\n\t\t\t\t\treturn xZero - yZero;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tconst idxX = this.normalizeWorkerIndex(x.workerIndex) || '';\n\t\t\tconst idxY = this.normalizeWorkerIndex(y.workerIndex) || '';\n\t\t\tconst idxCompare = idxX.localeCompare(idxY, undefined, { numeric: true, sensitivity: 'base' });\n\t\t\tif (idxCompare !== 0) {\n\t\t\t\treturn idxCompare;\n\t\t\t}\n\n\t\t\tconst instX = this.normalizeWorkerIndex(x.workerInstance) || '';\n\t\t\tconst instY = this.normalizeWorkerIndex(y.workerInstance) || '';\n\t\t\treturn instX.localeCompare(instY, undefined, { numeric: true, sensitivity: 'base' });\n\t\t});\n\n\t\treturn eligible[0];\n\t}\n\n\tprivate normalizeWorkerIndex(value?: string | number | null): string | null {\n\t\tif (value === null || value === undefined) {\n\t\t\treturn null;\n\t\t}\n\n\t\tconst normalized = String(value).trim();\n\t\treturn normalized.length ? normalized : null;\n\t}\n\n\tprivate getWorkerLoad(worker: WorkerConnection): number {\n\t\treturn worker.activeTasks.reduce((sum, task) => sum + task.weight, 0);\n\t}\n\n\tprivate isWorkerZero(worker: WorkerConnection): boolean {\n\t\treturn this.normalizeWorkerIndex(worker.workerIndex) === '0';\n\t}\n\n\tprivate assignTaskToWorker(worker: WorkerConnection, task: TaskPayload) {\n\t\tlet method = this._methodManager.getMethod(task.method);\n\n\t\tif (!method) {\n\t\t\tconsole.error('Failed to send task to worker - Could not find method:', task.method);\n\t\t\treturn;\n\t\t}\n\n\t\tworker.activeTasks.push({\n\t\t\ttaskId: task.taskId,\n\t\t\tweight: method && method.workerTaskWeight ? method.workerTaskWeight : 1,\n\t\t\tmethod: task.method\n\t\t});\n\t\tthis._debugNoWorkerTaskIds.delete(task.taskId);\n\n\t\tlet payload: TaskPayload = {\n\t\t\ttype: 'task',\n\t\t\ttaskId: task.taskId,\n\t\t\tmessageId: task.messageId,\n\t\t\tmethod: task.method,\n\t\t\tparams: task.params,\n\t\t\tuserContext: task.userContext || {}\n\t\t};\n\n\t\tlet timeoutHandle = setTimeout(() => {\n\t\t\tlet pending = this._pendingTasks.get(task.taskId);\n\n\t\t\tif (pending) {\n\t\t\t\tif (pending.promise) {\n\t\t\t\t\tpending.promise.reject('Task timed out after 2m in WorkerDispatcherManager for method: ' + task.method);\n\t\t\t\t}\n\n\t\t\t\tthis._pendingTasks.delete(task.taskId);\n\t\t\t}\n\n\t\t\tconsole.log(new Date(), 'TIMEOUT HIT', 'task', task.taskId, task.messageId, task.method, JSON.stringify(this._workers.map(a => a.activeTasks)));\n\n\t\t\tworker.activeTasks = worker.activeTasks.filter(a => a.taskId !== task.taskId);\n\n\t\t\tif (task.userContext && task.userContext.id_ws) {\n\t\t\t\tlet timeoutRes: ServerResponseModel = {\n\t\t\t\t\tmessageId: task.messageId,\n\t\t\t\t\thasError: true,\n\t\t\t\t\tdata: 'Task timed out after 2m in WorkerDispatcherManager for method: ' + task.method\n\t\t\t\t};\n\n\t\t\t\tlet clientWS = this._websocketManager.getWebSocket(task.userContext.id_ws);\n\n\t\t\t\tif (clientWS) {\n\t\t\t\t\tthis._websocketManager.send(clientWS, timeoutRes);\n\t\t\t\t}\n\t\t\t}\n\t\t}, method.timeoutOverride || (1000 * 60 * 2));\n\n\t\t// If we already stored a promise for this task (from sendInternalPromise), add the timeout now\n\t\tlet existing = this._pendingTasks.get(task.taskId);\n\t\tif (!existing) {\n\t\t\texisting = { timeout: null, method: task.method };\n\t\t}\n\t\telse {\n\t\t\texisting.method = task.method;\n\t\t}\n\n\t\texisting.timeout = timeoutHandle;\n\t\tthis._pendingTasks.set(task.taskId, existing);\n\n\t\ttry {\n\t\t\tthis.sendWorkerPayload(worker.ws, payload);\n\t\t}\n\t\tcatch (err) {\n\t\t\tconsole.error('Failed to send task to worker:', err);\n\n\t\t\tclearTimeout(timeoutHandle);\n\t\t\tworker.activeTasks = worker.activeTasks.filter(a => a.taskId !== task.taskId);\n\n\t\t\t// Put task back in front\n\t\t\tthis._taskQueue.unshift(task);\n\t\t\tthis.dispatchQueue();\n\t\t}\n\t}\n\n\tprivate logNoWorkerAvailability() {\n\t\tif (!this.shouldDebug()) {\n\t\t\treturn;\n\t\t}\n\n\t\tconst workers = this._workers.map(worker => ({\n\t\t\tid: worker.id,\n\t\t\tindex: this.normalizeWorkerIndex(worker.workerIndex),\n\t\t\tinstance: this.normalizeWorkerIndex(worker.workerInstance),\n\t\t\tactiveTasks: worker.activeTasks.length\n\t\t}));\n\n\t\tfor (const task of this._taskQueue) {\n\t\t\tconst method = this._methodManager.getMethod(task.method);\n\t\t\tconst targetWorkerIndex = this.normalizeWorkerIndex(method?.targetWorkerIndex);\n\t\t\tconst targetWorkerInstance = this.normalizeWorkerIndex(method?.targetWorkerInstance);\n\n\t\t\tif (!targetWorkerIndex && !targetWorkerInstance) {\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tif (this._debugNoWorkerTaskIds.has(task.taskId)) {\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tconst hasMatch = this._workers.some(worker =>\n\t\t\t\t(!targetWorkerIndex || this.normalizeWorkerIndex(worker.workerIndex) === targetWorkerIndex) &&\n\t\t\t\t(!targetWorkerInstance || this.normalizeWorkerIndex(worker.workerInstance) === targetWorkerInstance)\n\t\t\t);\n\n\t\t\tif (!hasMatch) {\n\t\t\t\tthis._debugNoWorkerTaskIds.add(task.taskId);\n\t\t\t\tconsole.warn(new Date(), '[WorkerDispatcher] No matching worker for task', {\n\t\t\t\t\ttaskId: task.taskId,\n\t\t\t\t\tmethod: task.method,\n\t\t\t\t\ttargetWorkerIndex,\n\t\t\t\t\ttargetWorkerInstance,\n\t\t\t\t\tworkers\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t}\n\n\tprivate shouldDebug(): boolean {\n\t\treturn this._methodManager.getEnableDebug() || process.env.WORKER_DISPATCH_DEBUG === 'true';\n\t}\n\n\t/**\n\t * Handle messages coming back from a worker (like 'taskComplete').\n\t */\n\tpublic handleWorkerMessage(workerId: string, rawMessage: WebSocket.RawData) {\n\t\tlet data: TaskResponse;\n\n\t\ttry {\n\t\t\tif (typeof rawMessage === 'string') {\n\t\t\t\tdata = JSON.parse(rawMessage, dateReviver);\n\t\t\t}\n\t\t\telse if (Buffer.isBuffer(rawMessage)) {\n\t\t\t\tdata = unpack(rawMessage);\n\t\t\t}\n\t\t\telse if (Array.isArray(rawMessage)) {\n\t\t\t\tconst chunks = rawMessage as unknown as ReadonlyArray<Uint8Array>;\n\t\t\t\tdata = unpack(Buffer.concat(chunks));\n\t\t\t}\n\t\t\telse if (rawMessage instanceof ArrayBuffer) {\n\t\t\t\tdata = unpack(Buffer.from(rawMessage));\n\t\t\t}\n\t\t\telse if (ArrayBuffer.isView(rawMessage)) {\n\t\t\t\tconst view = rawMessage as NodeJS.ArrayBufferView;\n\t\t\t\tdata = unpack(Buffer.from(view.buffer, view.byteOffset, view.byteLength));\n\t\t\t}\n\t\t\telse {\n\t\t\t\tconsole.error('Unsupported worker message type received:', typeof rawMessage);\n\t\t\t\treturn;\n\t\t\t}\n\t\t}\n\t\tcatch (err) {\n\t\t\tconsole.error('Failed to parse worker message:', err);\n\t\t\treturn;\n\t\t}\n\n\t\tif (data.type === 'taskComplete') {\n\t\t\tlet worker = this._workers.find(x => x.id === workerId);\n\n\t\t\tif (!worker) {\n\t\t\t\tconsole.error('Unknown worker for taskComplete:', workerId);\n\t\t\t\treturn;\n\t\t\t}\n\n\t\t\tlet { taskId, messageId, error, result } = data;\n\t\t\tlet unpackedResult = result;\n\n\t\t\tif (!error && unpackedResult === null && data['packedResult']) {\n\t\t\t\ttry {\n\t\t\t\t\tunpackedResult = unpack(data['packedResult']);\n\t\t\t\t}\n\t\t\t\tcatch (err) {\n\t\t\t\t\tconsole.error('Failed to unpack worker result', err);\n\t\t\t\t}\n\t\t\t}\n\t\t\tworker.activeTasks = worker.activeTasks.filter(a => a.taskId !== taskId);\n\n\t\t\tlet pendingTask = this._pendingTasks.get(taskId);\n\n\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\tconsole.log(new Date(), 'Recv from Worker Server', data);\n\t\t\t}\n\n\t\t\tif (pendingTask) {\n\t\t\t\tif (pendingTask.promise) {\n\t\t\t\t\tif (error) {\n\t\t\t\t\t\tpendingTask.promise.reject(unpackedResult);\n\t\t\t\t\t}\n\t\t\t\t\telse {\n\t\t\t\t\t\tpendingTask.promise.resolve(unpackedResult);\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tclearTimeout(pendingTask.timeout);\n\t\t\t\tthis._pendingTasks.delete(taskId);\n\t\t\t}\n\n\t\t\t// Look up original request if it came from a client\n\t\t\tlet clientReqId = this._clientRequests[taskId];\n\n\t\t\tif (clientReqId) {\n\t\t\t\tlet res: ServerResponseModel = {\n\t\t\t\t\tmessageId: messageId,\n\t\t\t\t\thasError: false,\n\t\t\t\t\tdata: result\n\t\t\t\t};\n\n\t\t\t\tif (error) {\n\t\t\t\t\tres.hasError = true;\n\t\t\t\t\tres.data = result;\n\t\t\t\t}\n\n\t\t\t\tlet clientReqWS = this._websocketManager.getWebSocket(clientReqId);\n\n\t\t\t\tif (clientReqWS) {\n\t\t\t\t\tif (!error && data['packedResult']) {\n\t\t\t\t\t\tthis._websocketManager.sendPackedBuffer(clientReqWS, messageId, false, data['packedResult'], data['encoding'] || 'msgpack');\n\t\t\t\t\t}\n\t\t\t\t\telse {\n\t\t\t\t\t\tthis._websocketManager.send(clientReqWS, res);\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif (this._clientRequests[taskId]) {\n\t\t\t\t\tdelete this._clientRequests[taskId];\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Try to dispatch more from the queue\n\t\t\tif (this._taskQueue.length) {\n\t\t\t\tthis.dispatchQueue();\n\t\t\t}\n\t\t}\n\t}\n\n\tpublic sendWorkerPayload(ws: WebSocket.WebSocket, payload: TaskPayload | string) {\n\t\tif (!ws) {\n\t\t\treturn;\n\t\t}\n\n\t\tif (ws.readyState !== ws.OPEN) {\n\t\t\tif (typeof payload !== 'string') {\n\t\t\t\tconst task = payload;\n\n\t\t\t\tthis._taskQueue.unshift(task);\n\t\t\t\tthis.dispatchQueue();\n\n\t\t\t\tthis.disconnectWorker(ws['id_worker']);\n\t\t\t\tws.close();\n\n\t\t\tlet pendingTask = this._pendingTasks.get(task.taskId);\n\n\t\t\tif (pendingTask) {\n\t\t\t\tclearTimeout(pendingTask.timeout);\n\t\t\t\tthis._pendingTasks.delete(task.taskId);\n\n\t\t\t\tif (pendingTask.promise) {\n\t\t\t\t\tpendingTask.promise.reject('Worker socket not open.');\n\t\t\t\t}\n\t\t\t}\n\t\t\t}\n\t\t\telse {\n\t\t\t\tthis.disconnectWorker(ws['id_worker']);\n\t\t\t\tws.close();\n\t\t\t}\n\n\t\t\treturn;\n\t\t}\n\n\t\tif (typeof payload === 'string') {\n\t\t\ttry {\n\t\t\t\tws.send(payload);\n\t\t\t}\n\t\t\tcatch (err) {\n\t\t\t\tthis.disconnectWorker(ws['id_worker']);\n\t\t\t\tws.close();\n\t\t\t\tconsole.error('Failed to send worker payload:', err);\n\t\t\t}\n\n\t\t\treturn;\n\t\t}\n\n\t\tconst task = payload;\n\t\tconst payloadBuffer = pack(task);\n\n\t\ttry {\n\t\t\tws.send(payloadBuffer);\n\t\t}\n\t\tcatch (err) {\n\t\t\tthis._taskQueue.unshift(task);\n\t\t\tthis.dispatchQueue();\n\n\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\tconsole.log(new Date(), 'Sending to Server', task);\n\t\t\t}\n\n\t\t\tthis.disconnectWorker(ws['id_worker']);\n\t\t\tws.close();\n\t\t\tconsole.error('Failed to send worker response:', err);\n\n\t\t\tlet pendingTask = this._pendingTasks.get(task.taskId);\n\n\t\t\tif (pendingTask) {\n\t\t\t\tclearTimeout(pendingTask.timeout);\n\t\t\t\tthis._pendingTasks.delete(task.taskId);\n\n\t\t\t\tif (pendingTask.promise) {\n\t\t\t\t\tpendingTask.promise.reject('Failed to send worker response: ' + err?.toString());\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n"]}
+{"version":3,"sources":["../../src/managers/worker-dispatcher.manager.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAEA,yCAAuE;AAGvE,qCAAwC;AA4BxC;IAmBC;QAhBQ,aAAQ,GAAuB,EAAE,CAAC;QAClC,eAAU,GAAkB,EAAE,CAAC;QAC/B,oBAAe,GAAiC,EAAE,CAAC;QACnD,kBAAa,GAA6B,IAAI,GAAG,EAAE,CAAC;QACpD,0BAAqB,GAAgB,IAAI,GAAG,EAAE,CAAC;QAC/C,iBAAY,GAMf,IAAI,GAAG,EAAE,CAAC;QAEP,oBAAe,GAAG,EAAE,CAAC;QACrB,iCAA4B,GAAG,GAAG,CAAC;IAE5B,CAAC;IAET,8BAAM,GAAb,UAAc,gBAAkC,EAAE,aAA4B;QAC7E,IAAM,uBAAuB,GAAG,IAAI,uBAAuB,EAAE,CAAC;QAC9D,uBAAuB,CAAC,UAAU,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAC;QACpE,OAAO,uBAAuB,CAAC;IAChC,CAAC;IAEM,4CAAU,GAAjB,UAAkB,gBAAkC,EAAE,aAA4B;QACjF,IAAI,CAAC,iBAAiB,GAAG,gBAAgB,CAAC;QAC1C,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;IACrC,CAAC;IAEM,gDAAc,GAArB;QACC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;IAC5D,CAAC;IAEM,2CAAS,GAAhB,UAAiB,EAAuB;QAAxC,iBAuBC;QAtBA,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAClB,EAAE,EAAE,EAAE,CAAC,WAAW,CAAC;YACnB,EAAE,EAAE,EAAE;YACN,WAAW,EAAE,EAAE;YACf,WAAW,EAAE,EAAE,CAAC,aAAa,CAAC;YAC9B,cAAc,EAAE,EAAE,CAAC,gBAAgB,CAAC;SACpC,CAAC,CAAC;QAEH,WAAW,CAAC;YACX,IAAI,KAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;gBAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,KAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAA,CAAC;oBAC7C,OAAO;wBACN,EAAE,EAAE,CAAC,CAAC,EAAE;wBACR,WAAW,EAAE,CAAC,CAAC,WAAW;qBAC1B,CAAA;gBACF,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACf,CAAC;QACF,CAAC,EAAE,IAAI,CAAC,CAAC;QAET,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YAC5B,IAAI,CAAC,aAAa,EAAE,CAAC;QACtB,CAAC;IACF,CAAC;IAEM,kDAAgB,GAAvB,UAAwB,QAAgB;QACvC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,CAAC,EAAE,KAAK,QAAQ,EAAjB,CAAiB,CAAC,CAAC;IAC9D,CAAC;IAEM,4CAAU,GAAjB;QACC,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;IACjC,CAAC;IAEM,kDAAgB,GAAvB;QACC,IAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAC,KAAK,EAAE,MAAM,IAAK,OAAA,KAAK,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAA5D,CAA4D,EAAE,CAAC,CAAC,CAAC;QAC7H,OAAO;YACN,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM;YAClC,YAAY,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI;YACrC,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;YACjC,WAAW,aAAA;YACX,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAA,MAAM,IAAI,OAAA,CAAC;gBACrC,EAAE,EAAE,MAAM,CAAC,EAAE;gBACb,KAAK,EAAE,MAAM,CAAC,WAAW;gBACzB,QAAQ,EAAE,MAAM,CAAC,cAAc;gBAC/B,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;aAC/D,CAAC,EALmC,CAKnC,CAAC;SACH,CAAC;IACH,CAAC;IAEM,qDAAmB,GAA1B,UAA2B,UAAkB;QAA7C,iBAiBC;QAhBA,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC3B,OAAO,KAAK,CAAC;QACd,CAAC;QAED,IAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QACzD,IAAM,iBAAiB,GAAG,IAAI,CAAC,oBAAoB,CAAC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,iBAAiB,CAAC,CAAC;QAC/E,IAAM,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,oBAAoB,CAAC,CAAC;QAErF,IAAI,CAAC,iBAAiB,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACjD,OAAO,IAAI,CAAC;QACb,CAAC;QAED,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAA,MAAM;YAC/B,OAAA,CAAC,CAAC,iBAAiB,IAAI,KAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,iBAAiB,CAAC;gBAC3F,CAAC,CAAC,oBAAoB,IAAI,KAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,cAAc,CAAC,KAAK,oBAAoB,CAAC;QADpG,CACoG,CACpG,CAAC;IACH,CAAC;IAED;;OAEG;IACI,gDAAc,GAArB,UAAsB,SAAiB,EAAE,MAAc,EAAE,MAAa,EAAE,WAAiE;QACxI,IAAI,MAAM,GAAG,OAAO,GAAG,IAAA,0BAAiB,GAAE,CAAC;QAC3C,IAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE5B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACpB,IAAI,EAAE,MAAM;YACZ,MAAM,QAAA;YACN,SAAS,WAAA;YACT,MAAM,QAAA;YACN,MAAM,QAAA;YACN,WAAW,aAAA;YACX,QAAQ,UAAA;SACR,CAAC,CAAC;QAEH,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE;YAC7B,QAAQ,UAAA;YACR,MAAM,QAAA;YACN,SAAS,WAAA;YACT,WAAW,aAAA;SACX,CAAC,CAAC;QAEH,IAAI,WAAW,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;YACtC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC;QAClD,CAAC;QAED,IAAI,CAAC,aAAa,EAAE,CAAC;IACtB,CAAC;IAED;;OAEG;IACI,qDAAmB,GAA1B,UAA2B,MAAc,EAAE,MAAkB;QAA7D,iBAwCC;QAxC0C,uBAAA,EAAA,WAAkB;QAC5D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC;QACnB,CAAC;QAED,gDAAgD;QAChD,OAAO,IAAI,OAAO,CAAC,UAAC,OAAO,EAAE,MAAM;YAClC,IAAI,MAAM,GAAG,OAAO,GAAG,IAAA,0BAAiB,GAAE,CAAC;YAC3C,IAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAE5B,KAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACpB,IAAI,EAAE,MAAM;gBACZ,MAAM,QAAA;gBACN,MAAM,QAAA;gBACN,MAAM,QAAA;gBACN,SAAS,EAAE,CAAC;gBACZ,WAAW,EAAE;oBACZ,IAAI,EAAE,iBAAiB;iBACvB;gBACD,QAAQ,UAAA;aACR,CAAC,CAAC;YAEH,KAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE;gBAC7B,QAAQ,UAAA;gBACR,MAAM,QAAA;gBACN,SAAS,EAAE,CAAC;gBACZ,WAAW,EAAE,EAAE,IAAI,EAAE,iBAAiB,EAAE;aACxC,CAAC,CAAC;YAEH,KAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,EAAE;gBAC9B,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE,EAAE,OAAO,SAAA,EAAE,MAAM,QAAA,EAAE;aAC5B,CAAC,CAAC;YAEH,IAAI,KAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;gBAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,uBAAuB,EAAE,KAAI,CAAC,UAAU,CAAC,CAAC;YACnE,CAAC;YAED,KAAI,CAAC,aAAa,EAAE,CAAC;QACtB,CAAC,CAAC,CAAC;IACJ,CAAC;IAED;;OAEG;IACI,kDAAgB,GAAvB,UAAwB,MAAc,EAAE,MAAW;QAAX,uBAAA,EAAA,WAAW;QAClD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC;QACnB,CAAC;QAED,IAAI,MAAM,GAAG,OAAO,GAAG,IAAA,0BAAiB,GAAE,CAAC;QAC3C,IAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE5B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACpB,IAAI,EAAE,MAAM;YACZ,MAAM,QAAA;YACN,MAAM,QAAA;YACN,MAAM,QAAA;YACN,SAAS,EAAE,CAAC;YACZ,WAAW,EAAE;gBACZ,IAAI,EAAE,iBAAiB;aACvB;YACD,QAAQ,UAAA;SACR,CAAC,CAAC;QAEH,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE;YAC7B,QAAQ,UAAA;YACR,MAAM,QAAA;YACN,SAAS,EAAE,CAAC;YACZ,WAAW,EAAE,EAAE,IAAI,EAAE,iBAAiB,EAAE;SACxC,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;YAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,oBAAoB,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAChE,CAAC;QAED,IAAI,CAAC,aAAa,EAAE,CAAC;IACtB,CAAC;IAED;;OAEG;IACK,+CAAa,GAArB;QAAA,iBAiCC;QAhCA,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YAC7B,OAAO;QACR,CAAC;QAED,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,IAAI,QAAQ,GAAG,KAAK,CAAC;YAErB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACjD,IAAI,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;gBAC9B,IAAI,YAAY,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;gBAEhD,IAAI,YAAY,EAAE,CAAC;oBAClB,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;oBAC7B,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;oBAC5C,QAAQ,GAAG,IAAI,CAAC;oBAChB,MAAM;gBACP,CAAC;YACF,CAAC;YAED,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACf,IAAI,CAAC,uBAAuB,EAAE,CAAC;gBAC/B,UAAU,CAAC;oBACV,KAAI,CAAC,aAAa,EAAE,CAAC;gBACtB,CAAC,EAAE,EAAE,CAAC,CAAC;gBAEP,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;oBAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,qBAAqB,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;gBACxF,CAAC;gBAED,OAAO,CAAC,qDAAqD;YAC9D,CAAC;QACF,CAAC;IACF,CAAC;IAED;;OAEG;IACK,mDAAiB,GAAzB,UAA0B,IAAiB;QAA3C,iBAmFC;QAlFA,IAAI,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxD,IAAI,WAAW,GAAG,MAAM,IAAI,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9G,IAAI,sBAAsB,GAAG,MAAM,IAAI,MAAM,CAAC,yBAAyB,IAAI,MAAM,CAAC,yBAAyB,GAAG,CAAC;YAC9G,CAAC,CAAC,MAAM,CAAC,yBAAyB;YAClC,CAAC,CAAC,IAAI,CAAC;QACR,IAAI,iBAAiB,GAAG,IAAI,CAAC,oBAAoB,CAAC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,iBAAiB,CAAC,CAAC;QAC7E,IAAI,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,CAAC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,oBAAoB,CAAC,CAAC;QACnF,IAAM,aAAa,GAAG,CAAC,iBAAiB,CAAC;QAEzC,IAAI,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,KAAI,CAAC,eAAe,EAA3C,CAA2C,CAAC,CAAC;QACxF,IAAI,iBAAiB,EAAE,CAAC;YACvB,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,UAAA,MAAM,IAAI,OAAA,KAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,iBAAiB,EAAnE,CAAmE,CAAC,CAAC;QAC/G,CAAC;QACD,IAAI,oBAAoB,EAAE,CAAC;YAC1B,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,UAAA,MAAM,IAAI,OAAA,KAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,cAAc,CAAC,KAAK,oBAAoB,EAAzE,CAAyE,CAAC,CAAC;QACrH,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YACxB,OAAO,IAAI,CAAC;QACb,CAAC;QAED,IAAI,QAAQ,GAAG,UAAU,CAAC,MAAM,CAAC,UAAC,MAAM;YACvC,IAAI,CAAC,WAAW,EAAE,CAAC;gBAClB,IAAI,CAAC,sBAAsB,EAAE,CAAC;oBAC7B,OAAO,IAAI,CAAC;gBACb,CAAC;YACF,CAAC;YAED,IAAI,sBAAsB,EAAE,CAAC;gBAC5B,IAAM,eAAe,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAxB,CAAwB,CAAC,CAAC,MAAM,CAAC;gBACxF,IAAI,eAAe,IAAI,sBAAsB,EAAE,CAAC;oBAC/C,OAAO,KAAK,CAAC;gBACd,CAAC;YACF,CAAC;YAED,IAAI,CAAC,WAAW,EAAE,CAAC;gBAClB,OAAO,IAAI,CAAC;YACb,CAAC;YAED,IAAI,GAAG,GAAG,MAAM,CAAC,WAAW,IAAI,SAAS,CAAC;YAC1C,IAAI,OAAO,GAAG,CAAC,CAAC;YAEhB,KAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAC,CAAC;gBACvB,IAAI,CAAC,CAAC,CAAC,WAAW,IAAI,SAAS,CAAC,KAAK,GAAG,EAAE,CAAC;oBAC1C,OAAO,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAxB,CAAwB,CAAC,CAAC,MAAM,CAAC;gBACvE,CAAC;YACF,CAAC,CAAC,CAAC;YAEH,OAAO,OAAO,GAAG,WAAW,CAAC;QAC9B,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC;QACb,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC,UAAC,CAAC,EAAE,CAAC;YAClB,IAAM,MAAM,GAAG,KAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;YACrC,IAAM,MAAM,GAAG,KAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;YACrC,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;gBACvB,OAAO,MAAM,GAAG,MAAM,CAAC;YACxB,CAAC;YAED,IAAI,aAAa,EAAE,CAAC;gBACnB,IAAM,KAAK,GAAG,KAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3C,IAAM,KAAK,GAAG,KAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3C,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC;oBACrB,OAAO,KAAK,GAAG,KAAK,CAAC;gBACtB,CAAC;YACF,CAAC;YAED,IAAM,IAAI,GAAG,KAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;YAC5D,IAAM,IAAI,GAAG,KAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;YAC5D,IAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;YAC/F,IAAI,UAAU,KAAK,CAAC,EAAE,CAAC;gBACtB,OAAO,UAAU,CAAC;YACnB,CAAC;YAED,IAAM,KAAK,GAAG,KAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAChE,IAAM,KAAK,GAAG,KAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAChE,OAAO,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;QACtF,CAAC,CAAC,CAAC;QAEH,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAEO,sDAAoB,GAA5B,UAA6B,KAA8B;QAC1D,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YAC3C,OAAO,IAAI,CAAC;QACb,CAAC;QAED,IAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;QACxC,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9C,CAAC;IAEO,+CAAa,GAArB,UAAsB,MAAwB;QAC7C,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,UAAC,GAAG,EAAE,IAAI,IAAK,OAAA,GAAG,GAAG,IAAI,CAAC,MAAM,EAAjB,CAAiB,EAAE,CAAC,CAAC,CAAC;IACvE,CAAC;IAEO,8CAAY,GAApB,UAAqB,MAAwB;QAC5C,OAAO,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,GAAG,CAAC;IAC9D,CAAC;IAEO,oDAAkB,GAA1B,UAA2B,MAAwB,EAAE,IAAiB;QAAtE,iBA+GC;QA9GA,IAAI,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAExD,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,wDAAwD,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACrF,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtC,OAAO;QACR,CAAC;QAED,IAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAChC,IAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;QAChF,MAAM,CAAC,YAAY,GAAG,YAAY,CAAC;QACnC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC5B,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAClC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAE3C,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,MAAM,IAAI,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;YACvE,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,SAAS,EAAE,YAAY;YACvB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,WAAW,EAAE,IAAI,CAAC,WAAW;SAC7B,CAAC,CAAC;QACH,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE/C,IAAI,OAAO,GAAgB;YAC1B,IAAI,EAAE,MAAM;YACZ,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,EAAE;YACnC,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,YAAY,cAAA;SACZ,CAAC;QAEF,IAAI,aAAa,GAAG,UAAU,CAAC;YAC9B,IAAI,OAAO,GAAG,KAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAClD,IAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,IAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;YACjC,IAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACrE,IAAM,mBAAmB,GAAG,IAAA,cAAK,EAAC,SAAS,GAAG,YAAY,CAAC,CAAC;YAE5D,IAAI,OAAO,EAAE,CAAC;gBACb,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;oBACrB,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,iEAAiE,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;gBACzG,CAAC;gBAED,KAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACxC,CAAC;YAED,KAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE;gBACxC,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,WAAW,aAAA;gBACX,mBAAmB,qBAAA;gBACnB,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,MAAM,EAAE;oBACP,EAAE,EAAE,MAAM,CAAC,EAAE;oBACb,KAAK,EAAE,MAAM,CAAC,WAAW;oBACzB,QAAQ,EAAE,MAAM,CAAC,cAAc;iBAC/B;gBACD,OAAO,EAAE,KAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,CAAC,WAAW,EAAb,CAAa,CAAC;aAC9C,CAAC,CAAC;YAEH,MAAM,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAxB,CAAwB,CAAC,CAAC;YAC9E,KAAI,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEtC,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;gBAChD,IAAI,UAAU,GAAwB;oBACrC,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,QAAQ,EAAE,IAAI;oBACd,IAAI,EAAE,iEAAiE,GAAG,IAAI,CAAC,MAAM;iBACrF,CAAC;gBAEF,IAAI,QAAQ,GAAG,KAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;gBAE3E,IAAI,QAAQ,EAAE,CAAC;oBACd,KAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;gBACnD,CAAC;YACF,CAAC;QACF,CAAC,EAAE,MAAM,CAAC,eAAe,IAAI,CAAC,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QAE9C,+FAA+F;QAC/F,IAAI,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,QAAQ,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;QACnD,CAAC;aACI,CAAC;YACL,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC/B,CAAC;QAED,QAAQ,CAAC,OAAO,GAAG,aAAa,CAAC;QACjC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAE9C,IAAI,CAAC;YACJ,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,GAAG,EAAE,CAAC;YACZ,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;YAErD,YAAY,CAAC,aAAa,CAAC,CAAC;YAC5B,MAAM,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAxB,CAAwB,CAAC,CAAC;YAE9E,yBAAyB;YACzB,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC9B,IAAI,CAAC,aAAa,EAAE,CAAC;QACtB,CAAC;IACF,CAAC;IAEO,yDAAuB,GAA/B;;QAAA,iBAyCC;QAxCA,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACzB,OAAO;QACR,CAAC;QAED,IAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAA,MAAM,IAAI,OAAA,CAAC;YAC5C,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,KAAK,EAAE,KAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,WAAW,CAAC;YACpD,QAAQ,EAAE,KAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,cAAc,CAAC;YAC1D,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM;SACtC,CAAC,EAL0C,CAK1C,CAAC,CAAC;gCAEO,IAAI;YACd,IAAM,MAAM,GAAG,OAAK,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1D,IAAM,iBAAiB,GAAG,OAAK,oBAAoB,CAAC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,iBAAiB,CAAC,CAAC;YAC/E,IAAM,oBAAoB,GAAG,OAAK,oBAAoB,CAAC,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,oBAAoB,CAAC,CAAC;YAErF,IAAI,CAAC,iBAAiB,IAAI,CAAC,oBAAoB,EAAE,CAAC;;YAElD,CAAC;YAED,IAAI,OAAK,qBAAqB,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;;YAElD,CAAC;YAED,IAAM,QAAQ,GAAG,OAAK,QAAQ,CAAC,IAAI,CAAC,UAAA,MAAM;gBACzC,OAAA,CAAC,CAAC,iBAAiB,IAAI,KAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,iBAAiB,CAAC;oBAC3F,CAAC,CAAC,oBAAoB,IAAI,KAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,cAAc,CAAC,KAAK,oBAAoB,CAAC;YADpG,CACoG,CACpG,CAAC;YAEF,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACf,OAAK,qBAAqB,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAC5C,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,EAAE,gDAAgD,EAAE;oBAC1E,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,iBAAiB,mBAAA;oBACjB,oBAAoB,sBAAA;oBACpB,OAAO,SAAA;iBACP,CAAC,CAAC;YACJ,CAAC;;;;YA3BF,KAAmB,IAAA,KAAA,SAAA,IAAI,CAAC,UAAU,CAAA,gBAAA;gBAA7B,IAAM,IAAI,WAAA;wBAAJ,IAAI;aA4Bd;;;;;;;;;IACF,CAAC;IAEO,6CAAW,GAAnB;QACC,OAAO,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,IAAI,OAAO,CAAC,GAAG,CAAC,qBAAqB,KAAK,MAAM,CAAC;IAC7F,CAAC;IAEO,gDAAc,GAAtB,UAAuB,KAAa,EAAE,OAA4B;QACjE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,YAC7B,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAC5B,KAAK,OAAA,IACF,OAAO,EACT,CAAC,CAAC;IACL,CAAC;IAEO,+CAAa,GAArB,UAAsB,KAAU;QAC/B,IAAI,CAAC;YACJ,IAAM,MAAI,GAAG,IAAI,OAAO,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,UAAC,GAAG,EAAE,GAAG;gBACrC,IAAI,GAAG,KAAK,WAAW,EAAE,CAAC;oBACzB,OAAO,SAAS,CAAC;gBAClB,CAAC;gBACD,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;oBAC7B,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;gBACvB,CAAC;gBACD,IAAI,GAAG,YAAY,KAAK,EAAE,CAAC;oBAC1B,OAAO;wBACN,IAAI,EAAE,GAAG,CAAC,IAAI;wBACd,OAAO,EAAE,GAAG,CAAC,OAAO;wBACpB,KAAK,EAAE,GAAG,CAAC,KAAK;qBAChB,CAAC;gBACH,CAAC;gBACD,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;oBACpC,IAAI,MAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;wBACnB,OAAO,YAAY,CAAC;oBACrB,CAAC;oBACD,MAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACf,CAAC;gBACD,OAAO,GAAG,CAAC;YACZ,CAAC,CAAC,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,SAAS,CAAC;gBACrB,KAAK,EAAE,qBAAqB;gBAC5B,OAAO,EAAE,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,KAAI,MAAM,CAAC,GAAG,CAAC;aACpC,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAED;;OAEG;IACI,qDAAmB,GAA1B,UAA2B,QAAgB,EAAE,UAA6B;QACzE,IAAI,IAAkB,CAAC;QAEvB,IAAI,CAAC;YACJ,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;gBACpC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,oBAAW,CAAC,CAAC;YAC5C,CAAC;iBACI,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBACtC,IAAI,GAAG,IAAA,iBAAM,EAAC,UAAU,CAAC,CAAC;YAC3B,CAAC;iBACI,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;gBACpC,IAAM,MAAM,GAAG,UAAkD,CAAC;gBAClE,IAAI,GAAG,IAAA,iBAAM,EAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;YACtC,CAAC;iBACI,IAAI,UAAU,YAAY,WAAW,EAAE,CAAC;gBAC5C,IAAI,GAAG,IAAA,iBAAM,EAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;YACxC,CAAC;iBACI,IAAI,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;gBACzC,IAAM,IAAI,GAAG,UAAoC,CAAC;gBAClD,IAAI,GAAG,IAAA,iBAAM,EAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;YAC3E,CAAC;iBACI,CAAC;gBACL,OAAO,CAAC,KAAK,CAAC,2CAA2C,EAAE,OAAO,UAAU,CAAC,CAAC;gBAC9E,OAAO;YACR,CAAC;QACF,CAAC;QACD,OAAO,GAAG,EAAE,CAAC;YACZ,OAAO,CAAC,KAAK,CAAC,iCAAiC,EAAE,GAAG,CAAC,CAAC;YACtD,OAAO;QACR,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAClC,IAAI,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,CAAC,EAAE,KAAK,QAAQ,EAAjB,CAAiB,CAAC,CAAC;YAExD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACb,OAAO,CAAC,KAAK,CAAC,kCAAkC,EAAE,QAAQ,CAAC,CAAC;gBAC5D,OAAO;YACR,CAAC;YAEK,IAAA,QAAM,GAA+B,IAAI,OAAnC,EAAE,SAAS,GAAoB,IAAI,UAAxB,EAAE,KAAK,GAAa,IAAI,MAAjB,EAAE,MAAM,GAAK,IAAI,OAAT,CAAU;YAChD,IAAI,cAAc,GAAG,MAAM,CAAC;YAC5B,IAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAM,CAAC,CAAC;YAC7C,IAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC/B,IAAM,YAAY,GAAG,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,YAAY,CAAC;YAC1C,IAAM,QAAQ,GAAG,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,QAAQ,CAAC;YAClC,IAAM,oBAAoB,GAAG,YAAY,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,WAAW,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACrF,IAAM,WAAW,GAAG,QAAQ,IAAI,YAAY,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACrF,IAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAChE,IAAM,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;YAE9E,IAAI,CAAC,KAAK,IAAI,cAAc,KAAK,IAAI,IAAI,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;gBAC/D,IAAI,CAAC;oBACJ,cAAc,GAAG,IAAA,iBAAM,EAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;gBAC/C,CAAC;gBACD,OAAO,GAAG,EAAE,CAAC;oBACZ,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;gBACtD,CAAC;YACF,CAAC;YACD,MAAM,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,CAAC,MAAM,KAAK,QAAM,EAAnB,CAAmB,CAAC,CAAC;YACzE,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,QAAM,CAAC,CAAC;YAEjC,IAAI,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAM,CAAC,CAAC;YAEjD,IAAI,KAAK,IAAI,CAAC,oBAAoB,KAAK,IAAI,IAAI,oBAAoB,IAAI,IAAI,CAAC,4BAA4B,CAAC,EAAE,CAAC;gBAC3G,IAAI,CAAC,cAAc,CAAC,oBAAoB,EAAE;oBACzC,MAAM,UAAA;oBACN,MAAM,EAAE,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,MAAM;oBAC3B,SAAS,WAAA;oBACT,KAAK,OAAA;oBACL,oBAAoB,sBAAA;oBACpB,WAAW,aAAA;oBACX,OAAO,SAAA;oBACP,WAAW,aAAA;oBACX,WAAW,EAAE,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,WAAW;oBAChC,MAAM,EAAE;wBACP,EAAE,EAAE,MAAM,CAAC,EAAE;wBACb,KAAK,EAAE,MAAM,CAAC,WAAW;wBACzB,QAAQ,EAAE,MAAM,CAAC,cAAc;qBAC/B;oBACD,KAAK,EAAE,IAAI,CAAC,gBAAgB,EAAE;iBAC9B,CAAC,CAAC;YACJ,CAAC;YAED,IAAI,WAAW,EAAE,CAAC;gBACjB,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;oBACzB,IAAI,KAAK,EAAE,CAAC;wBACX,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;oBAC5C,CAAC;yBACI,CAAC;wBACL,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;oBAC7C,CAAC;gBACF,CAAC;gBAED,YAAY,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;gBAClC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,QAAM,CAAC,CAAC;YACnC,CAAC;YAED,oDAAoD;YACpD,IAAI,WAAW,GAAG,IAAI,CAAC,eAAe,CAAC,QAAM,CAAC,CAAC;YAE/C,IAAI,WAAW,EAAE,CAAC;gBACjB,IAAI,GAAG,GAAwB;oBAC9B,SAAS,EAAE,SAAS;oBACpB,QAAQ,EAAE,KAAK;oBACf,IAAI,EAAE,MAAM;iBACZ,CAAC;gBAEF,IAAI,KAAK,EAAE,CAAC;oBACX,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC;oBACpB,GAAG,CAAC,IAAI,GAAG,MAAM,CAAC;gBACnB,CAAC;gBAED,IAAI,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;gBAEnE,IAAI,WAAW,EAAE,CAAC;oBACjB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;wBACpC,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,CAAC,cAAc,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI,SAAS,CAAC,CAAC;oBAC7H,CAAC;yBACI,CAAC;wBACL,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;oBAC/C,CAAC;gBACF,CAAC;gBAED,IAAI,IAAI,CAAC,eAAe,CAAC,QAAM,CAAC,EAAE,CAAC;oBAClC,OAAO,IAAI,CAAC,eAAe,CAAC,QAAM,CAAC,CAAC;gBACrC,CAAC;YACF,CAAC;YAED,sCAAsC;YACtC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;gBAC5B,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,CAAC;QACF,CAAC;IACF,CAAC;IAEM,mDAAiB,GAAxB,UAAyB,EAAuB,EAAE,OAA6B;QAC9E,IAAI,CAAC,EAAE,EAAE,CAAC;YACT,OAAO;QACR,CAAC;QAED,IAAI,EAAE,CAAC,UAAU,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;YAC/B,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;gBACjC,IAAM,MAAI,GAAG,OAAO,CAAC;gBAErB,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,MAAI,CAAC,CAAC;gBAC9B,IAAI,CAAC,aAAa,EAAE,CAAC;gBAErB,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC;gBACvC,EAAE,CAAC,KAAK,EAAE,CAAC;gBAEZ,IAAI,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAI,CAAC,MAAM,CAAC,CAAC;gBAEtD,IAAI,WAAW,EAAE,CAAC;oBACjB,YAAY,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;oBAClC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,MAAI,CAAC,MAAM,CAAC,CAAC;oBAEvC,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;wBACzB,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,yBAAyB,CAAC,CAAC;oBACvD,CAAC;gBACF,CAAC;YACD,CAAC;iBACI,CAAC;gBACL,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC;gBACvC,EAAE,CAAC,KAAK,EAAE,CAAC;YACZ,CAAC;YAED,OAAO;QACR,CAAC;QAED,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YACjC,IAAI,CAAC;gBACJ,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAClB,CAAC;YACD,OAAO,GAAG,EAAE,CAAC;gBACZ,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC;gBACvC,EAAE,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;YACtD,CAAC;YAED,OAAO;QACR,CAAC;QAED,IAAM,IAAI,GAAG,OAAO,CAAC;QACrB,IAAM,aAAa,GAAG,IAAA,eAAI,EAAC,IAAI,CAAC,CAAC;QAEjC,IAAI,CAAC;YACJ,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACxB,CAAC;QACD,OAAO,GAAG,EAAE,CAAC;YACZ,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC9B,IAAI,CAAC,aAAa,EAAE,CAAC;YAErB,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;gBAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,mBAAmB,EAAE,IAAI,CAAC,CAAC;YACpD,CAAC;YAED,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC;YACvC,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,iCAAiC,EAAE,GAAG,CAAC,CAAC;YAEtD,IAAI,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEtD,IAAI,WAAW,EAAE,CAAC;gBACjB,YAAY,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;gBAClC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAEvC,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;oBACzB,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,kCAAkC,IAAG,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,QAAQ,EAAE,CAAA,CAAC,CAAC;gBAClF,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IACF,8BAAC;AAAD,CAzwBA,AAywBC,IAAA;AAzwBY,0DAAuB","file":"worker-dispatcher.manager.js","sourcesContent":["import * as WebSocket from 'ws';\nimport { ServerResponseModel, TaskPayload, TaskResponse } from '../models/server-message.model';\nimport { dateReviver, objectIdHexString, round } from '../util/common';\nimport { MethodManager } from './method.manager';\nimport { WebSocketManager } from './websocket.manager';\nimport { pack, unpack } from 'msgpackr';\n\nexport interface WorkerConnection {\n\tid: string;\n\tws: WebSocket.WebSocket;\n\tactiveTasks: {\n\t\ttaskId: string;\n\t\tweight: number;\n\t\tmethod?: string;\n\t\tstartedAt?: number;\n\t\tmessageId?: number;\n\t\tuserContext?: { id_user?: string; user?: string; id_ws?: string };\n\t}[];\n\tworkerIndex?: string;\n\tworkerInstance?: string;\n}\n\ninterface PendingTask {\n\ttimeout: NodeJS.Timeout;\n\tpromise?: {\n\t\t// eslint-disable-next-line no-unused-vars\n\t\tresolve: (value: any) => void;\n\t\t// eslint-disable-next-line no-unused-vars\n\t\treject: (reason?: any) => void;\n\t};\n\tmethod?: string;\n}\n\nexport class WorkerDispatcherManager {\n\tprivate _websocketManager: WebSocketManager;\n\tprivate _methodManager: MethodManager;\n\tprivate _workers: WorkerConnection[] = [];\n\tprivate _taskQueue: TaskPayload[] = [];\n\tprivate _clientRequests: { [taskId: string]: string } = {};\n\tprivate _pendingTasks: Map<string, PendingTask> = new Map();\n\tprivate _debugNoWorkerTaskIds: Set<string> = new Set();\n\tprivate _taskTimings: Map<string, {\n\t\tqueuedAt: number;\n\t\tdispatchedAt?: number;\n\t\tmethod?: string;\n\t\tmessageId?: number;\n\t\tuserContext?: { id_user?: string; user?: string; id_ws?: string };\n\t}> = new Map();\n\n\tprivate MAX_CONCURRENCY = 10;\n\tprivate WORKER_TASK_LOG_THRESHOLD_MS = 200;\n\n\tconstructor() {}\n\n\tstatic create(websocketManager: WebSocketManager, methodManager: MethodManager) {\n\t\tconst workerDispatcherManager = new WorkerDispatcherManager();\n\t\tworkerDispatcherManager.initialize(websocketManager, methodManager);\n\t\treturn workerDispatcherManager;\n\t}\n\n\tpublic initialize(websocketManager: WebSocketManager, methodManager: MethodManager) {\n\t\tthis._websocketManager = websocketManager;\n\t\tthis._methodManager = methodManager;\n\t}\n\n\tpublic isSafeShutdown() {\n\t\treturn !this._taskQueue.length && !this._pendingTasks.size;\n\t}\n\n\tpublic addWorker(ws: WebSocket.WebSocket) {\n\t\tthis._workers.push({\n\t\t\tid: ws['id_worker'],\n\t\t\tws: ws,\n\t\t\tactiveTasks: [],\n\t\t\tworkerIndex: ws['workerIndex'],\n\t\t\tworkerInstance: ws['workerInstance']\n\t\t});\n\n\t\tsetInterval(() => {\n\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\tconsole.log(JSON.stringify(this._workers.map(a => {\n\t\t\t\t\treturn {\n\t\t\t\t\t\tid: a.id,\n\t\t\t\t\t\tactiveTasks: a.activeTasks\n\t\t\t\t\t}\n\t\t\t\t}), null, 2));\n\t\t\t}\n\t\t}, 5000);\n\n\t\tif (this._taskQueue.length) {\n\t\t\tthis.dispatchQueue();\n\t\t}\n\t}\n\n\tpublic disconnectWorker(workerId: string) {\n\t\tthis._workers = this._workers.filter(w => w.id !== workerId);\n\t}\n\n\tpublic hasWorkers() {\n\t\treturn this._workers.length > 0;\n\t}\n\n\tpublic getQueueSnapshot() {\n\t\tconst activeTasks = this._workers.reduce((total, worker) => total + (worker.activeTasks ? worker.activeTasks.length : 0), 0);\n\t\treturn {\n\t\t\tqueueDepth: this._taskQueue.length,\n\t\t\tpendingTasks: this._pendingTasks.size,\n\t\t\tworkerCount: this._workers.length,\n\t\t\tactiveTasks,\n\t\t\tworkers: this._workers.map(worker => ({\n\t\t\t\tid: worker.id,\n\t\t\t\tindex: worker.workerIndex,\n\t\t\t\tinstance: worker.workerInstance,\n\t\t\t\tactiveTasks: worker.activeTasks ? worker.activeTasks.length : 0\n\t\t\t}))\n\t\t};\n\t}\n\n\tpublic hasWorkersForMethod(methodName: string) {\n\t\tif (!this._workers.length) {\n\t\t\treturn false;\n\t\t}\n\n\t\tconst method = this._methodManager.getMethod(methodName);\n\t\tconst targetWorkerIndex = this.normalizeWorkerIndex(method?.targetWorkerIndex);\n\t\tconst targetWorkerInstance = this.normalizeWorkerIndex(method?.targetWorkerInstance);\n\n\t\tif (!targetWorkerIndex && !targetWorkerInstance) {\n\t\t\treturn true;\n\t\t}\n\n\t\treturn this._workers.some(worker =>\n\t\t\t(!targetWorkerIndex || this.normalizeWorkerIndex(worker.workerIndex) === targetWorkerIndex) &&\n\t\t\t(!targetWorkerInstance || this.normalizeWorkerIndex(worker.workerInstance) === targetWorkerInstance)\n\t\t);\n\t}\n\n\t/**\n\t * Add a new task to our in-memory queue and try to dispatch.\n\t */\n\tpublic sendClientTask(messageId: number, method: string, params: any[], userContext?: { id_user?: string; user?: string; id_ws?: string }) {\n\t\tlet taskId = 'task-' + objectIdHexString();\n\t\tconst queuedAt = Date.now();\n\n\t\tthis._taskQueue.push({\n\t\t\ttype: 'task',\n\t\t\ttaskId,\n\t\t\tmessageId,\n\t\t\tmethod,\n\t\t\tparams,\n\t\t\tuserContext,\n\t\t\tqueuedAt\n\t\t});\n\n\t\tthis._taskTimings.set(taskId, {\n\t\t\tqueuedAt,\n\t\t\tmethod,\n\t\t\tmessageId,\n\t\t\tuserContext\n\t\t});\n\n\t\tif (userContext && userContext.id_ws) {\n\t\t\tthis._clientRequests[taskId] = userContext.id_ws;\n\t\t}\n\n\t\tthis.dispatchQueue();\n\t}\n\n\t/**\n\t * Same as sendInternalTask but returns a Promise so you can `await` it.\n\t */\n\tpublic sendInternalPromise(method: string, params: any[] = []): Promise<any> {\n\t\tif (!Array.isArray(params)) {\n\t\t\tparams = [params];\n\t\t}\n\n\t\t// eslint-disable-next-line no-restricted-syntax\n\t\treturn new Promise((resolve, reject) => {\n\t\t\tlet taskId = 'task-' + objectIdHexString();\n\t\t\tconst queuedAt = Date.now();\n\n\t\t\tthis._taskQueue.push({\n\t\t\t\ttype: 'task',\n\t\t\t\ttaskId,\n\t\t\t\tmethod,\n\t\t\t\tparams,\n\t\t\t\tmessageId: 0,\n\t\t\t\tuserContext: {\n\t\t\t\t\tuser: 'Internal System'\n\t\t\t\t},\n\t\t\t\tqueuedAt\n\t\t\t});\n\n\t\t\tthis._taskTimings.set(taskId, {\n\t\t\t\tqueuedAt,\n\t\t\t\tmethod,\n\t\t\t\tmessageId: 0,\n\t\t\t\tuserContext: { user: 'Internal System' }\n\t\t\t});\n\n\t\t\tthis._pendingTasks.set(taskId, {\n\t\t\t\ttimeout: null,\n\t\t\t\tpromise: { resolve, reject }\n\t\t\t});\n\n\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\tconsole.log(new Date(), 'Send Internal Promise', this._taskQueue);\n\t\t\t}\n\n\t\t\tthis.dispatchQueue();\n\t\t});\n\t}\n\n\t/**\n\t * Send a task internally without returning a promise.\n\t */\n\tpublic sendInternalTask(method: string, params = []) {\n\t\tif (!Array.isArray(params)) {\n\t\t\tparams = [params];\n\t\t}\n\n\t\tlet taskId = 'task-' + objectIdHexString();\n\t\tconst queuedAt = Date.now();\n\n\t\tthis._taskQueue.push({\n\t\t\ttype: 'task',\n\t\t\ttaskId,\n\t\t\tmethod,\n\t\t\tparams,\n\t\t\tmessageId: 0,\n\t\t\tuserContext: {\n\t\t\t\tuser: 'Internal System'\n\t\t\t},\n\t\t\tqueuedAt\n\t\t});\n\n\t\tthis._taskTimings.set(taskId, {\n\t\t\tqueuedAt,\n\t\t\tmethod,\n\t\t\tmessageId: 0,\n\t\t\tuserContext: { user: 'Internal System' }\n\t\t});\n\n\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\tconsole.log(new Date(), 'Send Internal Task', this._taskQueue);\n\t\t}\n\n\t\tthis.dispatchQueue();\n\t}\n\n\t/**\n\t * The main loop that assigns tasks from _taskQueue to any worker that has capacity.\n\t */\n\tprivate dispatchQueue() {\n\t\tif (!this._taskQueue.length) {\n\t\t\treturn;\n\t\t}\n\n\t\twhile (this._taskQueue.length > 0) {\n\t\t\tlet assigned = false;\n\n\t\t\tfor (let i = 0; i < this._taskQueue.length; i++) {\n\t\t\t\tlet task = this._taskQueue[i];\n\t\t\t\tlet targetWorker = this.findWorkerForTask(task);\n\n\t\t\t\tif (targetWorker) {\n\t\t\t\t\tthis._taskQueue.splice(i, 1);\n\t\t\t\t\tthis.assignTaskToWorker(targetWorker, task);\n\t\t\t\t\tassigned = true;\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif (!assigned) {\n\t\t\t\tthis.logNoWorkerAvailability();\n\t\t\t\tsetTimeout(() => {\n\t\t\t\t\tthis.dispatchQueue();\n\t\t\t\t}, 25);\n\n\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\tconsole.log(new Date(), 'No Worker Available', JSON.stringify(this._workers, null, 2));\n\t\t\t\t}\n\n\t\t\t\treturn; // nothing can run right now due to per-worker limits\n\t\t\t}\n\t\t}\n\t}\n\n\t/**\n\t * Returns the worker with the fewest activeTasks that is under maxConcurrency and per-method limit.\n\t */\n\tprivate findWorkerForTask(task: TaskPayload): WorkerConnection | null {\n\t\tlet method = this._methodManager.getMethod(task.method);\n\t\tlet methodLimit = method && method.maxConcurrency && method.maxConcurrency > 0 ? method.maxConcurrency : null;\n\t\tlet methodLimitPerInstance = method && method.maxConcurrencyPerInstance && method.maxConcurrencyPerInstance > 0\n\t\t\t? method.maxConcurrencyPerInstance\n\t\t\t: null;\n\t\tlet targetWorkerIndex = this.normalizeWorkerIndex(method?.targetWorkerIndex);\n\t\tlet targetWorkerInstance = this.normalizeWorkerIndex(method?.targetWorkerInstance);\n\t\tconst preferNonZero = !targetWorkerIndex;\n\n\t\tlet candidates = this._workers.filter(x => x.activeTasks.length < this.MAX_CONCURRENCY);\n\t\tif (targetWorkerIndex) {\n\t\t\tcandidates = candidates.filter(worker => this.normalizeWorkerIndex(worker.workerIndex) === targetWorkerIndex);\n\t\t}\n\t\tif (targetWorkerInstance) {\n\t\t\tcandidates = candidates.filter(worker => this.normalizeWorkerIndex(worker.workerInstance) === targetWorkerInstance);\n\t\t}\n\t\tif (!candidates.length) {\n\t\t\treturn null;\n\t\t}\n\n\t\tlet eligible = candidates.filter((worker) => {\n\t\t\tif (!methodLimit) {\n\t\t\t\tif (!methodLimitPerInstance) {\n\t\t\t\t\treturn true;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif (methodLimitPerInstance) {\n\t\t\t\tconst currentInstance = worker.activeTasks.filter(a => a.method === task.method).length;\n\t\t\t\tif (currentInstance >= methodLimitPerInstance) {\n\t\t\t\t\treturn false;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif (!methodLimit) {\n\t\t\t\treturn true;\n\t\t\t}\n\n\t\t\tlet idx = worker.workerIndex || 'unknown';\n\t\t\tlet current = 0;\n\n\t\t\tthis._workers.forEach((w) => {\n\t\t\t\tif ((w.workerIndex || 'unknown') === idx) {\n\t\t\t\t\tcurrent += w.activeTasks.filter(a => a.method === task.method).length;\n\t\t\t\t}\n\t\t\t});\n\n\t\t\treturn current < methodLimit;\n\t\t});\n\n\t\tif (!eligible.length) {\n\t\t\treturn null;\n\t\t}\n\n\t\teligible.sort((x, y) => {\n\t\t\tconst totalX = this.getWorkerLoad(x);\n\t\t\tconst totalY = this.getWorkerLoad(y);\n\t\t\tif (totalX !== totalY) {\n\t\t\t\treturn totalX - totalY;\n\t\t\t}\n\n\t\t\tif (preferNonZero) {\n\t\t\t\tconst xZero = this.isWorkerZero(x) ? 1 : 0;\n\t\t\t\tconst yZero = this.isWorkerZero(y) ? 1 : 0;\n\t\t\t\tif (xZero !== yZero) {\n\t\t\t\t\treturn xZero - yZero;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tconst idxX = this.normalizeWorkerIndex(x.workerIndex) || '';\n\t\t\tconst idxY = this.normalizeWorkerIndex(y.workerIndex) || '';\n\t\t\tconst idxCompare = idxX.localeCompare(idxY, undefined, { numeric: true, sensitivity: 'base' });\n\t\t\tif (idxCompare !== 0) {\n\t\t\t\treturn idxCompare;\n\t\t\t}\n\n\t\t\tconst instX = this.normalizeWorkerIndex(x.workerInstance) || '';\n\t\t\tconst instY = this.normalizeWorkerIndex(y.workerInstance) || '';\n\t\t\treturn instX.localeCompare(instY, undefined, { numeric: true, sensitivity: 'base' });\n\t\t});\n\n\t\treturn eligible[0];\n\t}\n\n\tprivate normalizeWorkerIndex(value?: string | number | null): string | null {\n\t\tif (value === null || value === undefined) {\n\t\t\treturn null;\n\t\t}\n\n\t\tconst normalized = String(value).trim();\n\t\treturn normalized.length ? normalized : null;\n\t}\n\n\tprivate getWorkerLoad(worker: WorkerConnection): number {\n\t\treturn worker.activeTasks.reduce((sum, task) => sum + task.weight, 0);\n\t}\n\n\tprivate isWorkerZero(worker: WorkerConnection): boolean {\n\t\treturn this.normalizeWorkerIndex(worker.workerIndex) === '0';\n\t}\n\n\tprivate assignTaskToWorker(worker: WorkerConnection, task: TaskPayload) {\n\t\tlet method = this._methodManager.getMethod(task.method);\n\n\t\tif (!method) {\n\t\t\tconsole.error('Failed to send task to worker - Could not find method:', task.method);\n\t\t\tthis._taskTimings.delete(task.taskId);\n\t\t\treturn;\n\t\t}\n\n\t\tconst dispatchedAt = Date.now();\n\t\tconst timing = this._taskTimings.get(task.taskId) || { queuedAt: dispatchedAt };\n\t\ttiming.dispatchedAt = dispatchedAt;\n\t\ttiming.method = task.method;\n\t\ttiming.messageId = task.messageId;\n\t\ttiming.userContext = task.userContext;\n\t\tthis._taskTimings.set(task.taskId, timing);\n\n\t\tworker.activeTasks.push({\n\t\t\ttaskId: task.taskId,\n\t\t\tweight: method && method.workerTaskWeight ? method.workerTaskWeight : 1,\n\t\t\tmethod: task.method,\n\t\t\tstartedAt: dispatchedAt,\n\t\t\tmessageId: task.messageId,\n\t\t\tuserContext: task.userContext\n\t\t});\n\t\tthis._debugNoWorkerTaskIds.delete(task.taskId);\n\n\t\tlet payload: TaskPayload = {\n\t\t\ttype: 'task',\n\t\t\ttaskId: task.taskId,\n\t\t\tmessageId: task.messageId,\n\t\t\tmethod: task.method,\n\t\t\tparams: task.params,\n\t\t\tuserContext: task.userContext || {},\n\t\t\tqueuedAt: timing.queuedAt,\n\t\t\tdispatchedAt\n\t\t};\n\n\t\tlet timeoutHandle = setTimeout(() => {\n\t\t\tlet pending = this._pendingTasks.get(task.taskId);\n\t\t\tconst timeoutAt = Date.now();\n\t\t\tconst queuedAt = timing.queuedAt;\n\t\t\tconst queueWaitMs = queuedAt ? round(dispatchedAt - queuedAt) : null;\n\t\t\tconst dispatchToTimeoutMs = round(timeoutAt - dispatchedAt);\n\n\t\t\tif (pending) {\n\t\t\t\tif (pending.promise) {\n\t\t\t\t\tpending.promise.reject('Task timed out after 2m in WorkerDispatcherManager for method: ' + task.method);\n\t\t\t\t}\n\n\t\t\t\tthis._pendingTasks.delete(task.taskId);\n\t\t\t}\n\n\t\t\tthis.logWorkerEvent('workerTaskTimeout', {\n\t\t\t\ttaskId: task.taskId,\n\t\t\t\tmethod: task.method,\n\t\t\t\tmessageId: task.messageId,\n\t\t\t\tqueueWaitMs,\n\t\t\t\tdispatchToTimeoutMs,\n\t\t\t\tuserContext: task.userContext,\n\t\t\t\tworker: {\n\t\t\t\t\tid: worker.id,\n\t\t\t\t\tindex: worker.workerIndex,\n\t\t\t\t\tinstance: worker.workerInstance\n\t\t\t\t},\n\t\t\t\tworkers: this._workers.map(a => a.activeTasks)\n\t\t\t});\n\n\t\t\tworker.activeTasks = worker.activeTasks.filter(a => a.taskId !== task.taskId);\n\t\t\tthis._taskTimings.delete(task.taskId);\n\n\t\t\tif (task.userContext && task.userContext.id_ws) {\n\t\t\t\tlet timeoutRes: ServerResponseModel = {\n\t\t\t\t\tmessageId: task.messageId,\n\t\t\t\t\thasError: true,\n\t\t\t\t\tdata: 'Task timed out after 2m in WorkerDispatcherManager for method: ' + task.method\n\t\t\t\t};\n\n\t\t\t\tlet clientWS = this._websocketManager.getWebSocket(task.userContext.id_ws);\n\n\t\t\t\tif (clientWS) {\n\t\t\t\t\tthis._websocketManager.send(clientWS, timeoutRes);\n\t\t\t\t}\n\t\t\t}\n\t\t}, method.timeoutOverride || (1000 * 60 * 2));\n\n\t\t// If we already stored a promise for this task (from sendInternalPromise), add the timeout now\n\t\tlet existing = this._pendingTasks.get(task.taskId);\n\t\tif (!existing) {\n\t\t\texisting = { timeout: null, method: task.method };\n\t\t}\n\t\telse {\n\t\t\texisting.method = task.method;\n\t\t}\n\n\t\texisting.timeout = timeoutHandle;\n\t\tthis._pendingTasks.set(task.taskId, existing);\n\n\t\ttry {\n\t\t\tthis.sendWorkerPayload(worker.ws, payload);\n\t\t}\n\t\tcatch (err) {\n\t\t\tconsole.error('Failed to send task to worker:', err);\n\n\t\t\tclearTimeout(timeoutHandle);\n\t\t\tworker.activeTasks = worker.activeTasks.filter(a => a.taskId !== task.taskId);\n\n\t\t\t// Put task back in front\n\t\t\tthis._taskQueue.unshift(task);\n\t\t\tthis.dispatchQueue();\n\t\t}\n\t}\n\n\tprivate logNoWorkerAvailability() {\n\t\tif (!this.shouldDebug()) {\n\t\t\treturn;\n\t\t}\n\n\t\tconst workers = this._workers.map(worker => ({\n\t\t\tid: worker.id,\n\t\t\tindex: this.normalizeWorkerIndex(worker.workerIndex),\n\t\t\tinstance: this.normalizeWorkerIndex(worker.workerInstance),\n\t\t\tactiveTasks: worker.activeTasks.length\n\t\t}));\n\n\t\tfor (const task of this._taskQueue) {\n\t\t\tconst method = this._methodManager.getMethod(task.method);\n\t\t\tconst targetWorkerIndex = this.normalizeWorkerIndex(method?.targetWorkerIndex);\n\t\t\tconst targetWorkerInstance = this.normalizeWorkerIndex(method?.targetWorkerInstance);\n\n\t\t\tif (!targetWorkerIndex && !targetWorkerInstance) {\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tif (this._debugNoWorkerTaskIds.has(task.taskId)) {\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tconst hasMatch = this._workers.some(worker =>\n\t\t\t\t(!targetWorkerIndex || this.normalizeWorkerIndex(worker.workerIndex) === targetWorkerIndex) &&\n\t\t\t\t(!targetWorkerInstance || this.normalizeWorkerIndex(worker.workerInstance) === targetWorkerInstance)\n\t\t\t);\n\n\t\t\tif (!hasMatch) {\n\t\t\t\tthis._debugNoWorkerTaskIds.add(task.taskId);\n\t\t\t\tconsole.warn(new Date(), '[WorkerDispatcher] No matching worker for task', {\n\t\t\t\t\ttaskId: task.taskId,\n\t\t\t\t\tmethod: task.method,\n\t\t\t\t\ttargetWorkerIndex,\n\t\t\t\t\ttargetWorkerInstance,\n\t\t\t\t\tworkers\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t}\n\n\tprivate shouldDebug(): boolean {\n\t\treturn this._methodManager.getEnableDebug() || process.env.WORKER_DISPATCH_DEBUG === 'true';\n\t}\n\n\tprivate logWorkerEvent(event: string, details: Record<string, any>) {\n\t\tconsole.log(this.safeStringify({\n\t\t\tts: new Date().toISOString(),\n\t\t\tevent,\n\t\t\t...details\n\t\t}));\n\t}\n\n\tprivate safeStringify(value: any): string {\n\t\ttry {\n\t\t\tconst seen = new WeakSet();\n\t\t\treturn JSON.stringify(value, (key, val) => {\n\t\t\t\tif (key === '__proto__') {\n\t\t\t\t\treturn undefined;\n\t\t\t\t}\n\t\t\t\tif (typeof val === 'bigint') {\n\t\t\t\t\treturn val.toString();\n\t\t\t\t}\n\t\t\t\tif (val instanceof Error) {\n\t\t\t\t\treturn {\n\t\t\t\t\t\tname: val.name,\n\t\t\t\t\t\tmessage: val.message,\n\t\t\t\t\t\tstack: val.stack\n\t\t\t\t\t};\n\t\t\t\t}\n\t\t\t\tif (val && typeof val === 'object') {\n\t\t\t\t\tif (seen.has(val)) {\n\t\t\t\t\t\treturn '[Circular]';\n\t\t\t\t\t}\n\t\t\t\t\tseen.add(val);\n\t\t\t\t}\n\t\t\t\treturn val;\n\t\t\t});\n\t\t}\n\t\tcatch (err) {\n\t\t\treturn JSON.stringify({\n\t\t\t\terror: 'safeStringifyFailed',\n\t\t\t\tmessage: err?.message || String(err)\n\t\t\t});\n\t\t}\n\t}\n\n\t/**\n\t * Handle messages coming back from a worker (like 'taskComplete').\n\t */\n\tpublic handleWorkerMessage(workerId: string, rawMessage: WebSocket.RawData) {\n\t\tlet data: TaskResponse;\n\n\t\ttry {\n\t\t\tif (typeof rawMessage === 'string') {\n\t\t\t\tdata = JSON.parse(rawMessage, dateReviver);\n\t\t\t}\n\t\t\telse if (Buffer.isBuffer(rawMessage)) {\n\t\t\t\tdata = unpack(rawMessage);\n\t\t\t}\n\t\t\telse if (Array.isArray(rawMessage)) {\n\t\t\t\tconst chunks = rawMessage as unknown as ReadonlyArray<Uint8Array>;\n\t\t\t\tdata = unpack(Buffer.concat(chunks));\n\t\t\t}\n\t\t\telse if (rawMessage instanceof ArrayBuffer) {\n\t\t\t\tdata = unpack(Buffer.from(rawMessage));\n\t\t\t}\n\t\t\telse if (ArrayBuffer.isView(rawMessage)) {\n\t\t\t\tconst view = rawMessage as NodeJS.ArrayBufferView;\n\t\t\t\tdata = unpack(Buffer.from(view.buffer, view.byteOffset, view.byteLength));\n\t\t\t}\n\t\t\telse {\n\t\t\t\tconsole.error('Unsupported worker message type received:', typeof rawMessage);\n\t\t\t\treturn;\n\t\t\t}\n\t\t}\n\t\tcatch (err) {\n\t\t\tconsole.error('Failed to parse worker message:', err);\n\t\t\treturn;\n\t\t}\n\n\t\tif (data.type === 'taskComplete') {\n\t\t\tlet worker = this._workers.find(x => x.id === workerId);\n\n\t\t\tif (!worker) {\n\t\t\t\tconsole.error('Unknown worker for taskComplete:', workerId);\n\t\t\t\treturn;\n\t\t\t}\n\n\t\t\tlet { taskId, messageId, error, result } = data;\n\t\t\tlet unpackedResult = result;\n\t\t\tconst timing = this._taskTimings.get(taskId);\n\t\t\tconst completedAt = Date.now();\n\t\t\tconst dispatchedAt = timing?.dispatchedAt;\n\t\t\tconst queuedAt = timing?.queuedAt;\n\t\t\tconst dispatchToCompleteMs = dispatchedAt ? round(completedAt - dispatchedAt) : null;\n\t\t\tconst queueWaitMs = queuedAt && dispatchedAt ? round(dispatchedAt - queuedAt) : null;\n\t\t\tconst totalMs = queuedAt ? round(completedAt - queuedAt) : null;\n\t\t\tconst packedBytes = data['packedResult'] ? data['packedResult'].length : null;\n\n\t\t\tif (!error && unpackedResult === null && data['packedResult']) {\n\t\t\t\ttry {\n\t\t\t\t\tunpackedResult = unpack(data['packedResult']);\n\t\t\t\t}\n\t\t\t\tcatch (err) {\n\t\t\t\t\tconsole.error('Failed to unpack worker result', err);\n\t\t\t\t}\n\t\t\t}\n\t\t\tworker.activeTasks = worker.activeTasks.filter(a => a.taskId !== taskId);\n\t\t\tthis._taskTimings.delete(taskId);\n\n\t\t\tlet pendingTask = this._pendingTasks.get(taskId);\n\n\t\t\tif (error || (dispatchToCompleteMs !== null && dispatchToCompleteMs >= this.WORKER_TASK_LOG_THRESHOLD_MS)) {\n\t\t\t\tthis.logWorkerEvent('workerTaskComplete', {\n\t\t\t\t\ttaskId,\n\t\t\t\t\tmethod: pendingTask?.method,\n\t\t\t\t\tmessageId,\n\t\t\t\t\terror,\n\t\t\t\t\tdispatchToCompleteMs,\n\t\t\t\t\tqueueWaitMs,\n\t\t\t\t\ttotalMs,\n\t\t\t\t\tpackedBytes,\n\t\t\t\t\tuserContext: timing?.userContext,\n\t\t\t\t\tworker: {\n\t\t\t\t\t\tid: worker.id,\n\t\t\t\t\t\tindex: worker.workerIndex,\n\t\t\t\t\t\tinstance: worker.workerInstance\n\t\t\t\t\t},\n\t\t\t\t\tqueue: this.getQueueSnapshot()\n\t\t\t\t});\n\t\t\t}\n\n\t\t\tif (pendingTask) {\n\t\t\t\tif (pendingTask.promise) {\n\t\t\t\t\tif (error) {\n\t\t\t\t\t\tpendingTask.promise.reject(unpackedResult);\n\t\t\t\t\t}\n\t\t\t\t\telse {\n\t\t\t\t\t\tpendingTask.promise.resolve(unpackedResult);\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tclearTimeout(pendingTask.timeout);\n\t\t\t\tthis._pendingTasks.delete(taskId);\n\t\t\t}\n\n\t\t\t// Look up original request if it came from a client\n\t\t\tlet clientReqId = this._clientRequests[taskId];\n\n\t\t\tif (clientReqId) {\n\t\t\t\tlet res: ServerResponseModel = {\n\t\t\t\t\tmessageId: messageId,\n\t\t\t\t\thasError: false,\n\t\t\t\t\tdata: result\n\t\t\t\t};\n\n\t\t\t\tif (error) {\n\t\t\t\t\tres.hasError = true;\n\t\t\t\t\tres.data = result;\n\t\t\t\t}\n\n\t\t\t\tlet clientReqWS = this._websocketManager.getWebSocket(clientReqId);\n\n\t\t\t\tif (clientReqWS) {\n\t\t\t\t\tif (!error && data['packedResult']) {\n\t\t\t\t\t\tthis._websocketManager.sendPackedBuffer(clientReqWS, messageId, false, data['packedResult'], data['encoding'] || 'msgpack');\n\t\t\t\t\t}\n\t\t\t\t\telse {\n\t\t\t\t\t\tthis._websocketManager.send(clientReqWS, res);\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif (this._clientRequests[taskId]) {\n\t\t\t\t\tdelete this._clientRequests[taskId];\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Try to dispatch more from the queue\n\t\t\tif (this._taskQueue.length) {\n\t\t\t\tthis.dispatchQueue();\n\t\t\t}\n\t\t}\n\t}\n\n\tpublic sendWorkerPayload(ws: WebSocket.WebSocket, payload: TaskPayload | string) {\n\t\tif (!ws) {\n\t\t\treturn;\n\t\t}\n\n\t\tif (ws.readyState !== ws.OPEN) {\n\t\t\tif (typeof payload !== 'string') {\n\t\t\t\tconst task = payload;\n\n\t\t\t\tthis._taskQueue.unshift(task);\n\t\t\t\tthis.dispatchQueue();\n\n\t\t\t\tthis.disconnectWorker(ws['id_worker']);\n\t\t\t\tws.close();\n\n\t\t\tlet pendingTask = this._pendingTasks.get(task.taskId);\n\n\t\t\tif (pendingTask) {\n\t\t\t\tclearTimeout(pendingTask.timeout);\n\t\t\t\tthis._pendingTasks.delete(task.taskId);\n\n\t\t\t\tif (pendingTask.promise) {\n\t\t\t\t\tpendingTask.promise.reject('Worker socket not open.');\n\t\t\t\t}\n\t\t\t}\n\t\t\t}\n\t\t\telse {\n\t\t\t\tthis.disconnectWorker(ws['id_worker']);\n\t\t\t\tws.close();\n\t\t\t}\n\n\t\t\treturn;\n\t\t}\n\n\t\tif (typeof payload === 'string') {\n\t\t\ttry {\n\t\t\t\tws.send(payload);\n\t\t\t}\n\t\t\tcatch (err) {\n\t\t\t\tthis.disconnectWorker(ws['id_worker']);\n\t\t\t\tws.close();\n\t\t\t\tconsole.error('Failed to send worker payload:', err);\n\t\t\t}\n\n\t\t\treturn;\n\t\t}\n\n\t\tconst task = payload;\n\t\tconst payloadBuffer = pack(task);\n\n\t\ttry {\n\t\t\tws.send(payloadBuffer);\n\t\t}\n\t\tcatch (err) {\n\t\t\tthis._taskQueue.unshift(task);\n\t\t\tthis.dispatchQueue();\n\n\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\tconsole.log(new Date(), 'Sending to Server', task);\n\t\t\t}\n\n\t\t\tthis.disconnectWorker(ws['id_worker']);\n\t\t\tws.close();\n\t\t\tconsole.error('Failed to send worker response:', err);\n\n\t\t\tlet pendingTask = this._pendingTasks.get(task.taskId);\n\n\t\t\tif (pendingTask) {\n\t\t\t\tclearTimeout(pendingTask.timeout);\n\t\t\t\tthis._pendingTasks.delete(task.taskId);\n\n\t\t\t\tif (pendingTask.promise) {\n\t\t\t\t\tpendingTask.promise.reject('Failed to send worker response: ' + err?.toString());\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n"]}

package/managers/worker-server.manager.d.ts

@@ -4,10 +4,13 @@ export declare class WorkerServerManager {
     private _methodManager;
     private _serverConfig;
     private _runningTasks;
+    private _taskLogThresholdMs;
     constructor();
     static create(methodManager: MethodManager, serverConfig: any): WorkerServerManager;
     initialize(methodManager: MethodManager, serverConfig: any): void;
     private startWorkerInstance;
     private handleIncomingTask;
+    private logWorkerEvent;
+    private safeStringify;
     sendWorkerResponse(ws: any, payload: TaskResponse | string): void;
 }

package/managers/worker-server.manager.js

@@ -1,4 +1,15 @@
 "use strict";
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -69,6 +80,7 @@ var method_manager_1 = require("./method.manager");
 var WorkerServerManager = /** @class */ (function () {
     function WorkerServerManager() {
         this._runningTasks = [];
+        this._taskLogThresholdMs = 200;
     }
     WorkerServerManager.create = function (methodManager, serverConfig) {
         var workerServerManager = new WorkerServerManager();
@@ -234,7 +246,7 @@ var WorkerServerManager = /** @class */ (function () {
     };
     WorkerServerManager.prototype.handleIncomingTask = function (ws, data) {
         return __awaiter(this, void 0, void 0, function () {
-            var taskId, messageId, method, params, userContext, timedOut, timeoutHandle, managerThis, result, packedResult, payload, err_1;
+            var taskId, messageId, method, params, userContext, receivedAt, queuedAt, dispatchedAt, queueWaitMs, dispatchDelayMs, timedOut, timeoutHandle, managerThis, result, packedResult, finishedAt, runMs, totalMs, packedBytes, payload, err_1, errorAt_1, runMs_1, totalMs_1, errorAt, runMs, totalMs;
             var _a;
             var _this = this;
             var _b;
@@ -242,6 +254,11 @@ var WorkerServerManager = /** @class */ (function () {
                 switch (_c.label) {
                     case 0:
                         taskId = data.taskId, messageId = data.messageId, method = data.method, params = data.params, userContext = data.userContext;
+                        receivedAt = Date.now();
+                        queuedAt = data.queuedAt;
+                        dispatchedAt = data.dispatchedAt;
+                        queueWaitMs = queuedAt && dispatchedAt ? (0, common_1.round)(dispatchedAt - queuedAt) : (queuedAt ? (0, common_1.round)(receivedAt - queuedAt) : null);
+                        dispatchDelayMs = dispatchedAt ? (0, common_1.round)(receivedAt - dispatchedAt) : null;
                         this._runningTasks.push(taskId);
                         if (!(!taskId || !method || !this._methodManager.getMethod(method))) return [3 /*break*/, 3];
                         if (!!this._methodManager.getMethod(method)) return [3 /*break*/, 2];
@@ -252,6 +269,15 @@ var WorkerServerManager = /** @class */ (function () {
                         _c.label = 2;
                     case 2:
                         // console.log('Invalid task message received', data);
+                        this.logWorkerEvent('workerTaskInvalid', {
+                            taskId: taskId,
+                            method: method,
+                            messageId: messageId,
+                            queueWaitMs: queueWaitMs,
+                            dispatchDelayMs: dispatchDelayMs,
+                            userContext: userContext,
+                            params: params
+                        });
                         this.sendWorkerResponse(ws, {
                             type: 'taskComplete',
                             taskId: taskId,
@@ -263,11 +289,26 @@ var WorkerServerManager = /** @class */ (function () {
                     case 3:
                         timedOut = false;
                         timeoutHandle = setTimeout(function () { return __awaiter(_this, void 0, void 0, function () {
+                            var timeoutAt, runMs, totalMs;
                             return __generator(this, function (_a) {
                                 switch (_a.label) {
                                     case 0:
                                         timedOut = true;
+                                        timeoutAt = Date.now();
+                                        runMs = (0, common_1.round)(timeoutAt - receivedAt);
+                                        totalMs = queuedAt ? (0, common_1.round)(timeoutAt - queuedAt) : null;
                                         console.error(new Date(), 'Worker timed out on task:', taskId, 'Method:', method);
+                                        this.logWorkerEvent('workerTaskTimeout', {
+                                            taskId: taskId,
+                                            method: method,
+                                            messageId: messageId,
+                                            runMs: runMs,
+                                            queueWaitMs: queueWaitMs,
+                                            dispatchDelayMs: dispatchDelayMs,
+                                            totalMs: totalMs,
+                                            userContext: userContext,
+                                            params: params
+                                        });
                                         this.sendWorkerResponse(ws, {
                                             type: 'taskComplete',
                                             taskId: taskId,
@@ -303,6 +344,24 @@ var WorkerServerManager = /** @class */ (function () {
                         catch (packErr) {
                             console.error(new Date(), 'Worker pack error', packErr);
                         }
+                        finishedAt = Date.now();
+                        runMs = (0, common_1.round)(finishedAt - receivedAt);
+                        totalMs = queuedAt ? (0, common_1.round)(finishedAt - queuedAt) : null;
+                        packedBytes = packedResult ? packedResult.length : null;
+                        if (runMs >= this._taskLogThresholdMs) {
+                            this.logWorkerEvent('workerTaskComplete', {
+                                taskId: taskId,
+                                method: method,
+                                messageId: messageId,
+                                runMs: runMs,
+                                queueWaitMs: queueWaitMs,
+                                dispatchDelayMs: dispatchDelayMs,
+                                totalMs: totalMs,
+                                packedBytes: packedBytes,
+                                userContext: userContext,
+                                params: params
+                            });
+                        }
                         if (this._methodManager.getEnableDebug()) {
                             console.log(new Date(), 'Finished method', method);
                         }
@@ -328,8 +387,23 @@ var WorkerServerManager = /** @class */ (function () {
                         err_1 = _c.sent();
                         if (!timedOut) {
                             clearTimeout(timeoutHandle);
+                            errorAt_1 = Date.now();
+                            runMs_1 = (0, common_1.round)(errorAt_1 - receivedAt);
+                            totalMs_1 = queuedAt ? (0, common_1.round)(errorAt_1 - queuedAt) : null;
                             console.error('Worker failed task:', taskId, 'Method:', method, err_1);
                             err_1.message = 'Worker failed task:' + taskId + ' Method:' + method + ' - ' + err_1.message;
+                            this.logWorkerEvent('workerTaskError', {
+                                taskId: taskId,
+                                method: method,
+                                messageId: messageId,
+                                runMs: runMs_1,
+                                queueWaitMs: queueWaitMs,
+                                dispatchDelayMs: dispatchDelayMs,
+                                totalMs: totalMs_1,
+                                userContext: userContext,
+                                params: params,
+                                error: err_1
+                            });
                             this.sendWorkerResponse(ws, {
                                 type: 'taskComplete',
                                 taskId: taskId,
@@ -339,6 +413,21 @@ var WorkerServerManager = /** @class */ (function () {
                             });
                             throw err_1;
                         }
+                        errorAt = Date.now();
+                        runMs = (0, common_1.round)(errorAt - receivedAt);
+                        totalMs = queuedAt ? (0, common_1.round)(errorAt - queuedAt) : null;
+                        this.logWorkerEvent('workerTaskError', {
+                            taskId: taskId,
+                            method: method,
+                            messageId: messageId,
+                            runMs: runMs,
+                            queueWaitMs: queueWaitMs,
+                            dispatchDelayMs: dispatchDelayMs,
+                            totalMs: totalMs,
+                            userContext: userContext,
+                            params: params,
+                            error: err_1
+                        });
                         console.error(new Date(), "Error in Method: ".concat(method, " - ").concat(JSON.stringify(data, null, 2)));
                         return [4 /*yield*/, this._methodManager.callMethod('insertErrorLog', "Error in Method: ".concat(method, " - ").concat(JSON.stringify(data, null, 2)))];
                     case 7:
@@ -350,6 +439,42 @@ var WorkerServerManager = /** @class */ (function () {
             });
         });
     };
+    WorkerServerManager.prototype.logWorkerEvent = function (event, details) {
+        console.log(this.safeStringify(__assign({ ts: new Date().toISOString(), event: event }, details)));
+    };
+    WorkerServerManager.prototype.safeStringify = function (value) {
+        try {
+            var seen_1 = new WeakSet();
+            return JSON.stringify(value, function (key, val) {
+                if (key === '__proto__') {
+                    return undefined;
+                }
+                if (typeof val === 'bigint') {
+                    return val.toString();
+                }
+                if (val instanceof Error) {
+                    return {
+                        name: val.name,
+                        message: val.message,
+                        stack: val.stack
+                    };
+                }
+                if (val && typeof val === 'object') {
+                    if (seen_1.has(val)) {
+                        return '[Circular]';
+                    }
+                    seen_1.add(val);
+                }
+                return val;
+            });
+        }
+        catch (err) {
+            return JSON.stringify({
+                error: 'safeStringifyFailed',
+                message: (err === null || err === void 0 ? void 0 : err.message) || String(err)
+            });
+        }
+    };
     WorkerServerManager.prototype.sendWorkerResponse = function (ws, payload) {
         if (this._methodManager.getEnableDebug()) {
             console.log(new Date(), 'Sending', typeof payload === 'string' ? payload : '[binary]');

package/managers/worker-server.manager.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/managers/worker-server.manager.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,qCAAwC;AACxC,8BAAgC;AAEhC,yCAA6C;AAC7C,mDAAiD;AAEjD;IAKI;QAFQ,kBAAa,GAAG,EAAE,CAAC;IAEZ,CAAC;IAET,0BAAM,GAAb,UAAc,aAA4B,EAAE,YAAY;QACpD,IAAM,mBAAmB,GAAG,IAAI,mBAAmB,EAAE,CAAC;QACtD,mBAAmB,CAAC,UAAU,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;QAC5D,OAAO,mBAAmB,CAAC;IAC/B,CAAC;IAEM,wCAAU,GAAjB,UAAkB,aAA4B,EAAE,YAAY;QACxD,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;QAClC,IAAI,CAAC,mBAAmB,EAAE,CAAC;IAC/B,CAAC;IAEO,iDAAmB,GAA3B;QAAA,iBAqKC;QApKG,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,qEAAqE,CAAC,CAAC;QAE/F,IAAM,WAAW,GAAG,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAC;QAC/E,IAAM,cAAc,GAAG,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC,CAAC;QACvF,IAAI,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,GAAG,yBAAyB,GAAG,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,GAAG,eAAe,GAAG,WAAW,GAAG,kBAAkB,GAAG,cAAc,CAAC;QACpL,IAAM,EAAE,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC;QAEhC,IAAI,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC;QAC1B,IAAI,QAAQ,GAAG,IAAI,CAAC;QACpB,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAM,mBAAmB,GAAG,KAAK,CAAC;QAClC,IAAM,cAAc,GAAG,CAAC,CAAC;QACzB,IAAM,YAAY,GAAG,mBAAmB,GAAG,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC;QAChE,IAAI,WAAW,GAAG,IAAI,CAAC;QACvB,IAAI,MAAM,GAAG,KAAK,CAAC;QAEnB,wCAAwC;QACxC,WAAW,GAAG,UAAU,CAAC;YACrB,IAAI,CAAC,MAAM,EAAE,CAAC;gBACV,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,2CAA2C,CAAC,CAAC;gBACvE,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC,kCAAkC;YACtD,CAAC;QACL,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,qBAAqB;QAEhC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE;YACV,MAAM,GAAG,IAAI,CAAC;YACd,YAAY,CAAC,WAAW,CAAC,CAAC;YAE1B,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,oCAAoC,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;YACvH,KAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;YAEpC,QAAQ,GAAG,WAAW,CAAC;gBACnB,IAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACvB,IAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/C,IAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC;gBACvD,IAAI,SAAS,GAAG,YAAY,IAAI,WAAW,GAAG,cAAc,EAAE,CAAC;oBAC3D,EAAE,CAAC,KAAK,EAAE,CAAC;oBACX,OAAO;gBACX,CAAC;gBACD,WAAW,IAAI,CAAC,CAAC;gBACjB,KAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;YACxC,CAAC,EAAE,mBAAmB,CAAC,CAAC;QAC5B,CAAC,CAAC,CAAC;QAET,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,UAAO,OAA0B;;;;;wBACjD,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC;6BAClB,CAAA,OAAO,OAAO,KAAK,QAAQ,CAAA,EAA3B,wBAA2B;wBAC9B,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;wBAClD,CAAC;6BAEG,CAAA,OAAO,KAAK,MAAM,CAAA,EAAlB,wBAAkB;wBACrB,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,yBAAyB,CAAC,CAAC;wBACpD,CAAC;wBAED,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;;;6BAE5B,CAAA,OAAO,KAAK,MAAM,CAAA,EAAlB,wBAAkB;wBAC1B,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,WAAW,CAAC,CAAC;wBACtC,CAAC;wBACD,WAAW,GAAG,CAAC,CAAC;;;wBAIhB,IAAI,CAAC;4BACJ,KAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,oBAAW,CAAC,CAAC;wBACxC,CAAC;wBACD,OAAO,CAAC,EAAE,CAAC;4BACV,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC;4BACvC,sBAAO;wBACR,CAAC;6BAEG,CAAA,KAAG,CAAC,IAAI,KAAK,MAAM,CAAA,EAAnB,wBAAmB;wBACtB,qBAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,KAAG,CAAC,EAAA;;wBAAtC,SAAsC,CAAC;;4BAIzC,sBAAO;;wBAKR,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;4BAC9B,aAAa,GAAG,OAAO,CAAC;wBACzB,CAAC;6BACI,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;4BAC3B,MAAM,GAAG,OAA+C,CAAC;4BAC/D,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;wBACvC,CAAC;6BACI,IAAI,OAAO,YAAY,WAAW,EAAE,CAAC;4BACzC,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;wBACtC,CAAC;6BACI,IAAI,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;4BAChC,IAAI,GAAG,OAAiC,CAAC;4BAC/C,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;wBAC5E,CAAC;6BACI,CAAC;4BACL,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,OAAc,CAAC,CAAC;wBAC7C,CAAC;wBAED,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,uBAAuB,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;wBACxE,CAAC;wBAED,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BAC5B,SAAS,GAAG,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;4BAE/C,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;gCAC1B,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;oCAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,kCAAkC,CAAC,CAAC;gCAC7D,CAAC;gCAED,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;gCACpC,sBAAO;4BACR,CAAC;iCACI,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;gCAC/B,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;oCAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,oBAAoB,CAAC,CAAC;gCAC/C,CAAC;gCACD,WAAW,GAAG,CAAC,CAAC;gCAChB,sBAAO;4BACR,CAAC;wBACF,CAAC;wBAID,IAAI,CAAC;4BACJ,GAAG,GAAG,IAAA,iBAAM,EAAC,aAAa,CAAC,CAAC;wBAC7B,CAAC;wBACD,OAAO,CAAC,EAAE,CAAC;4BACV,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE,CAAC,CAAC,CAAC;4BAC9C,sBAAO;wBACR,CAAC;6BAEG,CAAA,GAAG,CAAC,IAAI,KAAK,MAAM,CAAA,EAAnB,wBAAmB;wBACtB,qBAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,GAAG,CAAC,EAAA;;wBAAtC,SAAsC,CAAC;;;;;aAExC,CAAC,CAAC;QAEG,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE;YACX,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,sDAAsD,CAAC,CAAC;YAChF,UAAU,CAAC;gBACP,KAAI,CAAC,mBAAmB,EAAE,CAAC;YAC/B,CAAC,EAAE,IAAI,CAAC,CAAC;YAET,IAAI,QAAQ,EAAE,CAAC;gBACX,aAAa,CAAC,QAAQ,CAAC,CAAC;YAC5B,CAAC;YACD,YAAY,CAAC,WAAW,CAAC,CAAC;QAC9B,CAAC,CAAC,CAAC;QAET,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,UAAC,GAAG;YAClB,IAAM,OAAO,GAAG,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,KAAI,EAAE,CAAC;YACnC,IAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC,iCAAiC,CAAC,CAAC;YAC3E,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrB,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,kBAAkB,EAAE,GAAG,CAAC,CAAC;YACpD,CAAC;iBACI,IAAI,KAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;gBAC/C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,2CAA2C,EAAE,OAAO,CAAC,CAAC;YAC/E,CAAC;YACD,EAAE,CAAC,KAAK,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC;IACD,CAAC;IAEa,gDAAkB,GAAhC,UAAiC,EAAuB,EAAE,IAAiB;;;;;;;;;wBACjE,MAAM,GAA6C,IAAI,OAAjD,EAAE,SAAS,GAAkC,IAAI,UAAtC,EAAE,MAAM,GAA0B,IAAI,OAA9B,EAAE,MAAM,GAAkB,IAAI,OAAtB,EAAE,WAAW,GAAK,IAAI,YAAT,CAAU;wBAC9D,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;6BAC5B,CAAA,CAAC,MAAM,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA,EAA5D,wBAA4D;6BACxD,CAAC,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,MAAM,CAAC,EAAtC,wBAAsC;wBACtC,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,kEAAkE,EAAE,MAAM,CAAC,CAAC;wBACtG,qBAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,gBAAgB,EAAE,yDAAkD,MAAM,gBAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,CAAC,EAAA;;wBAArJ,SAAqJ,CAAC;;;wBAG1J,sDAAsD;wBACtD,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE;4BACxB,IAAI,EAAE,cAAc;4BACpB,MAAM,QAAA;4BACN,SAAS,WAAA;4BACT,KAAK,EAAE,IAAI;4BACX,MAAM,EAAE,cAAc;yBACzB,CAAC,CAAC;wBAEH,sBAAO;;wBAGP,QAAQ,GAAG,KAAK,CAAC;wBACjB,aAAa,GAAG,UAAU,CAAC;;;;wCAC3B,QAAQ,GAAG,IAAI,CAAC;wCAChB,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,2BAA2B,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;wCAElF,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE;4CACxB,IAAI,EAAE,cAAc;4CACpB,MAAM,QAAA;4CACN,SAAS,WAAA;4CACT,KAAK,EAAE,IAAI;4CACX,MAAM,EAAE,gBAAgB;yCAC3B,CAAC,CAAC;wCAEH,qBAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,gBAAgB,EAAE,6BAAsB,MAAM,gBAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,CAAC,EAAA;;wCAAzH,SAAyH,CAAC;;;;6BAC7H,EAAE,CAAA,MAAA,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,MAAM,CAAC,0CAAE,eAAe,KAAI,CAAC,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;;;;wBAGtE,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,cAAc,EAAE,8BAAa,CAAC,SAAS,EAAE;4BAC9E,OAAO,EAAE,CAAA,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,OAAO,KAAI,EAAE;4BACnC,IAAI,EAAE,CAAA,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,IAAI,KAAI,EAAE;4BAC7B,KAAK,EAAE,CAAA,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,KAAK,KAAI,EAAE;yBAClC,CAAC,CAAC;wBAEH,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BACvC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,gBAAgB,EAAE,MAAM,CAAC,CAAC;wBACtD,CAAC;wBAEG,qBAAM,CAAA,KAAA,IAAI,CAAC,cAAc,CAAC,UAAU,CAAA,CAAC,IAAI,0BAAC,WAAW,EAAE,MAAM,UAAK,MAAM,YAAC;;wBAAlF,MAAM,GAAG,SAAyE;wBAClF,YAAY,GAAwB,IAAI,CAAC;wBAE7C,IAAI,CAAC;4BACJ,YAAY,GAAW,IAAA,eAAI,EAAC,MAAM,CAAC,CAAC;wBACrC,CAAC;wBACD,OAAO,OAAO,EAAE,CAAC;4BAChB,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;wBACzD,CAAC;wBAED,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,iBAAiB,EAAE,MAAM,CAAC,CAAC;wBACpD,CAAC;wBAED,IAAI,CAAC,QAAQ,EAAE,CAAC;4BACf,YAAY,CAAC,aAAa,CAAC,CAAC;4BACtB,OAAO,GAAiB;gCAC7B,IAAI,EAAE,cAAc;gCACpB,MAAM,QAAA;gCACN,SAAS,WAAA;gCACT,KAAK,EAAE,KAAK;gCACZ,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM;gCACpC,YAAY,cAAA;gCACZ,QAAQ,EAAE,SAAS;6BACnB,CAAC;4BAEF,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;wBACtC,CAAC;wBAEQ,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,KAAK,MAAM,EAAZ,CAAY,CAAC,CAAC;wBAElE,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BACvC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;wBAC3F,CAAC;;;;wBAGD,IAAI,CAAC,QAAQ,EAAE,CAAC;4BACZ,YAAY,CAAC,aAAa,CAAC,CAAC;4BAC5B,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAG,CAAC,CAAC;4BACrE,KAAG,CAAC,OAAO,GAAG,qBAAqB,GAAG,MAAM,GAAG,UAAU,GAAG,MAAM,GAAG,KAAK,GAAG,KAAG,CAAC,OAAO,CAAC;4BACzF,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE;gCACxB,IAAI,EAAE,cAAc;gCACpB,MAAM,QAAA;gCACN,SAAS,WAAA;gCACT,KAAK,EAAE,IAAI;gCACX,MAAM,EAAE,KAAG,IAAI,eAAe;6BACjC,CAAC,CAAC;4BACH,MAAM,KAAG,CAAC;wBACd,CAAC;wBAED,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAG,2BAAoB,MAAM,gBAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,CAAC,CAAC;wBAE5F,qBAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,gBAAgB,EAAE,2BAAoB,MAAM,gBAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,CAAC,EAAA;;wBAAvH,SAAuH,CAAC;wBAExH,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,KAAK,MAAM,EAAZ,CAAY,CAAC,CAAC;;;;;;KAEzE;IAEG,gDAAkB,GAAzB,UAA0B,EAAE,EAAE,OAA8B;QAC3D,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;YAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,SAAS,EAAE,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QACxF,CAAC;QAED,IAAI,EAAE,IAAI,EAAE,CAAC,UAAU,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;YACrC,IAAI,CAAC;gBACJ,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;oBACjC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAClB,CAAC;qBACI,CAAC;oBACL,EAAE,CAAC,IAAI,CAAC,IAAA,eAAI,EAAC,OAAO,CAAC,CAAC,CAAC;gBACxB,CAAC;YACF,CAAC;YACD,OAAO,GAAG,EAAE,CAAC;gBACZ,OAAO,CAAC,KAAK,CAAC,iCAAiC,EAAE,GAAG,CAAC,CAAC;YACvD,CAAC;QACF,CAAC;aACI,IAAI,EAAE,EAAE,CAAC;YACb,EAAE,CAAC,KAAK,EAAE,CAAC;QACZ,CAAC;IACF,CAAC;IACF,0BAAC;AAAD,CA1TA,AA0TC,IAAA;AA1TY,kDAAmB","file":"worker-server.manager.js","sourcesContent":["import { pack, unpack } from 'msgpackr';\nimport * as WebSocket from 'ws';\nimport { TaskPayload, TaskResponse } from '../models/server-message.model';\nimport { dateReviver } from '../util/common';\nimport { MethodManager } from './method.manager';\n\nexport class WorkerServerManager {\n    private _methodManager: MethodManager;\n    private _serverConfig;\n    private _runningTasks = [];\n\n    constructor() {}\n    \n    static create(methodManager: MethodManager, serverConfig) {\n        const workerServerManager = new WorkerServerManager();\n        workerServerManager.initialize(methodManager, serverConfig);\n        return workerServerManager;\n    }\n\n    public initialize(methodManager: MethodManager, serverConfig) {\n        this._methodManager = methodManager;\n        this._serverConfig = serverConfig;\n        this.startWorkerInstance();\n    }\n\n    private startWorkerInstance() {\n        console.log(new Date(), 'Worker instance started, connecting to main server via WebSocket...');\n    \n        const workerIndex = encodeURIComponent(String(process.env.WORKER_INDEX || ''));\n        const workerInstance = encodeURIComponent(String(process.env.NODE_APP_INSTANCE || ''));\n        let wsUrl = this._serverConfig['SERVER_URL'] + '/websocket?workerToken=' + this._serverConfig['WORKER_TOKEN'] + '&workerIndex=' + workerIndex + '&workerInstance=' + workerInstance;\n        const ws = new WebSocket(wsUrl);\n    \n        let lastComm = new Date();\n        let interval = null;\n        let missedPongs = 0;\n        const heartbeatIntervalMs = 15000;\n        const maxMissedPongs = 2;\n        const maxSilenceMs = heartbeatIntervalMs * (maxMissedPongs + 1);\n        let openTimeout = null;\n        let opened = false;\n    \n        // Set timeout if the socket never opens\n        openTimeout = setTimeout(() => {\n            if (!opened) {\n                console.error(new Date(), 'WebSocket connection timeout. Retrying...');\n                ws.terminate(); // force close if still connecting\n            }\n        }, 10000); // 10 seconds timeout\n    \n        ws.on('open', () => {\n            opened = true;\n            clearTimeout(openTimeout);\n    \n            console.log(new Date(), 'Connected to main server as worker', process.env.WORKER_INDEX, process.env.NODE_APP_INSTANCE);\n            this.sendWorkerResponse(ws, 'ping');\n    \n            interval = setInterval(() => {\n                const now = Date.now();\n                const last = lastComm ? lastComm.getTime() : 0;\n                const silenceMs = last ? now - last : maxSilenceMs + 1;\n                if (silenceMs > maxSilenceMs || missedPongs > maxMissedPongs) {\n                    ws.close();\n                    return;\n                }\n                missedPongs += 1;\n                this.sendWorkerResponse(ws, 'ping');\n            }, heartbeatIntervalMs);\n        });\n    \n\t\tws.on('message', async (rawData: WebSocket.RawData) => {\n\t\t\tlastComm = new Date();\n\t\t\tif (typeof rawData === 'string') {\n\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\tconsole.log(new Date(), 'Message Recv', rawData);\n\t\t\t\t}\n\n\t\t\t\tif (rawData === 'ping') {\n\t\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\t\tconsole.log(new Date(), 'Recv Ping, Sending Pong');\n\t\t\t\t\t}\n\t\t\t\t\t\n\t\t\t\t\tthis.sendWorkerResponse(ws, 'pong');\n\t\t\t\t}\n\t\t\t\telse if (rawData === 'pong') {\n\t\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\t\tconsole.log(new Date(), 'Recv Pong');\n\t\t\t\t\t}\n\t\t\t\t\tmissedPongs = 0;\n\t\t\t\t}\n\t\t\t\telse {\n\t\t\t\t\tlet msg: any;\n\t\t\t\t\ttry {\n\t\t\t\t\t\tmsg = JSON.parse(rawData, dateReviver);\n\t\t\t\t\t}\n\t\t\t\t\tcatch (e) {\n\t\t\t\t\t\tconsole.error('Worker parse error', e);\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\n\t\t\t\t\tif (msg.type === 'task') {\n\t\t\t\t\t\tawait this.handleIncomingTask(ws, msg);\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\treturn;\n\t\t\t}\n\n\t\t\tlet messageBuffer: Buffer;\n\n\t\t\tif (Buffer.isBuffer(rawData)) {\n\t\t\t\tmessageBuffer = rawData;\n\t\t\t}\n\t\t\telse if (Array.isArray(rawData)) {\n\t\t\t\tconst chunks = rawData as unknown as ReadonlyArray<Uint8Array>;\n\t\t\t\tmessageBuffer = Buffer.concat(chunks);\n\t\t\t}\n\t\t\telse if (rawData instanceof ArrayBuffer) {\n\t\t\t\tmessageBuffer = Buffer.from(rawData);\n\t\t\t}\n\t\t\telse if (ArrayBuffer.isView(rawData)) {\n\t\t\t\tconst view = rawData as NodeJS.ArrayBufferView;\n\t\t\t\tmessageBuffer = Buffer.from(view.buffer, view.byteOffset, view.byteLength);\n\t\t\t}\n\t\t\telse {\n\t\t\t\tmessageBuffer = Buffer.from(rawData as any);\n\t\t\t}\n\n\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\tconsole.log(new Date(), 'Message Recv (binary)', messageBuffer.length);\n\t\t\t}\n\n\t\t\tif (messageBuffer.length === 4) {\n\t\t\t\tlet heartbeat = messageBuffer.toString('utf8');\n\n\t\t\t\tif (heartbeat === 'ping') {\n\t\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\t\tconsole.log(new Date(), 'Recv Ping (binary), Sending Pong');\n\t\t\t\t\t}\n\n\t\t\t\t\tthis.sendWorkerResponse(ws, 'pong');\n\t\t\t\t\treturn;\n\t\t\t\t}\n\t\t\t\telse if (heartbeat === 'pong') {\n\t\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\t\tconsole.log(new Date(), 'Recv Pong (binary)');\n\t\t\t\t\t}\n\t\t\t\t\tmissedPongs = 0;\n\t\t\t\t\treturn;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tlet msg: TaskPayload;\n\n\t\t\ttry {\n\t\t\t\tmsg = unpack(messageBuffer);\n\t\t\t}\n\t\t\tcatch (e) {\n\t\t\t\tconsole.error('Worker binary parse error', e);\n\t\t\t\treturn;\n\t\t\t}\n\n\t\t\tif (msg.type === 'task') {\n\t\t\t\tawait this.handleIncomingTask(ws, msg);\n\t\t\t}\n\t\t});\n    \n        ws.on('close', () => {\n            console.log(new Date(), 'Disconnected from main server. Reconnecting in 1s...');\n            setTimeout(() => {\n                this.startWorkerInstance();\n            }, 1000);\n    \n            if (interval) {\n                clearInterval(interval);\n            }\n            clearTimeout(openTimeout);\n        });\n    \n\t\tws.on('error', (err) => {\n\t\t\tconst message = err?.message || '';\n\t\t\tconst isRestartNoise = message.includes('Unexpected server response: 502');\n\t\t\tif (!isRestartNoise) {\n\t\t\t\tconsole.error(new Date(), 'Worker WS error:', err);\n\t\t\t}\n\t\t\telse if (this._methodManager.getEnableDebug()) {\n\t\t\t\tconsole.log(new Date(), 'Worker WS reconnecting after main restart', message);\n\t\t\t}\n\t\t\tws.close();\n\t\t});\n    }\n\n    private async handleIncomingTask(ws: WebSocket.WebSocket, data: TaskPayload) {\n        let { taskId, messageId, method, params, userContext } = data;\n        this._runningTasks.push(taskId);\n        if (!taskId || !method || !this._methodManager.getMethod(method)) {\n            if (!this._methodManager.getMethod(method)) {\n                console.error(new Date(), 'No method in method manager for handleIncomingTask worker server', method);\n                await this._methodManager.callMethod('insertErrorLog', `No Method in worker server handleIncomingTask: ${method} - ${JSON.stringify(data, null, 2)}`);\n            }\n\n            // console.log('Invalid task message received', data);\n            this.sendWorkerResponse(ws, {\n                type: 'taskComplete',\n                taskId,\n                messageId,\n                error: true,\n                result: 'Invalid task'\n            });\n\n            return;\n        }\n\n        let timedOut = false;\n        let timeoutHandle = setTimeout(async () => {\n            timedOut = true;\n            console.error(new Date(), 'Worker timed out on task:', taskId, 'Method:', method);\n\n            this.sendWorkerResponse(ws, {\n                type: 'taskComplete',\n                taskId,\n                messageId,\n                error: true,\n                result: 'Task timed out'\n            });\n\n            await this._methodManager.callMethod('insertErrorLog', `Timeout in Method: ${method} - ${JSON.stringify(data, null, 2)}`);\n        }, this._methodManager.getMethod(method)?.timeoutOverride || (1000 * 60 * 2));\n\n        try {\n            let managerThis = Object.assign({}, this._methodManager, MethodManager.prototype, {\n                id_user: userContext?.id_user || '',\n                user: userContext?.user || '',\n                id_ws: userContext?.id_ws || ''\n            });\n\n            if (this._methodManager.getEnableDebug()) {\n                console.log(new Date(), 'Running method', method);\n            }\n\n\t\t\tlet result = await this._methodManager.callMethod.call(managerThis, method, ...params);\n\t\t\tlet packedResult: Uint8Array | Buffer = null;\n\n\t\t\ttry {\n\t\t\t\tpackedResult = <Buffer>pack(result);\n\t\t\t}\n\t\t\tcatch (packErr) {\n\t\t\t\tconsole.error(new Date(), 'Worker pack error', packErr);\n\t\t\t}\n\n\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\tconsole.log(new Date(), 'Finished method', method);\n\t\t\t}\n\n\t\t\tif (!timedOut) {\n\t\t\t\tclearTimeout(timeoutHandle);\n\t\t\t\tconst payload: TaskResponse = {\n\t\t\t\t\ttype: 'taskComplete',\n\t\t\t\t\ttaskId,\n\t\t\t\t\tmessageId,\n\t\t\t\t\terror: false,\n\t\t\t\t\tresult: packedResult ? null : result,\n\t\t\t\t\tpackedResult,\n\t\t\t\t\tencoding: 'msgpack'\n\t\t\t\t};\n\n\t\t\t\tthis.sendWorkerResponse(ws, payload);\n\t\t\t}\n\n            this._runningTasks = this._runningTasks.filter(a => a !== taskId);\n            \n            if (this._methodManager.getEnableDebug()) {\n                console.log(new Date(), 'Done with Task', JSON.stringify(this._runningTasks, null, 2));\n            }\n        }\n        catch (err) {\n            if (!timedOut) {\n                clearTimeout(timeoutHandle);\n                console.error('Worker failed task:', taskId, 'Method:', method, err);\n                err.message = 'Worker failed task:' + taskId + ' Method:' + method + ' - ' + err.message;\n                this.sendWorkerResponse(ws, {\n                    type: 'taskComplete',\n                    taskId,\n                    messageId,\n                    error: true,\n                    result: err || 'Unknown error'\n                });\n                throw err;\n            }\n\n            console.error(new Date(),  `Error in Method: ${method} - ${JSON.stringify(data, null, 2)}`);\n            \n            await this._methodManager.callMethod('insertErrorLog', `Error in Method: ${method} - ${JSON.stringify(data, null, 2)}`);\n            \n            this._runningTasks = this._runningTasks.filter(a => a !== taskId);\n        }\n    }\n\n\tpublic sendWorkerResponse(ws, payload: TaskResponse | string) {\n\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\tconsole.log(new Date(), 'Sending', typeof payload === 'string' ? payload : '[binary]');\n\t\t}\n\n\t\tif (ws && ws.readyState === ws.OPEN) {\n\t\t\ttry {\n\t\t\t\tif (typeof payload === 'string') {\n\t\t\t\t\tws.send(payload);\n\t\t\t\t}\n\t\t\t\telse {\n\t\t\t\t\tws.send(pack(payload));\n\t\t\t\t}\n\t\t\t}\n\t\t\tcatch (err) {\n\t\t\t\tconsole.error('Failed to send worker response:', err);\n\t\t\t}\n\t\t}\n\t\telse if (ws) {\n\t\t\tws.close();\n\t\t}\n\t}\n}\n"]}
+{"version":3,"sources":["../../src/managers/worker-server.manager.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,qCAAwC;AACxC,8BAAgC;AAEhC,yCAAoD;AACpD,mDAAiD;AAEjD;IAMI;QAHQ,kBAAa,GAAG,EAAE,CAAC;QACtB,wBAAmB,GAAG,GAAG,CAAC;IAEhB,CAAC;IAET,0BAAM,GAAb,UAAc,aAA4B,EAAE,YAAY;QACpD,IAAM,mBAAmB,GAAG,IAAI,mBAAmB,EAAE,CAAC;QACtD,mBAAmB,CAAC,UAAU,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;QAC5D,OAAO,mBAAmB,CAAC;IAC/B,CAAC;IAEM,wCAAU,GAAjB,UAAkB,aAA4B,EAAE,YAAY;QACxD,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;QAClC,IAAI,CAAC,mBAAmB,EAAE,CAAC;IAC/B,CAAC;IAEO,iDAAmB,GAA3B;QAAA,iBAqKC;QApKG,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,qEAAqE,CAAC,CAAC;QAE/F,IAAM,WAAW,GAAG,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAC;QAC/E,IAAM,cAAc,GAAG,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC,CAAC;QACvF,IAAI,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,GAAG,yBAAyB,GAAG,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,GAAG,eAAe,GAAG,WAAW,GAAG,kBAAkB,GAAG,cAAc,CAAC;QACpL,IAAM,EAAE,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC;QAEhC,IAAI,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC;QAC1B,IAAI,QAAQ,GAAG,IAAI,CAAC;QACpB,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAM,mBAAmB,GAAG,KAAK,CAAC;QAClC,IAAM,cAAc,GAAG,CAAC,CAAC;QACzB,IAAM,YAAY,GAAG,mBAAmB,GAAG,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC;QAChE,IAAI,WAAW,GAAG,IAAI,CAAC;QACvB,IAAI,MAAM,GAAG,KAAK,CAAC;QAEnB,wCAAwC;QACxC,WAAW,GAAG,UAAU,CAAC;YACrB,IAAI,CAAC,MAAM,EAAE,CAAC;gBACV,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,2CAA2C,CAAC,CAAC;gBACvE,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC,kCAAkC;YACtD,CAAC;QACL,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,qBAAqB;QAEhC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE;YACV,MAAM,GAAG,IAAI,CAAC;YACd,YAAY,CAAC,WAAW,CAAC,CAAC;YAE1B,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,oCAAoC,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;YACvH,KAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;YAEpC,QAAQ,GAAG,WAAW,CAAC;gBACnB,IAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACvB,IAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/C,IAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC;gBACvD,IAAI,SAAS,GAAG,YAAY,IAAI,WAAW,GAAG,cAAc,EAAE,CAAC;oBAC3D,EAAE,CAAC,KAAK,EAAE,CAAC;oBACX,OAAO;gBACX,CAAC;gBACD,WAAW,IAAI,CAAC,CAAC;gBACjB,KAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;YACxC,CAAC,EAAE,mBAAmB,CAAC,CAAC;QAC5B,CAAC,CAAC,CAAC;QAET,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,UAAO,OAA0B;;;;;wBACjD,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC;6BAClB,CAAA,OAAO,OAAO,KAAK,QAAQ,CAAA,EAA3B,wBAA2B;wBAC9B,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;wBAClD,CAAC;6BAEG,CAAA,OAAO,KAAK,MAAM,CAAA,EAAlB,wBAAkB;wBACrB,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,yBAAyB,CAAC,CAAC;wBACpD,CAAC;wBAED,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;;;6BAE5B,CAAA,OAAO,KAAK,MAAM,CAAA,EAAlB,wBAAkB;wBAC1B,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,WAAW,CAAC,CAAC;wBACtC,CAAC;wBACD,WAAW,GAAG,CAAC,CAAC;;;wBAIhB,IAAI,CAAC;4BACJ,KAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,oBAAW,CAAC,CAAC;wBACxC,CAAC;wBACD,OAAO,CAAC,EAAE,CAAC;4BACV,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC;4BACvC,sBAAO;wBACR,CAAC;6BAEG,CAAA,KAAG,CAAC,IAAI,KAAK,MAAM,CAAA,EAAnB,wBAAmB;wBACtB,qBAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,KAAG,CAAC,EAAA;;wBAAtC,SAAsC,CAAC;;4BAIzC,sBAAO;;wBAKR,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;4BAC9B,aAAa,GAAG,OAAO,CAAC;wBACzB,CAAC;6BACI,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;4BAC3B,MAAM,GAAG,OAA+C,CAAC;4BAC/D,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;wBACvC,CAAC;6BACI,IAAI,OAAO,YAAY,WAAW,EAAE,CAAC;4BACzC,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;wBACtC,CAAC;6BACI,IAAI,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;4BAChC,IAAI,GAAG,OAAiC,CAAC;4BAC/C,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;wBAC5E,CAAC;6BACI,CAAC;4BACL,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,OAAc,CAAC,CAAC;wBAC7C,CAAC;wBAED,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,uBAAuB,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;wBACxE,CAAC;wBAED,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BAC5B,SAAS,GAAG,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;4BAE/C,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;gCAC1B,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;oCAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,kCAAkC,CAAC,CAAC;gCAC7D,CAAC;gCAED,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;gCACpC,sBAAO;4BACR,CAAC;iCACI,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;gCAC/B,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;oCAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,oBAAoB,CAAC,CAAC;gCAC/C,CAAC;gCACD,WAAW,GAAG,CAAC,CAAC;gCAChB,sBAAO;4BACR,CAAC;wBACF,CAAC;wBAID,IAAI,CAAC;4BACJ,GAAG,GAAG,IAAA,iBAAM,EAAC,aAAa,CAAC,CAAC;wBAC7B,CAAC;wBACD,OAAO,CAAC,EAAE,CAAC;4BACV,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE,CAAC,CAAC,CAAC;4BAC9C,sBAAO;wBACR,CAAC;6BAEG,CAAA,GAAG,CAAC,IAAI,KAAK,MAAM,CAAA,EAAnB,wBAAmB;wBACtB,qBAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,GAAG,CAAC,EAAA;;wBAAtC,SAAsC,CAAC;;;;;aAExC,CAAC,CAAC;QAEG,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE;YACX,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,sDAAsD,CAAC,CAAC;YAChF,UAAU,CAAC;gBACP,KAAI,CAAC,mBAAmB,EAAE,CAAC;YAC/B,CAAC,EAAE,IAAI,CAAC,CAAC;YAET,IAAI,QAAQ,EAAE,CAAC;gBACX,aAAa,CAAC,QAAQ,CAAC,CAAC;YAC5B,CAAC;YACD,YAAY,CAAC,WAAW,CAAC,CAAC;QAC9B,CAAC,CAAC,CAAC;QAET,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,UAAC,GAAG;YAClB,IAAM,OAAO,GAAG,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,KAAI,EAAE,CAAC;YACnC,IAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC,iCAAiC,CAAC,CAAC;YAC3E,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrB,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,kBAAkB,EAAE,GAAG,CAAC,CAAC;YACpD,CAAC;iBACI,IAAI,KAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;gBAC/C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,2CAA2C,EAAE,OAAO,CAAC,CAAC;YAC/E,CAAC;YACD,EAAE,CAAC,KAAK,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC;IACD,CAAC;IAEa,gDAAkB,GAAhC,UAAiC,EAAuB,EAAE,IAAiB;;;;;;;;;wBACjE,MAAM,GAA6C,IAAI,OAAjD,EAAE,SAAS,GAAkC,IAAI,UAAtC,EAAE,MAAM,GAA0B,IAAI,OAA9B,EAAE,MAAM,GAAkB,IAAI,OAAtB,EAAE,WAAW,GAAK,IAAI,YAAT,CAAU;wBAC9D,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACxB,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;wBACzB,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;wBACjC,WAAW,GAAG,QAAQ,IAAI,YAAY,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,UAAU,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;wBAC3H,eAAe,GAAG,YAAY,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,UAAU,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;wBACzE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;6BAC5B,CAAA,CAAC,MAAM,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA,EAA5D,wBAA4D;6BACxD,CAAC,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,MAAM,CAAC,EAAtC,wBAAsC;wBACtC,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,kEAAkE,EAAE,MAAM,CAAC,CAAC;wBACtG,qBAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,gBAAgB,EAAE,yDAAkD,MAAM,gBAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,CAAC,EAAA;;wBAArJ,SAAqJ,CAAC;;;wBAG1J,sDAAsD;wBAC/D,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE;4BACxC,MAAM,QAAA;4BACN,MAAM,QAAA;4BACN,SAAS,WAAA;4BACT,WAAW,aAAA;4BACX,eAAe,iBAAA;4BACf,WAAW,aAAA;4BACX,MAAM,QAAA;yBACN,CAAC,CAAC;wBACM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE;4BACxB,IAAI,EAAE,cAAc;4BACpB,MAAM,QAAA;4BACN,SAAS,WAAA;4BACT,KAAK,EAAE,IAAI;4BACX,MAAM,EAAE,cAAc;yBACzB,CAAC,CAAC;wBAEH,sBAAO;;wBAGP,QAAQ,GAAG,KAAK,CAAC;wBACjB,aAAa,GAAG,UAAU,CAAC;;;;;wCAC3B,QAAQ,GAAG,IAAI,CAAC;wCACnB,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wCACvB,KAAK,GAAG,IAAA,cAAK,EAAC,SAAS,GAAG,UAAU,CAAC,CAAC;wCACtC,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;wCACrD,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,2BAA2B,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;wCAE3F,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE;4CACxC,MAAM,QAAA;4CACN,MAAM,QAAA;4CACN,SAAS,WAAA;4CACT,KAAK,OAAA;4CACL,WAAW,aAAA;4CACX,eAAe,iBAAA;4CACf,OAAO,SAAA;4CACP,WAAW,aAAA;4CACX,MAAM,QAAA;yCACN,CAAC,CAAC;wCAEM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE;4CACxB,IAAI,EAAE,cAAc;4CACpB,MAAM,QAAA;4CACN,SAAS,WAAA;4CACT,KAAK,EAAE,IAAI;4CACX,MAAM,EAAE,gBAAgB;yCAC3B,CAAC,CAAC;wCAEH,qBAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,gBAAgB,EAAE,6BAAsB,MAAM,gBAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,CAAC,EAAA;;wCAAzH,SAAyH,CAAC;;;;6BAC7H,EAAE,CAAA,MAAA,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,MAAM,CAAC,0CAAE,eAAe,KAAI,CAAC,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;;;;wBAGtE,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,cAAc,EAAE,8BAAa,CAAC,SAAS,EAAE;4BAC9E,OAAO,EAAE,CAAA,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,OAAO,KAAI,EAAE;4BACnC,IAAI,EAAE,CAAA,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,IAAI,KAAI,EAAE;4BAC7B,KAAK,EAAE,CAAA,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,KAAK,KAAI,EAAE;yBAClC,CAAC,CAAC;wBAEH,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BACvC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,gBAAgB,EAAE,MAAM,CAAC,CAAC;wBACtD,CAAC;wBAEG,qBAAM,CAAA,KAAA,IAAI,CAAC,cAAc,CAAC,UAAU,CAAA,CAAC,IAAI,0BAAC,WAAW,EAAE,MAAM,UAAK,MAAM,YAAC;;wBAAlF,MAAM,GAAG,SAAyE;wBAClF,YAAY,GAAwB,IAAI,CAAC;wBAE7C,IAAI,CAAC;4BACJ,YAAY,GAAW,IAAA,eAAI,EAAC,MAAM,CAAC,CAAC;wBACrC,CAAC;wBACD,OAAO,OAAO,EAAE,CAAC;4BAChB,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;wBACzD,CAAC;wBAEK,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACxB,KAAK,GAAG,IAAA,cAAK,EAAC,UAAU,GAAG,UAAU,CAAC,CAAC;wBACvC,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,UAAU,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;wBACzD,WAAW,GAAG,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;wBAE9D,IAAI,KAAK,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;4BACvC,IAAI,CAAC,cAAc,CAAC,oBAAoB,EAAE;gCACzC,MAAM,QAAA;gCACN,MAAM,QAAA;gCACN,SAAS,WAAA;gCACT,KAAK,OAAA;gCACL,WAAW,aAAA;gCACX,eAAe,iBAAA;gCACf,OAAO,SAAA;gCACP,WAAW,aAAA;gCACX,WAAW,aAAA;gCACX,MAAM,QAAA;6BACN,CAAC,CAAC;wBACJ,CAAC;wBAED,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,iBAAiB,EAAE,MAAM,CAAC,CAAC;wBACpD,CAAC;wBAED,IAAI,CAAC,QAAQ,EAAE,CAAC;4BACf,YAAY,CAAC,aAAa,CAAC,CAAC;4BACtB,OAAO,GAAiB;gCAC7B,IAAI,EAAE,cAAc;gCACpB,MAAM,QAAA;gCACN,SAAS,WAAA;gCACT,KAAK,EAAE,KAAK;gCACZ,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM;gCACpC,YAAY,cAAA;gCACZ,QAAQ,EAAE,SAAS;6BACnB,CAAC;4BAEF,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;wBACtC,CAAC;wBAEQ,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,KAAK,MAAM,EAAZ,CAAY,CAAC,CAAC;wBAElE,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BACvC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;wBAC3F,CAAC;;;;wBAGD,IAAI,CAAC,QAAQ,EAAE,CAAC;4BACZ,YAAY,CAAC,aAAa,CAAC,CAAC;4BAClC,YAAU,IAAI,CAAC,GAAG,EAAE,CAAC;4BACrB,UAAQ,IAAA,cAAK,EAAC,SAAO,GAAG,UAAU,CAAC,CAAC;4BACpC,YAAU,QAAQ,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,SAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;4BAChD,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAG,CAAC,CAAC;4BACrE,KAAG,CAAC,OAAO,GAAG,qBAAqB,GAAG,MAAM,GAAG,UAAU,GAAG,MAAM,GAAG,KAAK,GAAG,KAAG,CAAC,OAAO,CAAC;4BACrG,IAAI,CAAC,cAAc,CAAC,iBAAiB,EAAE;gCACtC,MAAM,QAAA;gCACN,MAAM,QAAA;gCACN,SAAS,WAAA;gCACT,KAAK,SAAA;gCACL,WAAW,aAAA;gCACX,eAAe,iBAAA;gCACf,OAAO,WAAA;gCACP,WAAW,aAAA;gCACX,MAAM,QAAA;gCACN,KAAK,EAAE,KAAG;6BACV,CAAC,CAAC;4BACS,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE;gCACxB,IAAI,EAAE,cAAc;gCACpB,MAAM,QAAA;gCACN,SAAS,WAAA;gCACT,KAAK,EAAE,IAAI;gCACX,MAAM,EAAE,KAAG,IAAI,eAAe;6BACjC,CAAC,CAAC;4BACH,MAAM,KAAG,CAAC;wBACd,CAAC;wBAEJ,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACrB,KAAK,GAAG,IAAA,cAAK,EAAC,OAAO,GAAG,UAAU,CAAC,CAAC;wBACpC,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;wBAC5D,IAAI,CAAC,cAAc,CAAC,iBAAiB,EAAE;4BACtC,MAAM,QAAA;4BACN,MAAM,QAAA;4BACN,SAAS,WAAA;4BACT,KAAK,OAAA;4BACL,WAAW,aAAA;4BACX,eAAe,iBAAA;4BACf,OAAO,SAAA;4BACP,WAAW,aAAA;4BACX,MAAM,QAAA;4BACN,KAAK,EAAE,KAAG;yBACV,CAAC,CAAC;wBAEM,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAG,2BAAoB,MAAM,gBAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,CAAC,CAAC;wBAE5F,qBAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,gBAAgB,EAAE,2BAAoB,MAAM,gBAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,CAAC,EAAA;;wBAAvH,SAAuH,CAAC;wBAExH,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,KAAK,MAAM,EAAZ,CAAY,CAAC,CAAC;;;;;;KAEzE;IAEI,4CAAc,GAAtB,UAAuB,KAAa,EAAE,OAA4B;QACjE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,YAC7B,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAC5B,KAAK,OAAA,IACF,OAAO,EACT,CAAC,CAAC;IACL,CAAC;IAEO,2CAAa,GAArB,UAAsB,KAAU;QAC/B,IAAI,CAAC;YACJ,IAAM,MAAI,GAAG,IAAI,OAAO,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,UAAC,GAAG,EAAE,GAAG;gBACrC,IAAI,GAAG,KAAK,WAAW,EAAE,CAAC;oBACzB,OAAO,SAAS,CAAC;gBAClB,CAAC;gBACD,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;oBAC7B,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;gBACvB,CAAC;gBACD,IAAI,GAAG,YAAY,KAAK,EAAE,CAAC;oBAC1B,OAAO;wBACN,IAAI,EAAE,GAAG,CAAC,IAAI;wBACd,OAAO,EAAE,GAAG,CAAC,OAAO;wBACpB,KAAK,EAAE,GAAG,CAAC,KAAK;qBAChB,CAAC;gBACH,CAAC;gBACD,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;oBACpC,IAAI,MAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;wBACnB,OAAO,YAAY,CAAC;oBACrB,CAAC;oBACD,MAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACf,CAAC;gBACD,OAAO,GAAG,CAAC;YACZ,CAAC,CAAC,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,SAAS,CAAC;gBACrB,KAAK,EAAE,qBAAqB;gBAC5B,OAAO,EAAE,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,KAAI,MAAM,CAAC,GAAG,CAAC;aACpC,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAEM,gDAAkB,GAAzB,UAA0B,EAAE,EAAE,OAA8B;QAC3D,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;YAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,SAAS,EAAE,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QACxF,CAAC;QAED,IAAI,EAAE,IAAI,EAAE,CAAC,UAAU,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;YACrC,IAAI,CAAC;gBACJ,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;oBACjC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAClB,CAAC;qBACI,CAAC;oBACL,EAAE,CAAC,IAAI,CAAC,IAAA,eAAI,EAAC,OAAO,CAAC,CAAC,CAAC;gBACxB,CAAC;YACF,CAAC;YACD,OAAO,GAAG,EAAE,CAAC;gBACZ,OAAO,CAAC,KAAK,CAAC,iCAAiC,EAAE,GAAG,CAAC,CAAC;YACvD,CAAC;QACF,CAAC;aACI,IAAI,EAAE,EAAE,CAAC;YACb,EAAE,CAAC,KAAK,EAAE,CAAC;QACZ,CAAC;IACF,CAAC;IACF,0BAAC;AAAD,CArbA,AAqbC,IAAA;AArbY,kDAAmB","file":"worker-server.manager.js","sourcesContent":["import { pack, unpack } from 'msgpackr';\nimport * as WebSocket from 'ws';\nimport { TaskPayload, TaskResponse } from '../models/server-message.model';\nimport { dateReviver, round } from '../util/common';\nimport { MethodManager } from './method.manager';\n\nexport class WorkerServerManager {\n    private _methodManager: MethodManager;\n    private _serverConfig;\n    private _runningTasks = [];\n\tprivate _taskLogThresholdMs = 200;\n\n    constructor() {}\n    \n    static create(methodManager: MethodManager, serverConfig) {\n        const workerServerManager = new WorkerServerManager();\n        workerServerManager.initialize(methodManager, serverConfig);\n        return workerServerManager;\n    }\n\n    public initialize(methodManager: MethodManager, serverConfig) {\n        this._methodManager = methodManager;\n        this._serverConfig = serverConfig;\n        this.startWorkerInstance();\n    }\n\n    private startWorkerInstance() {\n        console.log(new Date(), 'Worker instance started, connecting to main server via WebSocket...');\n    \n        const workerIndex = encodeURIComponent(String(process.env.WORKER_INDEX || ''));\n        const workerInstance = encodeURIComponent(String(process.env.NODE_APP_INSTANCE || ''));\n        let wsUrl = this._serverConfig['SERVER_URL'] + '/websocket?workerToken=' + this._serverConfig['WORKER_TOKEN'] + '&workerIndex=' + workerIndex + '&workerInstance=' + workerInstance;\n        const ws = new WebSocket(wsUrl);\n    \n        let lastComm = new Date();\n        let interval = null;\n        let missedPongs = 0;\n        const heartbeatIntervalMs = 15000;\n        const maxMissedPongs = 2;\n        const maxSilenceMs = heartbeatIntervalMs * (maxMissedPongs + 1);\n        let openTimeout = null;\n        let opened = false;\n    \n        // Set timeout if the socket never opens\n        openTimeout = setTimeout(() => {\n            if (!opened) {\n                console.error(new Date(), 'WebSocket connection timeout. Retrying...');\n                ws.terminate(); // force close if still connecting\n            }\n        }, 10000); // 10 seconds timeout\n    \n        ws.on('open', () => {\n            opened = true;\n            clearTimeout(openTimeout);\n    \n            console.log(new Date(), 'Connected to main server as worker', process.env.WORKER_INDEX, process.env.NODE_APP_INSTANCE);\n            this.sendWorkerResponse(ws, 'ping');\n    \n            interval = setInterval(() => {\n                const now = Date.now();\n                const last = lastComm ? lastComm.getTime() : 0;\n                const silenceMs = last ? now - last : maxSilenceMs + 1;\n                if (silenceMs > maxSilenceMs || missedPongs > maxMissedPongs) {\n                    ws.close();\n                    return;\n                }\n                missedPongs += 1;\n                this.sendWorkerResponse(ws, 'ping');\n            }, heartbeatIntervalMs);\n        });\n    \n\t\tws.on('message', async (rawData: WebSocket.RawData) => {\n\t\t\tlastComm = new Date();\n\t\t\tif (typeof rawData === 'string') {\n\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\tconsole.log(new Date(), 'Message Recv', rawData);\n\t\t\t\t}\n\n\t\t\t\tif (rawData === 'ping') {\n\t\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\t\tconsole.log(new Date(), 'Recv Ping, Sending Pong');\n\t\t\t\t\t}\n\t\t\t\t\t\n\t\t\t\t\tthis.sendWorkerResponse(ws, 'pong');\n\t\t\t\t}\n\t\t\t\telse if (rawData === 'pong') {\n\t\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\t\tconsole.log(new Date(), 'Recv Pong');\n\t\t\t\t\t}\n\t\t\t\t\tmissedPongs = 0;\n\t\t\t\t}\n\t\t\t\telse {\n\t\t\t\t\tlet msg: any;\n\t\t\t\t\ttry {\n\t\t\t\t\t\tmsg = JSON.parse(rawData, dateReviver);\n\t\t\t\t\t}\n\t\t\t\t\tcatch (e) {\n\t\t\t\t\t\tconsole.error('Worker parse error', e);\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\n\t\t\t\t\tif (msg.type === 'task') {\n\t\t\t\t\t\tawait this.handleIncomingTask(ws, msg);\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\treturn;\n\t\t\t}\n\n\t\t\tlet messageBuffer: Buffer;\n\n\t\t\tif (Buffer.isBuffer(rawData)) {\n\t\t\t\tmessageBuffer = rawData;\n\t\t\t}\n\t\t\telse if (Array.isArray(rawData)) {\n\t\t\t\tconst chunks = rawData as unknown as ReadonlyArray<Uint8Array>;\n\t\t\t\tmessageBuffer = Buffer.concat(chunks);\n\t\t\t}\n\t\t\telse if (rawData instanceof ArrayBuffer) {\n\t\t\t\tmessageBuffer = Buffer.from(rawData);\n\t\t\t}\n\t\t\telse if (ArrayBuffer.isView(rawData)) {\n\t\t\t\tconst view = rawData as NodeJS.ArrayBufferView;\n\t\t\t\tmessageBuffer = Buffer.from(view.buffer, view.byteOffset, view.byteLength);\n\t\t\t}\n\t\t\telse {\n\t\t\t\tmessageBuffer = Buffer.from(rawData as any);\n\t\t\t}\n\n\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\tconsole.log(new Date(), 'Message Recv (binary)', messageBuffer.length);\n\t\t\t}\n\n\t\t\tif (messageBuffer.length === 4) {\n\t\t\t\tlet heartbeat = messageBuffer.toString('utf8');\n\n\t\t\t\tif (heartbeat === 'ping') {\n\t\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\t\tconsole.log(new Date(), 'Recv Ping (binary), Sending Pong');\n\t\t\t\t\t}\n\n\t\t\t\t\tthis.sendWorkerResponse(ws, 'pong');\n\t\t\t\t\treturn;\n\t\t\t\t}\n\t\t\t\telse if (heartbeat === 'pong') {\n\t\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\t\tconsole.log(new Date(), 'Recv Pong (binary)');\n\t\t\t\t\t}\n\t\t\t\t\tmissedPongs = 0;\n\t\t\t\t\treturn;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tlet msg: TaskPayload;\n\n\t\t\ttry {\n\t\t\t\tmsg = unpack(messageBuffer);\n\t\t\t}\n\t\t\tcatch (e) {\n\t\t\t\tconsole.error('Worker binary parse error', e);\n\t\t\t\treturn;\n\t\t\t}\n\n\t\t\tif (msg.type === 'task') {\n\t\t\t\tawait this.handleIncomingTask(ws, msg);\n\t\t\t}\n\t\t});\n    \n        ws.on('close', () => {\n            console.log(new Date(), 'Disconnected from main server. Reconnecting in 1s...');\n            setTimeout(() => {\n                this.startWorkerInstance();\n            }, 1000);\n    \n            if (interval) {\n                clearInterval(interval);\n            }\n            clearTimeout(openTimeout);\n        });\n    \n\t\tws.on('error', (err) => {\n\t\t\tconst message = err?.message || '';\n\t\t\tconst isRestartNoise = message.includes('Unexpected server response: 502');\n\t\t\tif (!isRestartNoise) {\n\t\t\t\tconsole.error(new Date(), 'Worker WS error:', err);\n\t\t\t}\n\t\t\telse if (this._methodManager.getEnableDebug()) {\n\t\t\t\tconsole.log(new Date(), 'Worker WS reconnecting after main restart', message);\n\t\t\t}\n\t\t\tws.close();\n\t\t});\n    }\n\n    private async handleIncomingTask(ws: WebSocket.WebSocket, data: TaskPayload) {\n        let { taskId, messageId, method, params, userContext } = data;\n\t\tconst receivedAt = Date.now();\n\t\tconst queuedAt = data.queuedAt;\n\t\tconst dispatchedAt = data.dispatchedAt;\n\t\tconst queueWaitMs = queuedAt && dispatchedAt ? round(dispatchedAt - queuedAt) : (queuedAt ? round(receivedAt - queuedAt) : null);\n\t\tconst dispatchDelayMs = dispatchedAt ? round(receivedAt - dispatchedAt) : null;\n        this._runningTasks.push(taskId);\n        if (!taskId || !method || !this._methodManager.getMethod(method)) {\n            if (!this._methodManager.getMethod(method)) {\n                console.error(new Date(), 'No method in method manager for handleIncomingTask worker server', method);\n                await this._methodManager.callMethod('insertErrorLog', `No Method in worker server handleIncomingTask: ${method} - ${JSON.stringify(data, null, 2)}`);\n            }\n\n            // console.log('Invalid task message received', data);\n\t\t\tthis.logWorkerEvent('workerTaskInvalid', {\n\t\t\t\ttaskId,\n\t\t\t\tmethod,\n\t\t\t\tmessageId,\n\t\t\t\tqueueWaitMs,\n\t\t\t\tdispatchDelayMs,\n\t\t\t\tuserContext,\n\t\t\t\tparams\n\t\t\t});\n            this.sendWorkerResponse(ws, {\n                type: 'taskComplete',\n                taskId,\n                messageId,\n                error: true,\n                result: 'Invalid task'\n            });\n\n            return;\n        }\n\n        let timedOut = false;\n        let timeoutHandle = setTimeout(async () => {\n            timedOut = true;\n\t\t\tconst timeoutAt = Date.now();\n\t\t\tconst runMs = round(timeoutAt - receivedAt);\n\t\t\tconst totalMs = queuedAt ? round(timeoutAt - queuedAt) : null;\n            console.error(new Date(), 'Worker timed out on task:', taskId, 'Method:', method);\n\n\t\t\tthis.logWorkerEvent('workerTaskTimeout', {\n\t\t\t\ttaskId,\n\t\t\t\tmethod,\n\t\t\t\tmessageId,\n\t\t\t\trunMs,\n\t\t\t\tqueueWaitMs,\n\t\t\t\tdispatchDelayMs,\n\t\t\t\ttotalMs,\n\t\t\t\tuserContext,\n\t\t\t\tparams\n\t\t\t});\n\n            this.sendWorkerResponse(ws, {\n                type: 'taskComplete',\n                taskId,\n                messageId,\n                error: true,\n                result: 'Task timed out'\n            });\n\n            await this._methodManager.callMethod('insertErrorLog', `Timeout in Method: ${method} - ${JSON.stringify(data, null, 2)}`);\n        }, this._methodManager.getMethod(method)?.timeoutOverride || (1000 * 60 * 2));\n\n        try {\n            let managerThis = Object.assign({}, this._methodManager, MethodManager.prototype, {\n                id_user: userContext?.id_user || '',\n                user: userContext?.user || '',\n                id_ws: userContext?.id_ws || ''\n            });\n\n            if (this._methodManager.getEnableDebug()) {\n                console.log(new Date(), 'Running method', method);\n            }\n\n\t\t\tlet result = await this._methodManager.callMethod.call(managerThis, method, ...params);\n\t\t\tlet packedResult: Uint8Array | Buffer = null;\n\n\t\t\ttry {\n\t\t\t\tpackedResult = <Buffer>pack(result);\n\t\t\t}\n\t\t\tcatch (packErr) {\n\t\t\t\tconsole.error(new Date(), 'Worker pack error', packErr);\n\t\t\t}\n\n\t\t\tconst finishedAt = Date.now();\n\t\t\tconst runMs = round(finishedAt - receivedAt);\n\t\t\tconst totalMs = queuedAt ? round(finishedAt - queuedAt) : null;\n\t\t\tconst packedBytes = packedResult ? packedResult.length : null;\n\n\t\t\tif (runMs >= this._taskLogThresholdMs) {\n\t\t\t\tthis.logWorkerEvent('workerTaskComplete', {\n\t\t\t\t\ttaskId,\n\t\t\t\t\tmethod,\n\t\t\t\t\tmessageId,\n\t\t\t\t\trunMs,\n\t\t\t\t\tqueueWaitMs,\n\t\t\t\t\tdispatchDelayMs,\n\t\t\t\t\ttotalMs,\n\t\t\t\t\tpackedBytes,\n\t\t\t\t\tuserContext,\n\t\t\t\t\tparams\n\t\t\t\t});\n\t\t\t}\n\n\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\tconsole.log(new Date(), 'Finished method', method);\n\t\t\t}\n\n\t\t\tif (!timedOut) {\n\t\t\t\tclearTimeout(timeoutHandle);\n\t\t\t\tconst payload: TaskResponse = {\n\t\t\t\t\ttype: 'taskComplete',\n\t\t\t\t\ttaskId,\n\t\t\t\t\tmessageId,\n\t\t\t\t\terror: false,\n\t\t\t\t\tresult: packedResult ? null : result,\n\t\t\t\t\tpackedResult,\n\t\t\t\t\tencoding: 'msgpack'\n\t\t\t\t};\n\n\t\t\t\tthis.sendWorkerResponse(ws, payload);\n\t\t\t}\n\n            this._runningTasks = this._runningTasks.filter(a => a !== taskId);\n            \n            if (this._methodManager.getEnableDebug()) {\n                console.log(new Date(), 'Done with Task', JSON.stringify(this._runningTasks, null, 2));\n            }\n        }\n        catch (err) {\n            if (!timedOut) {\n                clearTimeout(timeoutHandle);\n\t\t\t\tconst errorAt = Date.now();\n\t\t\t\tconst runMs = round(errorAt - receivedAt);\n\t\t\t\tconst totalMs = queuedAt ? round(errorAt - queuedAt) : null;\n                console.error('Worker failed task:', taskId, 'Method:', method, err);\n                err.message = 'Worker failed task:' + taskId + ' Method:' + method + ' - ' + err.message;\n\t\t\t\tthis.logWorkerEvent('workerTaskError', {\n\t\t\t\t\ttaskId,\n\t\t\t\t\tmethod,\n\t\t\t\t\tmessageId,\n\t\t\t\t\trunMs,\n\t\t\t\t\tqueueWaitMs,\n\t\t\t\t\tdispatchDelayMs,\n\t\t\t\t\ttotalMs,\n\t\t\t\t\tuserContext,\n\t\t\t\t\tparams,\n\t\t\t\t\terror: err\n\t\t\t\t});\n                this.sendWorkerResponse(ws, {\n                    type: 'taskComplete',\n                    taskId,\n                    messageId,\n                    error: true,\n                    result: err || 'Unknown error'\n                });\n                throw err;\n            }\n\n\t\t\tconst errorAt = Date.now();\n\t\t\tconst runMs = round(errorAt - receivedAt);\n\t\t\tconst totalMs = queuedAt ? round(errorAt - queuedAt) : null;\n\t\t\tthis.logWorkerEvent('workerTaskError', {\n\t\t\t\ttaskId,\n\t\t\t\tmethod,\n\t\t\t\tmessageId,\n\t\t\t\trunMs,\n\t\t\t\tqueueWaitMs,\n\t\t\t\tdispatchDelayMs,\n\t\t\t\ttotalMs,\n\t\t\t\tuserContext,\n\t\t\t\tparams,\n\t\t\t\terror: err\n\t\t\t});\n\n            console.error(new Date(),  `Error in Method: ${method} - ${JSON.stringify(data, null, 2)}`);\n            \n            await this._methodManager.callMethod('insertErrorLog', `Error in Method: ${method} - ${JSON.stringify(data, null, 2)}`);\n            \n            this._runningTasks = this._runningTasks.filter(a => a !== taskId);\n        }\n    }\n\n\tprivate logWorkerEvent(event: string, details: Record<string, any>) {\n\t\tconsole.log(this.safeStringify({\n\t\t\tts: new Date().toISOString(),\n\t\t\tevent,\n\t\t\t...details\n\t\t}));\n\t}\n\n\tprivate safeStringify(value: any): string {\n\t\ttry {\n\t\t\tconst seen = new WeakSet();\n\t\t\treturn JSON.stringify(value, (key, val) => {\n\t\t\t\tif (key === '__proto__') {\n\t\t\t\t\treturn undefined;\n\t\t\t\t}\n\t\t\t\tif (typeof val === 'bigint') {\n\t\t\t\t\treturn val.toString();\n\t\t\t\t}\n\t\t\t\tif (val instanceof Error) {\n\t\t\t\t\treturn {\n\t\t\t\t\t\tname: val.name,\n\t\t\t\t\t\tmessage: val.message,\n\t\t\t\t\t\tstack: val.stack\n\t\t\t\t\t};\n\t\t\t\t}\n\t\t\t\tif (val && typeof val === 'object') {\n\t\t\t\t\tif (seen.has(val)) {\n\t\t\t\t\t\treturn '[Circular]';\n\t\t\t\t\t}\n\t\t\t\t\tseen.add(val);\n\t\t\t\t}\n\t\t\t\treturn val;\n\t\t\t});\n\t\t}\n\t\tcatch (err) {\n\t\t\treturn JSON.stringify({\n\t\t\t\terror: 'safeStringifyFailed',\n\t\t\t\tmessage: err?.message || String(err)\n\t\t\t});\n\t\t}\n\t}\n\n\tpublic sendWorkerResponse(ws, payload: TaskResponse | string) {\n\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\tconsole.log(new Date(), 'Sending', typeof payload === 'string' ? payload : '[binary]');\n\t\t}\n\n\t\tif (ws && ws.readyState === ws.OPEN) {\n\t\t\ttry {\n\t\t\t\tif (typeof payload === 'string') {\n\t\t\t\t\tws.send(payload);\n\t\t\t\t}\n\t\t\t\telse {\n\t\t\t\t\tws.send(pack(payload));\n\t\t\t\t}\n\t\t\t}\n\t\t\tcatch (err) {\n\t\t\t\tconsole.error('Failed to send worker response:', err);\n\t\t\t}\n\t\t}\n\t\telse if (ws) {\n\t\t\tws.close();\n\t\t}\n\t}\n}\n"]}

package/models/server-message.model.d.ts

@@ -15,6 +15,8 @@ export interface TaskQueueItem extends Task {
         user?: string;
         id_ws?: string;
     };
+    queuedAt?: number;
+    dispatchedAt?: number;
 }
 export interface TaskPayload extends TaskQueueItem {
     type: 'task';

package/models/server-message.model.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/models/server-message.model.ts"],"names":[],"mappings":"","file":"server-message.model.js","sourcesContent":["export interface ServerResponseModel {\n\tmessageId: number;\n\thasError: boolean;\n\tdata: any;\n}\n\ninterface Task {\n\ttaskId: string;\n\tmessageId: number;\n}\n\nexport interface TaskQueueItem extends Task {\n\tmethod: string;\n\tparams: any[];\n\tuserContext: { id_user?: string; user?: string; id_ws?: string };\n}\n\nexport interface TaskPayload extends TaskQueueItem {\n   type: 'task';\n}\n\nexport interface TaskResponse extends Task {\n\ttype: 'taskComplete';\n\terror: boolean;\n\tresult: any;\n\tpackedResult?: Uint8Array | Buffer;\n\tencoding?: 'msgpack' | 'json';\n}\n"]}
+{"version":3,"sources":["../../src/models/server-message.model.ts"],"names":[],"mappings":"","file":"server-message.model.js","sourcesContent":["export interface ServerResponseModel {\n\tmessageId: number;\n\thasError: boolean;\n\tdata: any;\n}\n\ninterface Task {\n\ttaskId: string;\n\tmessageId: number;\n}\n\nexport interface TaskQueueItem extends Task {\n\tmethod: string;\n\tparams: any[];\n\tuserContext: { id_user?: string; user?: string; id_ws?: string };\n\tqueuedAt?: number;\n\tdispatchedAt?: number;\n}\n\nexport interface TaskPayload extends TaskQueueItem {\n   type: 'task';\n}\n\nexport interface TaskResponse extends Task {\n\ttype: 'taskComplete';\n\terror: boolean;\n\tresult: any;\n\tpackedResult?: Uint8Array | Buffer;\n\tencoding?: 'msgpack' | 'json';\n}\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@resolveio/server-lib",
-  "version": "20.14.17",
+  "version": "20.14.18",
   "description": "",
 	"scripts": {
 		"package": "./build_package.sh",