@resolveio/server-lib 22.3.221 → 22.3.222

package/managers/worker-server.manager.js

@@ -0,0 +1,582 @@
+"use strict";
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
+    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+var __read = (this && this.__read) || function (o, n) {
+    var m = typeof Symbol === "function" && o[Symbol.iterator];
+    if (!m) return o;
+    var i = m.call(o), r, ar = [], e;
+    try {
+        while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);
+    }
+    catch (error) { e = { error: error }; }
+    finally {
+        try {
+            if (r && !r.done && (m = i["return"])) m.call(i);
+        }
+        finally { if (e) throw e.error; }
+    }
+    return ar;
+};
+var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) {
+    if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
+        if (ar || !(i in from)) {
+            if (!ar) ar = Array.prototype.slice.call(from, 0, i);
+            ar[i] = from[i];
+        }
+    }
+    return to.concat(ar || Array.prototype.slice.call(from));
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WorkerServerManager = void 0;
+var msgpackr_1 = require("msgpackr");
+var WebSocket = require("ws");
+var common_1 = require("../util/common");
+var method_manager_1 = require("./method.manager");
+var WorkerServerManager = /** @class */ (function () {
+    function WorkerServerManager() {
+        this._runningTasks = [];
+        this._taskLogThresholdMs = 200;
+        this._isConnected = false;
+        this._lastConnectionAttemptAt = 0;
+        this._lastConnectedAt = 0;
+        this._lastDisconnectedAt = 0;
+        this._connectionLossStartedAt = 0;
+        this._lastDisconnectReason = '';
+    }
+    WorkerServerManager.create = function (methodManager, serverConfig) {
+        var workerServerManager = new WorkerServerManager();
+        workerServerManager.initialize(methodManager, serverConfig);
+        return workerServerManager;
+    };
+    WorkerServerManager.prototype.initialize = function (methodManager, serverConfig) {
+        var _this = this;
+        this._methodManager = methodManager;
+        this._serverConfig = serverConfig;
+        if (this._methodManager && typeof this._methodManager.onReady === 'function') {
+            this._methodManager.onReady(function () { return _this.startWorkerInstance(); });
+        }
+        else {
+            this.startWorkerInstance();
+        }
+    };
+    WorkerServerManager.prototype.markWorkerConnectionAttempt = function () {
+        var now = Date.now();
+        this._lastConnectionAttemptAt = now;
+        if (!this._isConnected && !this._connectionLossStartedAt) {
+            this._connectionLossStartedAt = now;
+        }
+    };
+    WorkerServerManager.prototype.markWorkerConnected = function () {
+        var now = Date.now();
+        this._isConnected = true;
+        this._lastConnectedAt = now;
+        this._lastDisconnectedAt = 0;
+        this._connectionLossStartedAt = 0;
+        this._lastDisconnectReason = '';
+    };
+    WorkerServerManager.prototype.markWorkerDisconnected = function (reason) {
+        var now = Date.now();
+        this._isConnected = false;
+        this._lastDisconnectedAt = now;
+        if (!this._connectionLossStartedAt) {
+            this._connectionLossStartedAt = now;
+        }
+        this._lastDisconnectReason = String(reason || 'Worker websocket disconnected.');
+    };
+    WorkerServerManager.prototype.getConnectionHealth = function () {
+        return {
+            isConnected: this._isConnected,
+            lastConnectionAttemptAt: this._lastConnectionAttemptAt,
+            lastConnectedAt: this._lastConnectedAt,
+            lastDisconnectedAt: this._lastDisconnectedAt,
+            connectionLossStartedAt: this._connectionLossStartedAt,
+            currentOutageMs: (!this._isConnected && this._connectionLossStartedAt)
+                ? Math.max(0, Date.now() - this._connectionLossStartedAt)
+                : 0,
+            lastDisconnectReason: this._lastDisconnectReason
+        };
+    };
+    WorkerServerManager.prototype.startWorkerInstance = function () {
+        var _this = this;
+        var workerIndex = encodeURIComponent(String(process.env.WORKER_INDEX || ''));
+        var workerInstance = encodeURIComponent(String(process.env.NODE_APP_INSTANCE || ''));
+        var workerToken = encodeURIComponent(String(this._serverConfig['WORKER_TOKEN'] || ''));
+        var wsUrl = this._serverConfig['SERVER_URL'] + '/websocket?workerToken=' + workerToken + '&workerIndex=' + workerIndex + '&workerInstance=' + workerInstance;
+        var ws = new WebSocket(wsUrl);
+        this.markWorkerConnectionAttempt();
+        var lastComm = new Date();
+        var interval = null;
+        var missedPongs = 0;
+        var heartbeatIntervalMs = 15000;
+        var maxMissedPongs = 2;
+        var maxSilenceMs = heartbeatIntervalMs * (maxMissedPongs + 1);
+        var openTimeout = null;
+        var opened = false;
+        // Set timeout if the socket never opens
+        openTimeout = setTimeout(function () {
+            if (!opened) {
+                _this.markWorkerDisconnected('WebSocket connection timeout.');
+                console.error(new Date(), 'WebSocket connection timeout. Retrying...');
+                ws.terminate(); // force close if still connecting
+            }
+        }, 10000); // 10 seconds timeout
+        ws.on('open', function () {
+            opened = true;
+            clearTimeout(openTimeout);
+            _this.markWorkerConnected();
+            console.log(new Date(), 'Connected to main server as worker', process.env.WORKER_INDEX, process.env.NODE_APP_INSTANCE);
+            _this.sendWorkerResponse(ws, 'ping');
+            interval = setInterval(function () {
+                var now = Date.now();
+                var last = lastComm ? lastComm.getTime() : 0;
+                var silenceMs = last ? now - last : maxSilenceMs + 1;
+                if (silenceMs > maxSilenceMs || missedPongs > maxMissedPongs) {
+                    _this.markWorkerDisconnected('Worker websocket heartbeat timed out.');
+                    ws.close();
+                    return;
+                }
+                missedPongs += 1;
+                _this.sendWorkerResponse(ws, 'ping');
+            }, heartbeatIntervalMs);
+        });
+        ws.on('message', function (rawData) { return __awaiter(_this, void 0, void 0, function () {
+            var msg_1, messageBuffer, chunks, view, heartbeat, msg;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        lastComm = new Date();
+                        if (!(typeof rawData === 'string')) return [3 /*break*/, 5];
+                        if (this._methodManager.getEnableDebug()) {
+                            console.log(new Date(), 'Message Recv', rawData);
+                        }
+                        if (!(rawData === 'ping')) return [3 /*break*/, 1];
+                        if (this._methodManager.getEnableDebug()) {
+                            console.log(new Date(), 'Recv Ping, Sending Pong');
+                        }
+                        this.sendWorkerResponse(ws, 'pong');
+                        return [3 /*break*/, 4];
+                    case 1:
+                        if (!(rawData === 'pong')) return [3 /*break*/, 2];
+                        if (this._methodManager.getEnableDebug()) {
+                            console.log(new Date(), 'Recv Pong');
+                        }
+                        missedPongs = 0;
+                        return [3 /*break*/, 4];
+                    case 2:
+                        try {
+                            msg_1 = JSON.parse(rawData, common_1.dateReviver);
+                        }
+                        catch (e) {
+                            console.error('Worker parse error', e);
+                            return [2 /*return*/];
+                        }
+                        if (!(msg_1.type === 'task')) return [3 /*break*/, 4];
+                        return [4 /*yield*/, this.handleIncomingTask(ws, msg_1)];
+                    case 3:
+                        _a.sent();
+                        _a.label = 4;
+                    case 4: return [2 /*return*/];
+                    case 5:
+                        if (Buffer.isBuffer(rawData)) {
+                            messageBuffer = rawData;
+                        }
+                        else if (Array.isArray(rawData)) {
+                            chunks = rawData;
+                            messageBuffer = Buffer.concat(chunks);
+                        }
+                        else if (rawData instanceof ArrayBuffer) {
+                            messageBuffer = Buffer.from(rawData);
+                        }
+                        else if (ArrayBuffer.isView(rawData)) {
+                            view = rawData;
+                            messageBuffer = Buffer.from(view.buffer, view.byteOffset, view.byteLength);
+                        }
+                        else {
+                            messageBuffer = Buffer.from(rawData);
+                        }
+                        if (this._methodManager.getEnableDebug()) {
+                            console.log(new Date(), 'Message Recv (binary)', messageBuffer.length);
+                        }
+                        if (messageBuffer.length === 4) {
+                            heartbeat = messageBuffer.toString('utf8');
+                            if (heartbeat === 'ping') {
+                                if (this._methodManager.getEnableDebug()) {
+                                    console.log(new Date(), 'Recv Ping (binary), Sending Pong');
+                                }
+                                this.sendWorkerResponse(ws, 'pong');
+                                return [2 /*return*/];
+                            }
+                            else if (heartbeat === 'pong') {
+                                if (this._methodManager.getEnableDebug()) {
+                                    console.log(new Date(), 'Recv Pong (binary)');
+                                }
+                                missedPongs = 0;
+                                return [2 /*return*/];
+                            }
+                        }
+                        try {
+                            msg = (0, msgpackr_1.unpack)(messageBuffer);
+                        }
+                        catch (e) {
+                            console.error('Worker binary parse error', e);
+                            return [2 /*return*/];
+                        }
+                        if (!(msg.type === 'task')) return [3 /*break*/, 7];
+                        return [4 /*yield*/, this.handleIncomingTask(ws, msg)];
+                    case 6:
+                        _a.sent();
+                        _a.label = 7;
+                    case 7: return [2 /*return*/];
+                }
+            });
+        }); });
+        ws.on('close', function (code, reason) {
+            var closeReason = reason && reason.length
+                ? "Worker websocket closed (".concat(code, "): ").concat(reason.toString())
+                : "Worker websocket closed (".concat(code, ")");
+            _this.markWorkerDisconnected(closeReason);
+            setTimeout(function () {
+                _this.startWorkerInstance();
+            }, 1000);
+            if (interval) {
+                clearInterval(interval);
+            }
+            clearTimeout(openTimeout);
+        });
+        ws.on('error', function (err) {
+            var message = (err === null || err === void 0 ? void 0 : err.message) || '';
+            var statusMatch = message.match(/Unexpected server response:\s*(\d{3})/);
+            var statusCode = statusMatch ? statusMatch[1] : null;
+            var isRestartNoise = statusCode === '502';
+            if (statusCode && !isRestartNoise) {
+                console.error(new Date(), "Worker WS error: Failure ".concat(statusCode));
+            }
+            else if (!isRestartNoise) {
+                console.error(new Date(), 'Worker WS error:', err);
+            }
+            else if (_this._methodManager.getEnableDebug()) {
+                console.log(new Date(), 'Worker WS reconnecting after main restart', message);
+            }
+            _this.markWorkerDisconnected(message || 'Worker websocket error.');
+            ws.close();
+        });
+    };
+    WorkerServerManager.prototype.handleIncomingTask = function (ws, data) {
+        return __awaiter(this, void 0, void 0, function () {
+            var taskId, messageId, method, params, userContext, receivedAt, queuedAt, dispatchedAt, queueWaitMs, dispatchDelayMs, missingMethod, error_1, timedOut, timeoutHandle, managerThis, result, packedResult, meta, publicationPayload, publicationMeta, isPublicationResponse, payload, err_1, errorAt_1, runMs_1, totalMs_1, errorAt, runMs, totalMs;
+            var _a;
+            var _this = this;
+            var _b;
+            return __generator(this, function (_c) {
+                switch (_c.label) {
+                    case 0:
+                        taskId = data.taskId, messageId = data.messageId, method = data.method, params = data.params, userContext = data.userContext;
+                        receivedAt = Date.now();
+                        queuedAt = data.queuedAt;
+                        dispatchedAt = data.dispatchedAt;
+                        queueWaitMs = queuedAt && dispatchedAt ? (0, common_1.round)(dispatchedAt - queuedAt) : (queuedAt ? (0, common_1.round)(receivedAt - queuedAt) : null);
+                        dispatchDelayMs = dispatchedAt ? (0, common_1.round)(receivedAt - dispatchedAt) : null;
+                        this._runningTasks.push(taskId);
+                        missingMethod = !method || !this._methodManager.getMethod(method);
+                        if (!(!taskId || !method || missingMethod)) return [3 /*break*/, 5];
+                        if (!missingMethod) return [3 /*break*/, 4];
+                        console.error(new Date(), 'No method in method manager for handleIncomingTask worker server', method);
+                        if (!this._methodManager.getMethod('insertErrorLog')) return [3 /*break*/, 4];
+                        _c.label = 1;
+                    case 1:
+                        _c.trys.push([1, 3, , 4]);
+                        return [4 /*yield*/, this._methodManager.callMethod('insertErrorLog', "No Method in worker server handleIncomingTask: ".concat(method, " - ").concat(JSON.stringify(data, null, 2)))];
+                    case 2:
+                        _c.sent();
+                        return [3 /*break*/, 4];
+                    case 3:
+                        error_1 = _c.sent();
+                        console.error(new Date(), 'insertErrorLog failed in worker-server', error_1);
+                        return [3 /*break*/, 4];
+                    case 4:
+                        // console.log('Invalid task message received', data);
+                        this.logWorkerEvent('workerTaskInvalid', {
+                            taskId: taskId,
+                            method: method,
+                            messageId: messageId,
+                            queueWaitMs: queueWaitMs,
+                            dispatchDelayMs: dispatchDelayMs,
+                            userContext: userContext,
+                            params: params
+                        });
+                        this.sendWorkerResponse(ws, {
+                            type: 'taskComplete',
+                            taskId: taskId,
+                            messageId: messageId,
+                            error: true,
+                            result: 'Invalid task'
+                        });
+                        return [2 /*return*/];
+                    case 5:
+                        timedOut = false;
+                        timeoutHandle = setTimeout(function () { return __awaiter(_this, void 0, void 0, function () {
+                            var timeoutAt, runMs, totalMs;
+                            return __generator(this, function (_a) {
+                                switch (_a.label) {
+                                    case 0:
+                                        timedOut = true;
+                                        timeoutAt = Date.now();
+                                        runMs = (0, common_1.round)(timeoutAt - receivedAt);
+                                        totalMs = queuedAt ? (0, common_1.round)(timeoutAt - queuedAt) : null;
+                                        console.error(new Date(), 'Worker timed out on task:', taskId, 'Method:', method);
+                                        this.logWorkerEvent('workerTaskTimeout', {
+                                            taskId: taskId,
+                                            method: method,
+                                            messageId: messageId,
+                                            runMs: runMs,
+                                            queueWaitMs: queueWaitMs,
+                                            dispatchDelayMs: dispatchDelayMs,
+                                            totalMs: totalMs,
+                                            userContext: userContext,
+                                            params: params
+                                        });
+                                        this.sendWorkerResponse(ws, {
+                                            type: 'taskComplete',
+                                            taskId: taskId,
+                                            messageId: messageId,
+                                            error: true,
+                                            result: 'Task timed out'
+                                        });
+                                        return [4 /*yield*/, this._methodManager.callMethod('insertErrorLog', "Timeout in Method: ".concat(method, " - ").concat(JSON.stringify(data, null, 2)))];
+                                    case 1:
+                                        _a.sent();
+                                        return [2 /*return*/];
+                                }
+                            });
+                        }); }, ((_b = this._methodManager.getMethod(method)) === null || _b === void 0 ? void 0 : _b.timeoutOverride) || (1000 * 60 * 2));
+                        _c.label = 6;
+                    case 6:
+                        _c.trys.push([6, 8, , 10]);
+                        managerThis = Object.assign({}, this._methodManager, method_manager_1.MethodManager.prototype, {
+                            id_user: (userContext === null || userContext === void 0 ? void 0 : userContext.id_user) || '',
+                            user: (userContext === null || userContext === void 0 ? void 0 : userContext.user) || '',
+                            id_ws: (userContext === null || userContext === void 0 ? void 0 : userContext.id_ws) || ''
+                        });
+                        if (this._methodManager.getEnableDebug()) {
+                            console.log(new Date(), 'Running method', method);
+                        }
+                        return [4 /*yield*/, (_a = this._methodManager.callMethod).call.apply(_a, __spreadArray([managerThis, method], __read(params), false))];
+                    case 7:
+                        result = _c.sent();
+                        packedResult = null;
+                        meta = null;
+                        publicationPayload = undefined;
+                        publicationMeta = null;
+                        isPublicationResponse = !!(result && result.__publicationResponse);
+                        try {
+                            if (isPublicationResponse) {
+                                publicationPayload = result.payload;
+                                publicationMeta = {
+                                    type: 'publication',
+                                    publication: result.publication,
+                                    subscriptionData: result.subscriptionData,
+                                    userSpecific: result.userSpecific,
+                                    snapshot: result.snapshot
+                                };
+                            }
+                            if (isPublicationResponse) {
+                                meta = publicationMeta;
+                                packedResult = (0, msgpackr_1.pack)(publicationPayload);
+                                result = null;
+                            }
+                            else {
+                                packedResult = (0, msgpackr_1.pack)(result);
+                            }
+                        }
+                        catch (packErr) {
+                            console.error(new Date(), 'Worker pack error', packErr);
+                            if (isPublicationResponse) {
+                                meta = publicationMeta;
+                                result = publicationPayload;
+                            }
+                        }
+                        if (this._methodManager.getEnableDebug()) {
+                            console.log(new Date(), 'Finished method', method);
+                        }
+                        if (!timedOut) {
+                            clearTimeout(timeoutHandle);
+                            payload = {
+                                type: 'taskComplete',
+                                taskId: taskId,
+                                messageId: messageId,
+                                error: false,
+                                result: packedResult ? null : result,
+                                packedResult: packedResult,
+                                encoding: 'msgpack',
+                                meta: meta
+                            };
+                            this.sendWorkerResponse(ws, payload);
+                        }
+                        this._runningTasks = this._runningTasks.filter(function (a) { return a !== taskId; });
+                        if (this._methodManager.getEnableDebug()) {
+                            console.log(new Date(), 'Done with Task', JSON.stringify(this._runningTasks, null, 2));
+                        }
+                        return [3 /*break*/, 10];
+                    case 8:
+                        err_1 = _c.sent();
+                        if (!timedOut) {
+                            clearTimeout(timeoutHandle);
+                            errorAt_1 = Date.now();
+                            runMs_1 = (0, common_1.round)(errorAt_1 - receivedAt);
+                            totalMs_1 = queuedAt ? (0, common_1.round)(errorAt_1 - queuedAt) : null;
+                            console.error('Worker failed task:', taskId, 'Method:', method, err_1);
+                            err_1.message = 'Worker failed task:' + taskId + ' Method:' + method + ' - ' + err_1.message;
+                            this.logWorkerEvent('workerTaskError', {
+                                taskId: taskId,
+                                method: method,
+                                messageId: messageId,
+                                runMs: runMs_1,
+                                queueWaitMs: queueWaitMs,
+                                dispatchDelayMs: dispatchDelayMs,
+                                totalMs: totalMs_1,
+                                userContext: userContext,
+                                params: params,
+                                error: err_1
+                            });
+                            this.sendWorkerResponse(ws, {
+                                type: 'taskComplete',
+                                taskId: taskId,
+                                messageId: messageId,
+                                error: true,
+                                result: err_1 || 'Unknown error'
+                            });
+                            throw err_1;
+                        }
+                        errorAt = Date.now();
+                        runMs = (0, common_1.round)(errorAt - receivedAt);
+                        totalMs = queuedAt ? (0, common_1.round)(errorAt - queuedAt) : null;
+                        this.logWorkerEvent('workerTaskError', {
+                            taskId: taskId,
+                            method: method,
+                            messageId: messageId,
+                            runMs: runMs,
+                            queueWaitMs: queueWaitMs,
+                            dispatchDelayMs: dispatchDelayMs,
+                            totalMs: totalMs,
+                            userContext: userContext,
+                            params: params,
+                            error: err_1
+                        });
+                        console.error(new Date(), "Error in Method: ".concat(method, " - ").concat(JSON.stringify(data, null, 2)));
+                        return [4 /*yield*/, this._methodManager.callMethod('insertErrorLog', "Error in Method: ".concat(method, " - ").concat(JSON.stringify(data, null, 2)))];
+                    case 9:
+                        _c.sent();
+                        this._runningTasks = this._runningTasks.filter(function (a) { return a !== taskId; });
+                        return [3 /*break*/, 10];
+                    case 10: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    WorkerServerManager.prototype.logWorkerEvent = function (event, details) {
+        console.log(this.safeStringify(__assign({ ts: new Date().toISOString(), event: event }, details)));
+    };
+    WorkerServerManager.prototype.safeStringify = function (value) {
+        try {
+            var seen_1 = new WeakSet();
+            return JSON.stringify(value, function (key, val) {
+                if (key === '__proto__') {
+                    return undefined;
+                }
+                if (typeof val === 'bigint') {
+                    return val.toString();
+                }
+                if (val instanceof Error) {
+                    return {
+                        name: val.name,
+                        message: val.message,
+                        stack: val.stack
+                    };
+                }
+                if (val && typeof val === 'object') {
+                    if (seen_1.has(val)) {
+                        return '[Circular]';
+                    }
+                    seen_1.add(val);
+                }
+                return val;
+            });
+        }
+        catch (err) {
+            return JSON.stringify({
+                error: 'safeStringifyFailed',
+                message: (err === null || err === void 0 ? void 0 : err.message) || String(err)
+            });
+        }
+    };
+    WorkerServerManager.prototype.sendWorkerResponse = function (ws, payload) {
+        if (this._methodManager.getEnableDebug()) {
+            console.log(new Date(), 'Sending', typeof payload === 'string' ? payload : '[binary]');
+        }
+        if (ws && ws.readyState === WebSocket.OPEN) {
+            try {
+                if (typeof payload === 'string') {
+                    ws.send(payload);
+                }
+                else {
+                    ws.send((0, msgpackr_1.pack)(payload));
+                }
+            }
+            catch (err) {
+                console.error('Failed to send worker response:', err);
+            }
+        }
+        else if (ws) {
+            ws.close();
+        }
+    };
+    return WorkerServerManager;
+}());
+exports.WorkerServerManager = WorkerServerManager;
+
+//# sourceMappingURL=worker-server.manager.js.map

package/managers/worker-server.manager.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/managers/worker-server.manager.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,qCAAwC;AACxC,8BAAgC;AAEhC,yCAAoD;AACpD,mDAAiD;AAYjD;IAYC;QATQ,kBAAa,GAAG,EAAE,CAAC;QACnB,wBAAmB,GAAG,GAAG,CAAC;QAC1B,iBAAY,GAAG,KAAK,CAAC;QACrB,6BAAwB,GAAG,CAAC,CAAC;QAC7B,qBAAgB,GAAG,CAAC,CAAC;QACrB,wBAAmB,GAAG,CAAC,CAAC;QACxB,6BAAwB,GAAG,CAAC,CAAC;QAC7B,0BAAqB,GAAG,EAAE,CAAC;IAEpB,CAAC;IAET,0BAAM,GAAb,UAAc,aAA4B,EAAE,YAAY;QACvD,IAAM,mBAAmB,GAAG,IAAI,mBAAmB,EAAE,CAAC;QACtD,mBAAmB,CAAC,UAAU,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;QAC5D,OAAO,mBAAmB,CAAC;IAC5B,CAAC;IAEM,wCAAU,GAAjB,UAAkB,aAA4B,EAAE,YAAY;QAA5D,iBASC;QARA,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;QAClC,IAAI,IAAI,CAAC,cAAc,IAAI,OAAO,IAAI,CAAC,cAAc,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;YAC9E,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,cAAM,OAAA,KAAI,CAAC,mBAAmB,EAAE,EAA1B,CAA0B,CAAC,CAAC;QAC/D,CAAC;aACI,CAAC;YACL,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC5B,CAAC;IACF,CAAC;IAEO,yDAA2B,GAAnC;QACC,IAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,wBAAwB,GAAG,GAAG,CAAC;QACpC,IAAI,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAC1D,IAAI,CAAC,wBAAwB,GAAG,GAAG,CAAC;QACrC,CAAC;IACF,CAAC;IAEO,iDAAmB,GAA3B;QACC,IAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,gBAAgB,GAAG,GAAG,CAAC;QAC5B,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,wBAAwB,GAAG,CAAC,CAAC;QAClC,IAAI,CAAC,qBAAqB,GAAG,EAAE,CAAC;IACjC,CAAC;IAEO,oDAAsB,GAA9B,UAA+B,MAAc;QAC5C,IAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,mBAAmB,GAAG,GAAG,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,wBAAwB,EAAE,CAAC;YACpC,IAAI,CAAC,wBAAwB,GAAG,GAAG,CAAC;QACrC,CAAC;QACD,IAAI,CAAC,qBAAqB,GAAG,MAAM,CAAC,MAAM,IAAI,gCAAgC,CAAC,CAAC;IACjF,CAAC;IAEM,iDAAmB,GAA1B;QACC,OAAO;YACN,WAAW,EAAE,IAAI,CAAC,YAAY;YAC9B,uBAAuB,EAAE,IAAI,CAAC,wBAAwB;YACtD,eAAe,EAAE,IAAI,CAAC,gBAAgB;YACtC,kBAAkB,EAAE,IAAI,CAAC,mBAAmB;YAC5C,uBAAuB,EAAE,IAAI,CAAC,wBAAwB;YACtD,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,wBAAwB,CAAC;gBACrE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,wBAAwB,CAAC;gBACzD,CAAC,CAAC,CAAC;YACJ,oBAAoB,EAAE,IAAI,CAAC,qBAAqB;SAChD,CAAC;IACH,CAAC;IAEO,iDAAmB,GAA3B;QAAA,iBAmLC;QAlLA,IAAM,WAAW,GAAG,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAC;QAC/E,IAAM,cAAc,GAAG,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC,CAAC;QACvF,IAAM,WAAW,GAAG,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACzF,IAAI,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,GAAG,yBAAyB,GAAG,WAAW,GAAG,eAAe,GAAG,WAAW,GAAG,kBAAkB,GAAG,cAAc,CAAC;QAC7J,IAAM,EAAE,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC;QAEhC,IAAI,CAAC,2BAA2B,EAAE,CAAC;QAEnC,IAAI,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC;QAC1B,IAAI,QAAQ,GAAG,IAAI,CAAC;QACpB,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAM,mBAAmB,GAAG,KAAK,CAAC;QAClC,IAAM,cAAc,GAAG,CAAC,CAAC;QACzB,IAAM,YAAY,GAAG,mBAAmB,GAAG,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC;QAChE,IAAI,WAAW,GAAG,IAAI,CAAC;QACvB,IAAI,MAAM,GAAG,KAAK,CAAC;QAEnB,wCAAwC;QACxC,WAAW,GAAG,UAAU,CAAC;YACxB,IAAI,CAAC,MAAM,EAAE,CAAC;gBACb,KAAI,CAAC,sBAAsB,CAAC,+BAA+B,CAAC,CAAC;gBAC7D,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,2CAA2C,CAAC,CAAC;gBACvE,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC,kCAAkC;YACnD,CAAC;QACF,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,qBAAqB;QAEhC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE;YACb,MAAM,GAAG,IAAI,CAAC;YACd,YAAY,CAAC,WAAW,CAAC,CAAC;YAC1B,KAAI,CAAC,mBAAmB,EAAE,CAAC;YAE3B,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,oCAAoC,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;YACvH,KAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;YAEpC,QAAQ,GAAG,WAAW,CAAC;gBACtB,IAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACvB,IAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/C,IAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC;gBACvD,IAAI,SAAS,GAAG,YAAY,IAAI,WAAW,GAAG,cAAc,EAAE,CAAC;oBAC9D,KAAI,CAAC,sBAAsB,CAAC,uCAAuC,CAAC,CAAC;oBACrE,EAAE,CAAC,KAAK,EAAE,CAAC;oBACX,OAAO;gBACR,CAAC;gBACD,WAAW,IAAI,CAAC,CAAC;gBACjB,KAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;YACrC,CAAC,EAAE,mBAAmB,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,UAAO,OAA0B;;;;;wBACjD,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC;6BAClB,CAAA,OAAO,OAAO,KAAK,QAAQ,CAAA,EAA3B,wBAA2B;wBAC9B,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;wBAClD,CAAC;6BAEG,CAAA,OAAO,KAAK,MAAM,CAAA,EAAlB,wBAAkB;wBACrB,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,yBAAyB,CAAC,CAAC;wBACpD,CAAC;wBAED,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;;;6BAE5B,CAAA,OAAO,KAAK,MAAM,CAAA,EAAlB,wBAAkB;wBAC1B,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,WAAW,CAAC,CAAC;wBACtC,CAAC;wBACD,WAAW,GAAG,CAAC,CAAC;;;wBAIhB,IAAI,CAAC;4BACJ,KAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,oBAAW,CAAC,CAAC;wBACxC,CAAC;wBACD,OAAO,CAAC,EAAE,CAAC;4BACV,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC;4BACvC,sBAAO;wBACR,CAAC;6BAEG,CAAA,KAAG,CAAC,IAAI,KAAK,MAAM,CAAA,EAAnB,wBAAmB;wBACtB,qBAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,KAAG,CAAC,EAAA;;wBAAtC,SAAsC,CAAC;;4BAIzC,sBAAO;;wBAKR,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;4BAC9B,aAAa,GAAG,OAAO,CAAC;wBACzB,CAAC;6BACI,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;4BAC3B,MAAM,GAAG,OAA+C,CAAC;4BAC/D,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;wBACvC,CAAC;6BACI,IAAI,OAAO,YAAY,WAAW,EAAE,CAAC;4BACzC,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;wBACtC,CAAC;6BACI,IAAI,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;4BAChC,IAAI,GAAG,OAAiC,CAAC;4BAC/C,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;wBAC5E,CAAC;6BACI,CAAC;4BACL,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,OAAc,CAAC,CAAC;wBAC7C,CAAC;wBAED,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,uBAAuB,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;wBACxE,CAAC;wBAED,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BAC5B,SAAS,GAAG,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;4BAE/C,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;gCAC1B,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;oCAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,kCAAkC,CAAC,CAAC;gCAC7D,CAAC;gCAED,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;gCACpC,sBAAO;4BACR,CAAC;iCACI,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;gCAC/B,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;oCAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,oBAAoB,CAAC,CAAC;gCAC/C,CAAC;gCACD,WAAW,GAAG,CAAC,CAAC;gCAChB,sBAAO;4BACR,CAAC;wBACF,CAAC;wBAID,IAAI,CAAC;4BACJ,GAAG,GAAG,IAAA,iBAAM,EAAC,aAAa,CAAC,CAAC;wBAC7B,CAAC;wBACD,OAAO,CAAC,EAAE,CAAC;4BACV,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE,CAAC,CAAC,CAAC;4BAC9C,sBAAO;wBACR,CAAC;6BAEG,CAAA,GAAG,CAAC,IAAI,KAAK,MAAM,CAAA,EAAnB,wBAAmB;wBACtB,qBAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,GAAG,CAAC,EAAA;;wBAAtC,SAAsC,CAAC;;;;;aAExC,CAAC,CAAC;QAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,UAAC,IAAI,EAAE,MAAM;YAC3B,IAAM,WAAW,GAAG,MAAM,IAAI,MAAM,CAAC,MAAM;gBAC1C,CAAC,CAAC,mCAA4B,IAAI,gBAAM,MAAM,CAAC,QAAQ,EAAE,CAAE;gBAC3D,CAAC,CAAC,mCAA4B,IAAI,MAAG,CAAC;YACvC,KAAI,CAAC,sBAAsB,CAAC,WAAW,CAAC,CAAC;YAEzC,UAAU,CAAC;gBACV,KAAI,CAAC,mBAAmB,EAAE,CAAC;YAC5B,CAAC,EAAE,IAAI,CAAC,CAAC;YAET,IAAI,QAAQ,EAAE,CAAC;gBACd,aAAa,CAAC,QAAQ,CAAC,CAAC;YACzB,CAAC;YACD,YAAY,CAAC,WAAW,CAAC,CAAC;QAC3B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,UAAC,GAAG;YAClB,IAAM,OAAO,GAAG,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,KAAI,EAAE,CAAC;YACnC,IAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAC3E,IAAM,UAAU,GAAG,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACvD,IAAM,cAAc,GAAG,UAAU,KAAK,KAAK,CAAC;YAC5C,IAAI,UAAU,IAAI,CAAC,cAAc,EAAE,CAAC;gBACnC,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,mCAA4B,UAAU,CAAE,CAAC,CAAC;YACrE,CAAC;iBACI,IAAI,CAAC,cAAc,EAAE,CAAC;gBAC1B,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,kBAAkB,EAAE,GAAG,CAAC,CAAC;YACpD,CAAC;iBACI,IAAI,KAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;gBAC/C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,2CAA2C,EAAE,OAAO,CAAC,CAAC;YAC/E,CAAC;YACD,KAAI,CAAC,sBAAsB,CAAC,OAAO,IAAI,yBAAyB,CAAC,CAAC;YAClE,EAAE,CAAC,KAAK,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC;IACJ,CAAC;IAEgB,gDAAkB,GAAhC,UAAiC,EAAuB,EAAE,IAAiB;;;;;;;;;wBACjE,MAAM,GAA6C,IAAI,OAAjD,EAAE,SAAS,GAAkC,IAAI,UAAtC,EAAE,MAAM,GAA0B,IAAI,OAA9B,EAAE,MAAM,GAAkB,IAAI,OAAtB,EAAE,WAAW,GAAK,IAAI,YAAT,CAAU;wBAC9D,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACxB,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;wBACzB,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;wBACjC,WAAW,GAAG,QAAQ,IAAI,YAAY,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,UAAU,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;wBAC3H,eAAe,GAAG,YAAY,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,UAAU,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;wBACzE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;wBAC1B,aAAa,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;6BACpE,CAAA,CAAC,MAAM,IAAI,CAAC,MAAM,IAAI,aAAa,CAAA,EAAnC,wBAAmC;6BAC/B,aAAa,EAAb,wBAAa;wBACb,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,kEAAkE,EAAE,MAAM,CAAC,CAAC;6BAC9G,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,gBAAgB,CAAC,EAA/C,wBAA+C;;;;wBAEjD,qBAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,gBAAgB,EAAE,yDAAkD,MAAM,gBAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,CAAC,EAAA;;wBAArJ,SAAqJ,CAAC;;;;wBAGtJ,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,wCAAwC,EAAE,OAAK,CAAC,CAAC;;;wBAKrE,sDAAsD;wBAC/D,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE;4BACxC,MAAM,QAAA;4BACN,MAAM,QAAA;4BACN,SAAS,WAAA;4BACT,WAAW,aAAA;4BACX,eAAe,iBAAA;4BACf,WAAW,aAAA;4BACX,MAAM,QAAA;yBACN,CAAC,CAAC;wBACM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE;4BACxB,IAAI,EAAE,cAAc;4BACpB,MAAM,QAAA;4BACN,SAAS,WAAA;4BACT,KAAK,EAAE,IAAI;4BACX,MAAM,EAAE,cAAc;yBACzB,CAAC,CAAC;wBAEH,sBAAO;;wBAGP,QAAQ,GAAG,KAAK,CAAC;wBACjB,aAAa,GAAG,UAAU,CAAC;;;;;wCAC3B,QAAQ,GAAG,IAAI,CAAC;wCACnB,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wCACvB,KAAK,GAAG,IAAA,cAAK,EAAC,SAAS,GAAG,UAAU,CAAC,CAAC;wCACtC,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;wCACrD,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,2BAA2B,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;wCAE3F,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE;4CACxC,MAAM,QAAA;4CACN,MAAM,QAAA;4CACN,SAAS,WAAA;4CACT,KAAK,OAAA;4CACL,WAAW,aAAA;4CACX,eAAe,iBAAA;4CACf,OAAO,SAAA;4CACP,WAAW,aAAA;4CACX,MAAM,QAAA;yCACN,CAAC,CAAC;wCAEM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE;4CACxB,IAAI,EAAE,cAAc;4CACpB,MAAM,QAAA;4CACN,SAAS,WAAA;4CACT,KAAK,EAAE,IAAI;4CACX,MAAM,EAAE,gBAAgB;yCAC3B,CAAC,CAAC;wCAEH,qBAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,gBAAgB,EAAE,6BAAsB,MAAM,gBAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,CAAC,EAAA;;wCAAzH,SAAyH,CAAC;;;;6BAC7H,EAAE,CAAA,MAAA,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,MAAM,CAAC,0CAAE,eAAe,KAAI,CAAC,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;;;;wBAGtE,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,cAAc,EAAE,8BAAa,CAAC,SAAS,EAAE;4BAC9E,OAAO,EAAE,CAAA,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,OAAO,KAAI,EAAE;4BACnC,IAAI,EAAE,CAAA,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,IAAI,KAAI,EAAE;4BAC7B,KAAK,EAAE,CAAA,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,KAAK,KAAI,EAAE;yBAClC,CAAC,CAAC;wBAEH,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BACvC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,gBAAgB,EAAE,MAAM,CAAC,CAAC;wBACtD,CAAC;wBAEG,qBAAM,CAAA,KAAA,IAAI,CAAC,cAAc,CAAC,UAAU,CAAA,CAAC,IAAI,0BAAC,WAAW,EAAE,MAAM,UAAK,MAAM,YAAC;;wBAAlF,MAAM,GAAG,SAAyE;wBAClF,YAAY,GAAwB,IAAI,CAAC;wBACzC,IAAI,GAAwB,IAAI,CAAC;wBACjC,kBAAkB,GAAQ,SAAS,CAAC;wBACpC,eAAe,GAAwB,IAAI,CAAC;wBAC1C,qBAAqB,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC,qBAAqB,CAAC,CAAC;wBAEzE,IAAI,CAAC;4BACJ,IAAI,qBAAqB,EAAE,CAAC;gCAC3B,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC;gCACpC,eAAe,GAAG;oCACjB,IAAI,EAAE,aAAa;oCACnB,WAAW,EAAE,MAAM,CAAC,WAAW;oCAC/B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;oCACzC,YAAY,EAAE,MAAM,CAAC,YAAY;oCACjC,QAAQ,EAAE,MAAM,CAAC,QAAQ;iCACzB,CAAC;4BACH,CAAC;4BAED,IAAI,qBAAqB,EAAE,CAAC;gCAC3B,IAAI,GAAG,eAAe,CAAC;gCACvB,YAAY,GAAW,IAAA,eAAI,EAAC,kBAAkB,CAAC,CAAC;gCAChD,MAAM,GAAG,IAAI,CAAC;4BACf,CAAC;iCACI,CAAC;gCACL,YAAY,GAAW,IAAA,eAAI,EAAC,MAAM,CAAC,CAAC;4BACrC,CAAC;wBACF,CAAC;wBACD,OAAO,OAAO,EAAE,CAAC;4BAChB,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;4BACxD,IAAI,qBAAqB,EAAE,CAAC;gCAC3B,IAAI,GAAG,eAAe,CAAC;gCACvB,MAAM,GAAG,kBAAkB,CAAC;4BAC7B,CAAC;wBACF,CAAC;wBAGD,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,iBAAiB,EAAE,MAAM,CAAC,CAAC;wBACpD,CAAC;wBAED,IAAI,CAAC,QAAQ,EAAE,CAAC;4BACf,YAAY,CAAC,aAAa,CAAC,CAAC;4BACtB,OAAO,GAAiB;gCAC7B,IAAI,EAAE,cAAc;gCACpB,MAAM,QAAA;gCACN,SAAS,WAAA;gCACT,KAAK,EAAE,KAAK;gCACZ,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM;gCACpC,YAAY,cAAA;gCACZ,QAAQ,EAAE,SAAS;gCACnB,IAAI,MAAA;6BACJ,CAAC;4BAEF,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;wBACtC,CAAC;wBAEQ,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,KAAK,MAAM,EAAZ,CAAY,CAAC,CAAC;wBAElE,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BACvC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;wBAC3F,CAAC;;;;wBAGD,IAAI,CAAC,QAAQ,EAAE,CAAC;4BACZ,YAAY,CAAC,aAAa,CAAC,CAAC;4BAClC,YAAU,IAAI,CAAC,GAAG,EAAE,CAAC;4BACrB,UAAQ,IAAA,cAAK,EAAC,SAAO,GAAG,UAAU,CAAC,CAAC;4BACpC,YAAU,QAAQ,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,SAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;4BAChD,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAG,CAAC,CAAC;4BACrE,KAAG,CAAC,OAAO,GAAG,qBAAqB,GAAG,MAAM,GAAG,UAAU,GAAG,MAAM,GAAG,KAAK,GAAG,KAAG,CAAC,OAAO,CAAC;4BACrG,IAAI,CAAC,cAAc,CAAC,iBAAiB,EAAE;gCACtC,MAAM,QAAA;gCACN,MAAM,QAAA;gCACN,SAAS,WAAA;gCACT,KAAK,SAAA;gCACL,WAAW,aAAA;gCACX,eAAe,iBAAA;gCACf,OAAO,WAAA;gCACP,WAAW,aAAA;gCACX,MAAM,QAAA;gCACN,KAAK,EAAE,KAAG;6BACV,CAAC,CAAC;4BACS,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE;gCACxB,IAAI,EAAE,cAAc;gCACpB,MAAM,QAAA;gCACN,SAAS,WAAA;gCACT,KAAK,EAAE,IAAI;gCACX,MAAM,EAAE,KAAG,IAAI,eAAe;6BACjC,CAAC,CAAC;4BACH,MAAM,KAAG,CAAC;wBACd,CAAC;wBAEJ,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACrB,KAAK,GAAG,IAAA,cAAK,EAAC,OAAO,GAAG,UAAU,CAAC,CAAC;wBACpC,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;wBAC5D,IAAI,CAAC,cAAc,CAAC,iBAAiB,EAAE;4BACtC,MAAM,QAAA;4BACN,MAAM,QAAA;4BACN,SAAS,WAAA;4BACT,KAAK,OAAA;4BACL,WAAW,aAAA;4BACX,eAAe,iBAAA;4BACf,OAAO,SAAA;4BACP,WAAW,aAAA;4BACX,MAAM,QAAA;4BACN,KAAK,EAAE,KAAG;yBACV,CAAC,CAAC;wBAEM,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAG,2BAAoB,MAAM,gBAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,CAAC,CAAC;wBAE5F,qBAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,gBAAgB,EAAE,2BAAoB,MAAM,gBAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,CAAC,EAAA;;wBAAvH,SAAuH,CAAC;wBAExH,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,KAAK,MAAM,EAAZ,CAAY,CAAC,CAAC;;;;;;KAEzE;IAEI,4CAAc,GAAtB,UAAuB,KAAa,EAAE,OAA4B;QACjE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,YAC7B,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAC5B,KAAK,OAAA,IACF,OAAO,EACT,CAAC,CAAC;IACL,CAAC;IAEO,2CAAa,GAArB,UAAsB,KAAU;QAC/B,IAAI,CAAC;YACJ,IAAM,MAAI,GAAG,IAAI,OAAO,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,UAAC,GAAG,EAAE,GAAG;gBACrC,IAAI,GAAG,KAAK,WAAW,EAAE,CAAC;oBACzB,OAAO,SAAS,CAAC;gBAClB,CAAC;gBACD,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;oBAC7B,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;gBACvB,CAAC;gBACD,IAAI,GAAG,YAAY,KAAK,EAAE,CAAC;oBAC1B,OAAO;wBACN,IAAI,EAAE,GAAG,CAAC,IAAI;wBACd,OAAO,EAAE,GAAG,CAAC,OAAO;wBACpB,KAAK,EAAE,GAAG,CAAC,KAAK;qBAChB,CAAC;gBACH,CAAC;gBACD,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;oBACpC,IAAI,MAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;wBACnB,OAAO,YAAY,CAAC;oBACrB,CAAC;oBACD,MAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACf,CAAC;gBACD,OAAO,GAAG,CAAC;YACZ,CAAC,CAAC,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,SAAS,CAAC;gBACrB,KAAK,EAAE,qBAAqB;gBAC5B,OAAO,EAAE,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,KAAI,MAAM,CAAC,GAAG,CAAC;aACpC,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAEM,gDAAkB,GAAzB,UAA0B,EAAE,EAAE,OAA8B;QAC3D,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;YAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,SAAS,EAAE,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QACxF,CAAC;QAED,IAAI,EAAE,IAAI,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,CAAC;gBACJ,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;oBACjC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAClB,CAAC;qBACI,CAAC;oBACL,EAAE,CAAC,IAAI,CAAC,IAAA,eAAI,EAAC,OAAO,CAAC,CAAC,CAAC;gBACxB,CAAC;YACF,CAAC;YACD,OAAO,GAAG,EAAE,CAAC;gBACZ,OAAO,CAAC,KAAK,CAAC,iCAAiC,EAAE,GAAG,CAAC,CAAC;YACvD,CAAC;QACF,CAAC;aACI,IAAI,EAAE,EAAE,CAAC;YACb,EAAE,CAAC,KAAK,EAAE,CAAC;QACZ,CAAC;IACF,CAAC;IACF,0BAAC;AAAD,CAvgBA,AAugBC,IAAA;AAvgBY,kDAAmB","file":"worker-server.manager.js","sourcesContent":["import { pack, unpack } from 'msgpackr';\nimport * as WebSocket from 'ws';\nimport { TaskPayload, TaskResponse } from '../models/server-message.model';\nimport { dateReviver, round } from '../util/common';\nimport { MethodManager } from './method.manager';\n\nexport interface WorkerServerConnectionHealth {\n\tisConnected: boolean;\n\tlastConnectionAttemptAt: number;\n\tlastConnectedAt: number;\n\tlastDisconnectedAt: number;\n\tconnectionLossStartedAt: number;\n\tcurrentOutageMs: number;\n\tlastDisconnectReason: string;\n}\n\nexport class WorkerServerManager {\n\tprivate _methodManager: MethodManager;\n\tprivate _serverConfig;\n\tprivate _runningTasks = [];\n\tprivate _taskLogThresholdMs = 200;\n\tprivate _isConnected = false;\n\tprivate _lastConnectionAttemptAt = 0;\n\tprivate _lastConnectedAt = 0;\n\tprivate _lastDisconnectedAt = 0;\n\tprivate _connectionLossStartedAt = 0;\n\tprivate _lastDisconnectReason = '';\n\n\tconstructor() {}\n\t\n\tstatic create(methodManager: MethodManager, serverConfig) {\n\t\tconst workerServerManager = new WorkerServerManager();\n\t\tworkerServerManager.initialize(methodManager, serverConfig);\n\t\treturn workerServerManager;\n\t}\n\n\tpublic initialize(methodManager: MethodManager, serverConfig) {\n\t\tthis._methodManager = methodManager;\n\t\tthis._serverConfig = serverConfig;\n\t\tif (this._methodManager && typeof this._methodManager.onReady === 'function') {\n\t\t\tthis._methodManager.onReady(() => this.startWorkerInstance());\n\t\t}\n\t\telse {\n\t\t\tthis.startWorkerInstance();\n\t\t}\n\t}\n\n\tprivate markWorkerConnectionAttempt() {\n\t\tconst now = Date.now();\n\t\tthis._lastConnectionAttemptAt = now;\n\t\tif (!this._isConnected && !this._connectionLossStartedAt) {\n\t\t\tthis._connectionLossStartedAt = now;\n\t\t}\n\t}\n\n\tprivate markWorkerConnected() {\n\t\tconst now = Date.now();\n\t\tthis._isConnected = true;\n\t\tthis._lastConnectedAt = now;\n\t\tthis._lastDisconnectedAt = 0;\n\t\tthis._connectionLossStartedAt = 0;\n\t\tthis._lastDisconnectReason = '';\n\t}\n\n\tprivate markWorkerDisconnected(reason: string) {\n\t\tconst now = Date.now();\n\t\tthis._isConnected = false;\n\t\tthis._lastDisconnectedAt = now;\n\t\tif (!this._connectionLossStartedAt) {\n\t\t\tthis._connectionLossStartedAt = now;\n\t\t}\n\t\tthis._lastDisconnectReason = String(reason || 'Worker websocket disconnected.');\n\t}\n\n\tpublic getConnectionHealth(): WorkerServerConnectionHealth {\n\t\treturn {\n\t\t\tisConnected: this._isConnected,\n\t\t\tlastConnectionAttemptAt: this._lastConnectionAttemptAt,\n\t\t\tlastConnectedAt: this._lastConnectedAt,\n\t\t\tlastDisconnectedAt: this._lastDisconnectedAt,\n\t\t\tconnectionLossStartedAt: this._connectionLossStartedAt,\n\t\t\tcurrentOutageMs: (!this._isConnected && this._connectionLossStartedAt)\n\t\t\t\t? Math.max(0, Date.now() - this._connectionLossStartedAt)\n\t\t\t\t: 0,\n\t\t\tlastDisconnectReason: this._lastDisconnectReason\n\t\t};\n\t}\n\n\tprivate startWorkerInstance() {\n\t\tconst workerIndex = encodeURIComponent(String(process.env.WORKER_INDEX || ''));\n\t\tconst workerInstance = encodeURIComponent(String(process.env.NODE_APP_INSTANCE || ''));\n\t\tconst workerToken = encodeURIComponent(String(this._serverConfig['WORKER_TOKEN'] || ''));\n\t\tlet wsUrl = this._serverConfig['SERVER_URL'] + '/websocket?workerToken=' + workerToken + '&workerIndex=' + workerIndex + '&workerInstance=' + workerInstance;\n\t\tconst ws = new WebSocket(wsUrl);\n\n\t\tthis.markWorkerConnectionAttempt();\n\n\t\tlet lastComm = new Date();\n\t\tlet interval = null;\n\t\tlet missedPongs = 0;\n\t\tconst heartbeatIntervalMs = 15000;\n\t\tconst maxMissedPongs = 2;\n\t\tconst maxSilenceMs = heartbeatIntervalMs * (maxMissedPongs + 1);\n\t\tlet openTimeout = null;\n\t\tlet opened = false;\n\n\t\t// Set timeout if the socket never opens\n\t\topenTimeout = setTimeout(() => {\n\t\t\tif (!opened) {\n\t\t\t\tthis.markWorkerDisconnected('WebSocket connection timeout.');\n\t\t\t\tconsole.error(new Date(), 'WebSocket connection timeout. Retrying...');\n\t\t\t\tws.terminate(); // force close if still connecting\n\t\t\t}\n\t\t}, 10000); // 10 seconds timeout\n\n\t\tws.on('open', () => {\n\t\t\topened = true;\n\t\t\tclearTimeout(openTimeout);\n\t\t\tthis.markWorkerConnected();\n\n\t\t\tconsole.log(new Date(), 'Connected to main server as worker', process.env.WORKER_INDEX, process.env.NODE_APP_INSTANCE);\n\t\t\tthis.sendWorkerResponse(ws, 'ping');\n\n\t\t\tinterval = setInterval(() => {\n\t\t\t\tconst now = Date.now();\n\t\t\t\tconst last = lastComm ? lastComm.getTime() : 0;\n\t\t\t\tconst silenceMs = last ? now - last : maxSilenceMs + 1;\n\t\t\t\tif (silenceMs > maxSilenceMs || missedPongs > maxMissedPongs) {\n\t\t\t\t\tthis.markWorkerDisconnected('Worker websocket heartbeat timed out.');\n\t\t\t\t\tws.close();\n\t\t\t\t\treturn;\n\t\t\t\t}\n\t\t\t\tmissedPongs += 1;\n\t\t\t\tthis.sendWorkerResponse(ws, 'ping');\n\t\t\t}, heartbeatIntervalMs);\n\t\t});\n\n\t\tws.on('message', async (rawData: WebSocket.RawData) => {\n\t\t\tlastComm = new Date();\n\t\t\tif (typeof rawData === 'string') {\n\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\tconsole.log(new Date(), 'Message Recv', rawData);\n\t\t\t\t}\n\n\t\t\t\tif (rawData === 'ping') {\n\t\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\t\tconsole.log(new Date(), 'Recv Ping, Sending Pong');\n\t\t\t\t\t}\n\t\t\t\t\t\n\t\t\t\t\tthis.sendWorkerResponse(ws, 'pong');\n\t\t\t\t}\n\t\t\t\telse if (rawData === 'pong') {\n\t\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\t\tconsole.log(new Date(), 'Recv Pong');\n\t\t\t\t\t}\n\t\t\t\t\tmissedPongs = 0;\n\t\t\t\t}\n\t\t\t\telse {\n\t\t\t\t\tlet msg: any;\n\t\t\t\t\ttry {\n\t\t\t\t\t\tmsg = JSON.parse(rawData, dateReviver);\n\t\t\t\t\t}\n\t\t\t\t\tcatch (e) {\n\t\t\t\t\t\tconsole.error('Worker parse error', e);\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\n\t\t\t\t\tif (msg.type === 'task') {\n\t\t\t\t\t\tawait this.handleIncomingTask(ws, msg);\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\treturn;\n\t\t\t}\n\n\t\t\tlet messageBuffer: Buffer;\n\n\t\t\tif (Buffer.isBuffer(rawData)) {\n\t\t\t\tmessageBuffer = rawData;\n\t\t\t}\n\t\t\telse if (Array.isArray(rawData)) {\n\t\t\t\tconst chunks = rawData as unknown as ReadonlyArray<Uint8Array>;\n\t\t\t\tmessageBuffer = Buffer.concat(chunks);\n\t\t\t}\n\t\t\telse if (rawData instanceof ArrayBuffer) {\n\t\t\t\tmessageBuffer = Buffer.from(rawData);\n\t\t\t}\n\t\t\telse if (ArrayBuffer.isView(rawData)) {\n\t\t\t\tconst view = rawData as NodeJS.ArrayBufferView;\n\t\t\t\tmessageBuffer = Buffer.from(view.buffer, view.byteOffset, view.byteLength);\n\t\t\t}\n\t\t\telse {\n\t\t\t\tmessageBuffer = Buffer.from(rawData as any);\n\t\t\t}\n\n\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\tconsole.log(new Date(), 'Message Recv (binary)', messageBuffer.length);\n\t\t\t}\n\n\t\t\tif (messageBuffer.length === 4) {\n\t\t\t\tlet heartbeat = messageBuffer.toString('utf8');\n\n\t\t\t\tif (heartbeat === 'ping') {\n\t\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\t\tconsole.log(new Date(), 'Recv Ping (binary), Sending Pong');\n\t\t\t\t\t}\n\n\t\t\t\t\tthis.sendWorkerResponse(ws, 'pong');\n\t\t\t\t\treturn;\n\t\t\t\t}\n\t\t\t\telse if (heartbeat === 'pong') {\n\t\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\t\tconsole.log(new Date(), 'Recv Pong (binary)');\n\t\t\t\t\t}\n\t\t\t\t\tmissedPongs = 0;\n\t\t\t\t\treturn;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tlet msg: TaskPayload;\n\n\t\t\ttry {\n\t\t\t\tmsg = unpack(messageBuffer);\n\t\t\t}\n\t\t\tcatch (e) {\n\t\t\t\tconsole.error('Worker binary parse error', e);\n\t\t\t\treturn;\n\t\t\t}\n\n\t\t\tif (msg.type === 'task') {\n\t\t\t\tawait this.handleIncomingTask(ws, msg);\n\t\t\t}\n\t\t});\n    \n\t\tws.on('close', (code, reason) => {\n\t\t\tconst closeReason = reason && reason.length\n\t\t\t\t? `Worker websocket closed (${code}): ${reason.toString()}`\n\t\t\t\t: `Worker websocket closed (${code})`;\n\t\t\tthis.markWorkerDisconnected(closeReason);\n\n\t\t\tsetTimeout(() => {\n\t\t\t\tthis.startWorkerInstance();\n\t\t\t}, 1000);\n\n\t\t\tif (interval) {\n\t\t\t\tclearInterval(interval);\n\t\t\t}\n\t\t\tclearTimeout(openTimeout);\n\t\t});\n\n\t\tws.on('error', (err) => {\n\t\t\tconst message = err?.message || '';\n\t\t\tconst statusMatch = message.match(/Unexpected server response:\\s*(\\d{3})/);\n\t\t\tconst statusCode = statusMatch ? statusMatch[1] : null;\n\t\t\tconst isRestartNoise = statusCode === '502';\n\t\t\tif (statusCode && !isRestartNoise) {\n\t\t\t\tconsole.error(new Date(), `Worker WS error: Failure ${statusCode}`);\n\t\t\t}\n\t\t\telse if (!isRestartNoise) {\n\t\t\t\tconsole.error(new Date(), 'Worker WS error:', err);\n\t\t\t}\n\t\t\telse if (this._methodManager.getEnableDebug()) {\n\t\t\t\tconsole.log(new Date(), 'Worker WS reconnecting after main restart', message);\n\t\t\t}\n\t\t\tthis.markWorkerDisconnected(message || 'Worker websocket error.');\n\t\t\tws.close();\n\t\t});\n\t}\n\n    private async handleIncomingTask(ws: WebSocket.WebSocket, data: TaskPayload) {\n        let { taskId, messageId, method, params, userContext } = data;\n\t\tconst receivedAt = Date.now();\n\t\tconst queuedAt = data.queuedAt;\n\t\tconst dispatchedAt = data.dispatchedAt;\n\t\tconst queueWaitMs = queuedAt && dispatchedAt ? round(dispatchedAt - queuedAt) : (queuedAt ? round(receivedAt - queuedAt) : null);\n\t\tconst dispatchDelayMs = dispatchedAt ? round(receivedAt - dispatchedAt) : null;\n        this._runningTasks.push(taskId);\n        const missingMethod = !method || !this._methodManager.getMethod(method);\n        if (!taskId || !method || missingMethod) {\n            if (missingMethod) {\n                console.error(new Date(), 'No method in method manager for handleIncomingTask worker server', method);\n\t\t\t\tif (this._methodManager.getMethod('insertErrorLog')) {\n\t\t\t\t\ttry {\n\t\t\t\t\t\tawait this._methodManager.callMethod('insertErrorLog', `No Method in worker server handleIncomingTask: ${method} - ${JSON.stringify(data, null, 2)}`);\n\t\t\t\t\t}\n\t\t\t\t\tcatch (error) {\n\t\t\t\t\t\tconsole.error(new Date(), 'insertErrorLog failed in worker-server', error);\n\t\t\t\t\t}\n\t\t\t\t}\n            }\n\n            // console.log('Invalid task message received', data);\n\t\t\tthis.logWorkerEvent('workerTaskInvalid', {\n\t\t\t\ttaskId,\n\t\t\t\tmethod,\n\t\t\t\tmessageId,\n\t\t\t\tqueueWaitMs,\n\t\t\t\tdispatchDelayMs,\n\t\t\t\tuserContext,\n\t\t\t\tparams\n\t\t\t});\n            this.sendWorkerResponse(ws, {\n                type: 'taskComplete',\n                taskId,\n                messageId,\n                error: true,\n                result: 'Invalid task'\n            });\n\n            return;\n        }\n\n        let timedOut = false;\n        let timeoutHandle = setTimeout(async () => {\n            timedOut = true;\n\t\t\tconst timeoutAt = Date.now();\n\t\t\tconst runMs = round(timeoutAt - receivedAt);\n\t\t\tconst totalMs = queuedAt ? round(timeoutAt - queuedAt) : null;\n            console.error(new Date(), 'Worker timed out on task:', taskId, 'Method:', method);\n\n\t\t\tthis.logWorkerEvent('workerTaskTimeout', {\n\t\t\t\ttaskId,\n\t\t\t\tmethod,\n\t\t\t\tmessageId,\n\t\t\t\trunMs,\n\t\t\t\tqueueWaitMs,\n\t\t\t\tdispatchDelayMs,\n\t\t\t\ttotalMs,\n\t\t\t\tuserContext,\n\t\t\t\tparams\n\t\t\t});\n\n            this.sendWorkerResponse(ws, {\n                type: 'taskComplete',\n                taskId,\n                messageId,\n                error: true,\n                result: 'Task timed out'\n            });\n\n            await this._methodManager.callMethod('insertErrorLog', `Timeout in Method: ${method} - ${JSON.stringify(data, null, 2)}`);\n        }, this._methodManager.getMethod(method)?.timeoutOverride || (1000 * 60 * 2));\n\n        try {\n            let managerThis = Object.assign({}, this._methodManager, MethodManager.prototype, {\n                id_user: userContext?.id_user || '',\n                user: userContext?.user || '',\n                id_ws: userContext?.id_ws || ''\n            });\n\n            if (this._methodManager.getEnableDebug()) {\n                console.log(new Date(), 'Running method', method);\n            }\n\n\t\t\tlet result = await this._methodManager.callMethod.call(managerThis, method, ...params);\n\t\t\tlet packedResult: Uint8Array | Buffer = null;\n\t\t\tlet meta: Record<string, any> = null;\n\t\t\tlet publicationPayload: any = undefined;\n\t\t\tlet publicationMeta: Record<string, any> = null;\n\t\t\tconst isPublicationResponse = !!(result && result.__publicationResponse);\n\n\t\t\ttry {\n\t\t\t\tif (isPublicationResponse) {\n\t\t\t\t\tpublicationPayload = result.payload;\n\t\t\t\t\tpublicationMeta = {\n\t\t\t\t\t\ttype: 'publication',\n\t\t\t\t\t\tpublication: result.publication,\n\t\t\t\t\t\tsubscriptionData: result.subscriptionData,\n\t\t\t\t\t\tuserSpecific: result.userSpecific,\n\t\t\t\t\t\tsnapshot: result.snapshot\n\t\t\t\t\t};\n\t\t\t\t}\n\n\t\t\t\tif (isPublicationResponse) {\n\t\t\t\t\tmeta = publicationMeta;\n\t\t\t\t\tpackedResult = <Buffer>pack(publicationPayload);\n\t\t\t\t\tresult = null;\n\t\t\t\t}\n\t\t\t\telse {\n\t\t\t\t\tpackedResult = <Buffer>pack(result);\n\t\t\t\t}\n\t\t\t}\n\t\t\tcatch (packErr) {\n\t\t\t\tconsole.error(new Date(), 'Worker pack error', packErr);\n\t\t\t\tif (isPublicationResponse) {\n\t\t\t\t\tmeta = publicationMeta;\n\t\t\t\t\tresult = publicationPayload;\n\t\t\t\t}\n\t\t\t}\n\n\n\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\tconsole.log(new Date(), 'Finished method', method);\n\t\t\t}\n\n\t\t\tif (!timedOut) {\n\t\t\t\tclearTimeout(timeoutHandle);\n\t\t\t\tconst payload: TaskResponse = {\n\t\t\t\t\ttype: 'taskComplete',\n\t\t\t\t\ttaskId,\n\t\t\t\t\tmessageId,\n\t\t\t\t\terror: false,\n\t\t\t\t\tresult: packedResult ? null : result,\n\t\t\t\t\tpackedResult,\n\t\t\t\t\tencoding: 'msgpack',\n\t\t\t\t\tmeta\n\t\t\t\t};\n\n\t\t\t\tthis.sendWorkerResponse(ws, payload);\n\t\t\t}\n\n            this._runningTasks = this._runningTasks.filter(a => a !== taskId);\n            \n            if (this._methodManager.getEnableDebug()) {\n                console.log(new Date(), 'Done with Task', JSON.stringify(this._runningTasks, null, 2));\n            }\n        }\n        catch (err) {\n            if (!timedOut) {\n                clearTimeout(timeoutHandle);\n\t\t\t\tconst errorAt = Date.now();\n\t\t\t\tconst runMs = round(errorAt - receivedAt);\n\t\t\t\tconst totalMs = queuedAt ? round(errorAt - queuedAt) : null;\n                console.error('Worker failed task:', taskId, 'Method:', method, err);\n                err.message = 'Worker failed task:' + taskId + ' Method:' + method + ' - ' + err.message;\n\t\t\t\tthis.logWorkerEvent('workerTaskError', {\n\t\t\t\t\ttaskId,\n\t\t\t\t\tmethod,\n\t\t\t\t\tmessageId,\n\t\t\t\t\trunMs,\n\t\t\t\t\tqueueWaitMs,\n\t\t\t\t\tdispatchDelayMs,\n\t\t\t\t\ttotalMs,\n\t\t\t\t\tuserContext,\n\t\t\t\t\tparams,\n\t\t\t\t\terror: err\n\t\t\t\t});\n                this.sendWorkerResponse(ws, {\n                    type: 'taskComplete',\n                    taskId,\n                    messageId,\n                    error: true,\n                    result: err || 'Unknown error'\n                });\n                throw err;\n            }\n\n\t\t\tconst errorAt = Date.now();\n\t\t\tconst runMs = round(errorAt - receivedAt);\n\t\t\tconst totalMs = queuedAt ? round(errorAt - queuedAt) : null;\n\t\t\tthis.logWorkerEvent('workerTaskError', {\n\t\t\t\ttaskId,\n\t\t\t\tmethod,\n\t\t\t\tmessageId,\n\t\t\t\trunMs,\n\t\t\t\tqueueWaitMs,\n\t\t\t\tdispatchDelayMs,\n\t\t\t\ttotalMs,\n\t\t\t\tuserContext,\n\t\t\t\tparams,\n\t\t\t\terror: err\n\t\t\t});\n\n            console.error(new Date(),  `Error in Method: ${method} - ${JSON.stringify(data, null, 2)}`);\n            \n            await this._methodManager.callMethod('insertErrorLog', `Error in Method: ${method} - ${JSON.stringify(data, null, 2)}`);\n            \n            this._runningTasks = this._runningTasks.filter(a => a !== taskId);\n        }\n    }\n\n\tprivate logWorkerEvent(event: string, details: Record<string, any>) {\n\t\tconsole.log(this.safeStringify({\n\t\t\tts: new Date().toISOString(),\n\t\t\tevent,\n\t\t\t...details\n\t\t}));\n\t}\n\n\tprivate safeStringify(value: any): string {\n\t\ttry {\n\t\t\tconst seen = new WeakSet();\n\t\t\treturn JSON.stringify(value, (key, val) => {\n\t\t\t\tif (key === '__proto__') {\n\t\t\t\t\treturn undefined;\n\t\t\t\t}\n\t\t\t\tif (typeof val === 'bigint') {\n\t\t\t\t\treturn val.toString();\n\t\t\t\t}\n\t\t\t\tif (val instanceof Error) {\n\t\t\t\t\treturn {\n\t\t\t\t\t\tname: val.name,\n\t\t\t\t\t\tmessage: val.message,\n\t\t\t\t\t\tstack: val.stack\n\t\t\t\t\t};\n\t\t\t\t}\n\t\t\t\tif (val && typeof val === 'object') {\n\t\t\t\t\tif (seen.has(val)) {\n\t\t\t\t\t\treturn '[Circular]';\n\t\t\t\t\t}\n\t\t\t\t\tseen.add(val);\n\t\t\t\t}\n\t\t\t\treturn val;\n\t\t\t});\n\t\t}\n\t\tcatch (err) {\n\t\t\treturn JSON.stringify({\n\t\t\t\terror: 'safeStringifyFailed',\n\t\t\t\tmessage: err?.message || String(err)\n\t\t\t});\n\t\t}\n\t}\n\n\tpublic sendWorkerResponse(ws, payload: TaskResponse | string) {\n\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\tconsole.log(new Date(), 'Sending', typeof payload === 'string' ? payload : '[binary]');\n\t\t}\n\n\t\tif (ws && ws.readyState === WebSocket.OPEN) {\n\t\t\ttry {\n\t\t\t\tif (typeof payload === 'string') {\n\t\t\t\t\tws.send(payload);\n\t\t\t\t}\n\t\t\t\telse {\n\t\t\t\t\tws.send(pack(payload));\n\t\t\t\t}\n\t\t\t}\n\t\t\tcatch (err) {\n\t\t\t\tconsole.error('Failed to send worker response:', err);\n\t\t\t}\n\t\t}\n\t\telse if (ws) {\n\t\t\tws.close();\n\t\t}\n\t}\n}\n"]}