@resolveio/server-lib 20.14.33 → 20.14.35

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@resolveio/server-lib",
-  "version": "20.14.33",
+  "version": "20.14.35",
   "description": "",
 	"scripts": {
 		"package": "./build_package.sh",

package/server-app.d.ts

@@ -29,6 +29,21 @@ export declare class ResolveIOMainServer {
     private _workerServerManager;
     private _httpServerClosePromise;
     private _websocketServerClosePromise;
+    private _wsConnectDebug;
+    private _perfDebug;
+    private _perfDebugIntervalMs;
+    private _perfDebugTimer;
+    private _perfDebugLastCpu;
+    private _perfDebugLastTs;
+    private _eventLoopHistogram;
+    private _cpuProfileOnStart;
+    private _cpuProfileAuto;
+    private _cpuProfileDurationMs;
+    private _cpuProfileThresholdPct;
+    private _cpuProfileTriggerCount;
+    private _cpuProfileHighCount;
+    private _cpuProfileDir;
+    private _cpuProfileSession;
     private _serverStartTime;
     private _lastErrorMsg;
     private _debugMsgRecv;
@@ -68,6 +83,22 @@ export declare class ResolveIOMainServer {
     private decodeBufferPayload;
     private parseTextFallback;
     private looksLikeTextPayload;
+    private resolveConnectDebug;
+    private resolvePerfDebug;
+    private resolveCpuProfileOnStart;
+    private resolveCpuProfileAuto;
+    private resolveCpuProfileDurationMs;
+    private resolveCpuProfileThresholdPct;
+    private resolveCpuProfileTriggerCount;
+    private resolveCpuProfileDir;
+    private parsePositiveInt;
+    private parsePositiveFloat;
+    private startPerfDebug;
+    private startCpuProfile;
+    private stopCpuProfile;
+    private resolveWritableProfileDir;
+    private parseDebugFlag;
+    private logConnectDebug;
     private triggerClientHeartbeat;
     private shouldDeferHeartbeat;
     private handleClientMessage;

package/server-app.js

@@ -76,9 +76,14 @@ exports.ResolveIOMainServer = void 0;
 var express = require("express");
 var xmlParser = require("express-xml-bodyparser");
 var http_1 = require("http");
+var fs = require("fs");
 var jwt = require("jsonwebtoken");
 var moment = require("moment-timezone");
 var msgpackr_1 = require("msgpackr");
+var os = require("os");
+var path = require("path");
+var perf_hooks_1 = require("perf_hooks");
+var inspector = require("inspector");
 var url_1 = require("url");
 var WebSocket = require("ws");
 var log_collection_1 = require("./collections/log.collection");
@@ -109,6 +114,21 @@ var ResolveIOMainServer = /** @class */ (function () {
         this._clientRoutes = [];
         this._httpServerClosePromise = null;
         this._websocketServerClosePromise = null;
+        this._wsConnectDebug = false;
+        this._perfDebug = false;
+        this._perfDebugIntervalMs = 2000;
+        this._perfDebugTimer = null;
+        this._perfDebugLastCpu = null;
+        this._perfDebugLastTs = 0;
+        this._eventLoopHistogram = null;
+        this._cpuProfileOnStart = false;
+        this._cpuProfileAuto = false;
+        this._cpuProfileDurationMs = 15000;
+        this._cpuProfileThresholdPct = 90;
+        this._cpuProfileTriggerCount = 3;
+        this._cpuProfileHighCount = 0;
+        this._cpuProfileDir = null;
+        this._cpuProfileSession = null;
         this._lastErrorMsg = null;
         this._debugMsgRecv = 0;
         this._debugMsgQueue = 0;
@@ -143,11 +163,23 @@ var ResolveIOMainServer = /** @class */ (function () {
                     case 0:
                         this._serverStartTime = new Date();
                         this._lastErrorMsg = null;
+                        this._wsConnectDebug = this.resolveConnectDebug();
+                        this._perfDebug = this.resolvePerfDebug();
+                        this._cpuProfileOnStart = this.resolveCpuProfileOnStart();
+                        this._cpuProfileAuto = this.resolveCpuProfileAuto();
+                        this._cpuProfileDurationMs = this.resolveCpuProfileDurationMs();
+                        this._cpuProfileThresholdPct = this.resolveCpuProfileThresholdPct();
+                        this._cpuProfileTriggerCount = this.resolveCpuProfileTriggerCount();
+                        this._cpuProfileDir = this.resolveCpuProfileDir();
                         _a = this;
                         return [4 /*yield*/, monitor_manager_1.MonitorManager.create()];
                     case 1:
                         _a._monitorManager = _b.sent();
                         this._monitorManagerFunction = new monitor_manager_1.MonitorManagerFunction();
+                        this.startPerfDebug();
+                        if (this._cpuProfileOnStart) {
+                            this.startCpuProfile('on-start');
+                        }
                         // Check for workers and decide what to start
                         this._isWorkersEnabled = process.env.IS_WORKERS_ENABLED === 'true';
                         this._isWorkerInstance = process.env.IS_WORKER_INSTANCE === 'true';
@@ -762,8 +794,9 @@ var ResolveIOMainServer = /** @class */ (function () {
         this._serverWSS.on('connection', function (ws, req) { return __awaiter(_this, void 0, void 0, function () {
             var workerId_1, workerIndex, workerInstance, rootUrl, requestUrl, workerIndexForLog, workerInstanceForLog, interval_1, lastComm_1, missedPongs_1, heartbeatIntervalMs, maxMissedPongs_1, maxSilenceMs_1;
             var _this = this;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
+            var _a, _b;
+            return __generator(this, function (_c) {
+                switch (_c.label) {
                     case 0:
                         if (!(req.url && req.url.includes('workerToken='))) return [3 /*break*/, 1];
                         workerId_1 = (0, common_1.objectIdHexString)();
@@ -778,7 +811,7 @@ var ResolveIOMainServer = /** @class */ (function () {
                                 workerIndex = requestUrl.searchParams.get('workerIndex');
                                 workerInstance = requestUrl.searchParams.get('workerInstance');
                             }
-                            catch (_b) {
+                            catch (_d) {
                                 workerIndex = null;
                                 workerInstance = null;
                             }
@@ -885,9 +918,17 @@ var ResolveIOMainServer = /** @class */ (function () {
                         ws['user_readonly'] = req['user_readonly'];
                         ws['doc_user'] = req['doc_user'];
                         this._websocketManager.addWebSocket(ws);
+                        this.logConnectDebug('WS client connected', {
+                            id_socket: ws['id_socket'],
+                            id_user: ws['id_user'],
+                            user: ws['user'],
+                            url: req === null || req === void 0 ? void 0 : req.url,
+                            ip: (_a = req === null || req === void 0 ? void 0 : req.socket) === null || _a === void 0 ? void 0 : _a.remoteAddress,
+                            origin: (_b = req === null || req === void 0 ? void 0 : req.headers) === null || _b === void 0 ? void 0 : _b.origin
+                        });
                         return [4 /*yield*/, this._subscriptionManager.createLoggedInUser(ws['id_socket'])];
                     case 2:
-                        _a.sent();
+                        _c.sent();
                         setTimeout(function () { return __awaiter(_this, void 0, void 0, function () {
                             return __generator(this, function (_a) {
                                 switch (_a.label) {
@@ -994,14 +1035,20 @@ var ResolveIOMainServer = /** @class */ (function () {
                             .on('close', function () { return __awaiter(_this, void 0, void 0, function () {
                             return __generator(this, function (_a) {
                                 switch (_a.label) {
-                                    case 0: return [4 /*yield*/, this.unsubscribeWS(ws)];
+                                    case 0:
+                                        this.logConnectDebug('WS client closed', {
+                                            id_socket: ws['id_socket'],
+                                            id_user: ws['id_user'],
+                                            user: ws['user']
+                                        });
+                                        return [4 /*yield*/, this.unsubscribeWS(ws)];
                                     case 1:
                                         _a.sent();
                                         return [2 /*return*/];
                                 }
                             });
                         }); });
-                        _a.label = 3;
+                        _c.label = 3;
                     case 3: return [2 /*return*/];
                 }
             });
@@ -1087,6 +1134,11 @@ var ResolveIOMainServer = /** @class */ (function () {
                         // If the top level is not an array, let's skip
                         if (!Array.isArray(socketData[0])) {
                             console.log('Invalid message format (expected array of arrays)', socketData);
+                            this.logConnectDebug('Invalid message format', {
+                                id_socket: ws ? ws['id_socket'] : null,
+                                user: ws ? ws['user'] : null,
+                                preview: Array.isArray(socketData) ? socketData.slice(0, 3) : socketData
+                            });
                             return [2 /*return*/];
                         }
                         _b.label = 1;
@@ -1164,6 +1216,221 @@ var ResolveIOMainServer = /** @class */ (function () {
         var first = trimmed[0];
         return first === '[' || first === '{' || first === '"' || first === 'p' || first === 'P';
     };
+    ResolveIOMainServer.prototype.resolveConnectDebug = function () {
+        var _a, _b, _c;
+        var config = resolveio_server_app_1.ResolveIOServer.getServerConfig ? resolveio_server_app_1.ResolveIOServer.getServerConfig() : null;
+        var raw = (_c = (_b = (_a = process.env.WS_CONNECT_DEBUG) !== null && _a !== void 0 ? _a : process.env.CONNECT_DEBUG) !== null && _b !== void 0 ? _b : config === null || config === void 0 ? void 0 : config['WS_CONNECT_DEBUG']) !== null && _c !== void 0 ? _c : config === null || config === void 0 ? void 0 : config['CONNECT_DEBUG'];
+        return this.parseDebugFlag(raw);
+    };
+    ResolveIOMainServer.prototype.resolvePerfDebug = function () {
+        var _a, _b, _c;
+        var config = resolveio_server_app_1.ResolveIOServer.getServerConfig ? resolveio_server_app_1.ResolveIOServer.getServerConfig() : null;
+        var raw = (_c = (_b = (_a = process.env.PERF_DEBUG) !== null && _a !== void 0 ? _a : process.env.CPU_DEBUG) !== null && _b !== void 0 ? _b : config === null || config === void 0 ? void 0 : config['PERF_DEBUG']) !== null && _c !== void 0 ? _c : config === null || config === void 0 ? void 0 : config['CPU_DEBUG'];
+        return this.parseDebugFlag(raw);
+    };
+    ResolveIOMainServer.prototype.resolveCpuProfileOnStart = function () {
+        var _a, _b, _c;
+        var config = resolveio_server_app_1.ResolveIOServer.getServerConfig ? resolveio_server_app_1.ResolveIOServer.getServerConfig() : null;
+        var raw = (_c = (_b = (_a = process.env.CPU_PROFILE_ON_START) !== null && _a !== void 0 ? _a : process.env.CPU_PROFILE_START) !== null && _b !== void 0 ? _b : config === null || config === void 0 ? void 0 : config['CPU_PROFILE_ON_START']) !== null && _c !== void 0 ? _c : config === null || config === void 0 ? void 0 : config['CPU_PROFILE_START'];
+        return this.parseDebugFlag(raw);
+    };
+    ResolveIOMainServer.prototype.resolveCpuProfileAuto = function () {
+        var _a;
+        var config = resolveio_server_app_1.ResolveIOServer.getServerConfig ? resolveio_server_app_1.ResolveIOServer.getServerConfig() : null;
+        var raw = (_a = process.env.CPU_PROFILE_AUTO) !== null && _a !== void 0 ? _a : config === null || config === void 0 ? void 0 : config['CPU_PROFILE_AUTO'];
+        return this.parseDebugFlag(raw);
+    };
+    ResolveIOMainServer.prototype.resolveCpuProfileDurationMs = function () {
+        var _a, _b, _c;
+        var config = resolveio_server_app_1.ResolveIOServer.getServerConfig ? resolveio_server_app_1.ResolveIOServer.getServerConfig() : null;
+        var raw = (_c = (_b = (_a = process.env.CPU_PROFILE_DURATION_MS) !== null && _a !== void 0 ? _a : process.env.PERF_PROFILE_DURATION_MS) !== null && _b !== void 0 ? _b : config === null || config === void 0 ? void 0 : config['CPU_PROFILE_DURATION_MS']) !== null && _c !== void 0 ? _c : config === null || config === void 0 ? void 0 : config['PERF_PROFILE_DURATION_MS'];
+        return this.parsePositiveInt(raw, this._cpuProfileDurationMs);
+    };
+    ResolveIOMainServer.prototype.resolveCpuProfileThresholdPct = function () {
+        var _a, _b, _c;
+        var config = resolveio_server_app_1.ResolveIOServer.getServerConfig ? resolveio_server_app_1.ResolveIOServer.getServerConfig() : null;
+        var raw = (_c = (_b = (_a = process.env.CPU_PROFILE_THRESHOLD_PCT) !== null && _a !== void 0 ? _a : process.env.PERF_PROFILE_THRESHOLD_PCT) !== null && _b !== void 0 ? _b : config === null || config === void 0 ? void 0 : config['CPU_PROFILE_THRESHOLD_PCT']) !== null && _c !== void 0 ? _c : config === null || config === void 0 ? void 0 : config['PERF_PROFILE_THRESHOLD_PCT'];
+        return this.parsePositiveFloat(raw, this._cpuProfileThresholdPct);
+    };
+    ResolveIOMainServer.prototype.resolveCpuProfileTriggerCount = function () {
+        var _a;
+        var config = resolveio_server_app_1.ResolveIOServer.getServerConfig ? resolveio_server_app_1.ResolveIOServer.getServerConfig() : null;
+        var raw = (_a = process.env.CPU_PROFILE_TRIGGER_COUNT) !== null && _a !== void 0 ? _a : config === null || config === void 0 ? void 0 : config['CPU_PROFILE_TRIGGER_COUNT'];
+        return this.parsePositiveInt(raw, this._cpuProfileTriggerCount);
+    };
+    ResolveIOMainServer.prototype.resolveCpuProfileDir = function () {
+        var _a;
+        var config = resolveio_server_app_1.ResolveIOServer.getServerConfig ? resolveio_server_app_1.ResolveIOServer.getServerConfig() : null;
+        var raw = (_a = process.env.CPU_PROFILE_DIR) !== null && _a !== void 0 ? _a : config === null || config === void 0 ? void 0 : config['CPU_PROFILE_DIR'];
+        return typeof raw === 'string' && raw.trim() ? raw.trim() : null;
+    };
+    ResolveIOMainServer.prototype.parsePositiveInt = function (value, fallback) {
+        var parsed = parseInt(value !== null && value !== void 0 ? value : '', 10);
+        if (Number.isNaN(parsed) || parsed <= 0) {
+            return fallback;
+        }
+        return parsed;
+    };
+    ResolveIOMainServer.prototype.parsePositiveFloat = function (value, fallback) {
+        var parsed = parseFloat(value !== null && value !== void 0 ? value : '');
+        if (Number.isNaN(parsed) || parsed <= 0) {
+            return fallback;
+        }
+        return parsed;
+    };
+    ResolveIOMainServer.prototype.startPerfDebug = function () {
+        var _this = this;
+        var _a;
+        if (!this._perfDebug || this._perfDebugTimer) {
+            return;
+        }
+        this._perfDebugIntervalMs = this.parsePositiveInt((_a = process.env.PERF_DEBUG_INTERVAL_MS) !== null && _a !== void 0 ? _a : process.env.CPU_DEBUG_INTERVAL_MS, this._perfDebugIntervalMs);
+        this._perfDebugLastCpu = process.cpuUsage();
+        this._perfDebugLastTs = Date.now();
+        this._eventLoopHistogram = (0, perf_hooks_1.monitorEventLoopDelay)({ resolution: 20 });
+        this._eventLoopHistogram.enable();
+        this._perfDebugTimer = setInterval(function () {
+            var now = Date.now();
+            var elapsedMs = now - _this._perfDebugLastTs;
+            var cpuDiff = _this._perfDebugLastCpu ? process.cpuUsage(_this._perfDebugLastCpu) : process.cpuUsage();
+            var cpuMs = (cpuDiff.user + cpuDiff.system) / 1000;
+            var cpuPct = elapsedMs > 0 ? (cpuMs / elapsedMs) * 100 : 0;
+            var mem = process.memoryUsage();
+            var heapUsedMb = (0, common_1.round)((mem.heapUsed / 1024 / 1024) * 10) / 10;
+            var rssMb = (0, common_1.round)((mem.rss / 1024 / 1024) * 10) / 10;
+            var handles = typeof process._getActiveHandles === 'function'
+                ? process._getActiveHandles().length
+                : null;
+            var requests = typeof process._getActiveRequests === 'function'
+                ? process._getActiveRequests().length
+                : null;
+            var histogram = _this._eventLoopHistogram;
+            var eventLoop = histogram ? {
+                meanMs: (0, common_1.round)((histogram.mean / 1e6) * 100) / 100,
+                p50Ms: (0, common_1.round)((histogram.percentile(50) / 1e6) * 100) / 100,
+                p95Ms: (0, common_1.round)((histogram.percentile(95) / 1e6) * 100) / 100,
+                p99Ms: (0, common_1.round)((histogram.percentile(99) / 1e6) * 100) / 100,
+                maxMs: (0, common_1.round)((histogram.max / 1e6) * 100) / 100
+            } : null;
+            histogram === null || histogram === void 0 ? void 0 : histogram.reset();
+            console.log(new Date(), '[Perf Debug]', JSON.stringify({
+                cpuPct: (0, common_1.round)(cpuPct * 10) / 10,
+                cpuMs: (0, common_1.round)(cpuMs),
+                elapsedMs: elapsedMs,
+                eventLoop: eventLoop,
+                heapUsedMb: heapUsedMb,
+                rssMb: rssMb,
+                activeHandles: handles,
+                activeRequests: requests,
+                msgRecv: _this._debugMsgRecv,
+                msgQueue: _this._debugMsgQueue
+            }));
+            if (_this._cpuProfileAuto) {
+                if (cpuPct >= _this._cpuProfileThresholdPct) {
+                    _this._cpuProfileHighCount += 1;
+                }
+                else {
+                    _this._cpuProfileHighCount = 0;
+                }
+                if (_this._cpuProfileHighCount >= _this._cpuProfileTriggerCount) {
+                    _this._cpuProfileHighCount = 0;
+                    _this.startCpuProfile('auto-high-cpu');
+                }
+            }
+            _this._perfDebugLastCpu = process.cpuUsage();
+            _this._perfDebugLastTs = now;
+        }, this._perfDebugIntervalMs);
+    };
+    ResolveIOMainServer.prototype.startCpuProfile = function (trigger) {
+        var _this = this;
+        if (this._cpuProfileSession) {
+            return;
+        }
+        try {
+            var session_1 = new inspector.Session();
+            session_1.connect();
+            session_1.post('Profiler.enable', function () {
+                session_1.post('Profiler.start', function () {
+                    _this._cpuProfileSession = session_1;
+                    console.log(new Date(), '[Perf Debug]', 'CPU profile started', trigger);
+                    setTimeout(function () {
+                        _this.stopCpuProfile(trigger);
+                    }, _this._cpuProfileDurationMs);
+                });
+            });
+        }
+        catch (error) {
+            console.error(new Date(), '[Perf Debug]', 'CPU profile start failed', error);
+        }
+    };
+    ResolveIOMainServer.prototype.stopCpuProfile = function (trigger) {
+        var _this = this;
+        var session = this._cpuProfileSession;
+        if (!session) {
+            return;
+        }
+        session.post('Profiler.stop', function (err, res) {
+            var _a;
+            if (err) {
+                console.error(new Date(), '[Perf Debug]', 'CPU profile stop failed', err);
+            }
+            else {
+                try {
+                    var dir = _this.resolveWritableProfileDir();
+                    var filename = "resolveio-cpuprofile-".concat(process.pid, "-").concat(Date.now(), ".cpuprofile");
+                    var filePath = path.join(dir, filename);
+                    fs.writeFileSync(filePath, JSON.stringify((_a = res === null || res === void 0 ? void 0 : res.profile) !== null && _a !== void 0 ? _a : {}));
+                    console.log(new Date(), '[Perf Debug]', 'CPU profile saved', filePath, trigger);
+                }
+                catch (writeErr) {
+                    console.error(new Date(), '[Perf Debug]', 'CPU profile write failed', writeErr);
+                }
+            }
+            try {
+                session.disconnect();
+            }
+            catch (_b) { }
+            _this._cpuProfileSession = null;
+        });
+    };
+    ResolveIOMainServer.prototype.resolveWritableProfileDir = function () {
+        var preferred = this._cpuProfileDir;
+        if (preferred) {
+            try {
+                fs.mkdirSync(preferred, { recursive: true });
+                return preferred;
+            }
+            catch (_a) { }
+        }
+        return os.tmpdir();
+    };
+    ResolveIOMainServer.prototype.parseDebugFlag = function (value) {
+        if (value === true) {
+            return true;
+        }
+        if (value === false || value === null || value === undefined) {
+            return false;
+        }
+        if (typeof value === 'number') {
+            return value === 1;
+        }
+        if (typeof value === 'string') {
+            var normalized = value.trim().toLowerCase();
+            return ['1', 'true', 'yes', 'y', 'on'].includes(normalized);
+        }
+        return false;
+    };
+    ResolveIOMainServer.prototype.logConnectDebug = function (message, details) {
+        if (!this._wsConnectDebug) {
+            return;
+        }
+        if (details) {
+            console.log(new Date(), '[Connect Debug]', message, JSON.stringify(details));
+        }
+        else {
+            console.log(new Date(), '[Connect Debug]', message);
+        }
+    };
     ResolveIOMainServer.prototype.triggerClientHeartbeat = function (ws) {
         return __awaiter(this, void 0, void 0, function () {
             var err_1;
@@ -1240,6 +1507,15 @@ var ResolveIOMainServer = /** @class */ (function () {
                         if (!(type === 'subscription')) return [3 /*break*/, 4];
                         subType = msg[4];
                         pub = msg[5];
+                        this.logConnectDebug('Subscription message', {
+                            subType: subType,
+                            publication: pub,
+                            messageId: messageId,
+                            messageRoute: messageRoute_1,
+                            args: Math.max(0, msg.length - 6),
+                            id_socket: ws ? ws['id_socket'] : null,
+                            user: ws ? ws['user'] : null
+                        });
                         if (!(subType === 'sub')) return [3 /*break*/, 2];
                         return [4 /*yield*/, this._subscriptionManager.subscribe(messageRoute_1, messageDate, ws, messageId, pub, msg.slice(6))];
                     case 1:
@@ -1533,6 +1809,11 @@ var ResolveIOMainServer = /** @class */ (function () {
                         if (this._subscriptionManager && this._methodManager.getEnableDebug()) {
                             console.log(new Date(), 'Server App', 'Unsub WS', ws['user'], ws['id_socket']);
                         }
+                        this.logConnectDebug('WS unsubscribe', {
+                            id_socket: ws ? ws['id_socket'] : null,
+                            id_user: ws ? ws['id_user'] : null,
+                            user: ws ? ws['user'] : null
+                        });
                         return [4 /*yield*/, this._subscriptionManager.unsubscribeAll(ws)];
                     case 1:
                         _a.sent();