@resolveio/server-lib 22.3.1 → 22.3.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@resolveio/server-lib",
-  "version": "22.3.1",
+  "version": "22.3.3",
   "description": "",
   "scripts": {
     "package": "./build_package.sh",
@@ -60,7 +60,7 @@
     "path": "0.12.7",
     "pdf-lib": "1.17.1",
     "pm2": "7.0.1",
-    "puppeteer": "24.29.0",
+    "puppeteer": "24.43.0",
     "rus-diff": "1.1.0",
     "rxjs": "7.8.2",
     "scmp": "2.1.0",
@@ -125,7 +125,7 @@
     "path": "0.12.7",
     "pdf-lib": "1.17.1",
     "pm2": "7.0.1",
-    "puppeteer": "24.29.0",
+    "puppeteer": "24.43.0",
     "rus-diff": "1.1.0",
     "scmp": "2.1.0",
     "simpl-schema": "1.12.0",

package/server-app.d.ts

@@ -63,11 +63,13 @@ export declare class ResolveIOMainServer {
     private _standaloneNodeReaperHighMinAgeSeconds;
     private _standaloneNodeReaperHighConsecutiveScans;
     private _standaloneNodeReaperDryRun;
+    private _standaloneNodeReaperAlertWindowMs;
     private _standaloneNodeReaperTimer;
     private _standaloneNodeReaperRunning;
     private _standaloneNodeReaperKillCounts;
     private _standaloneNodeReaperSuppressedSignatures;
     private _standaloneNodeReaperResourceHits;
+    private _standaloneNodeReaperAlertTimes;
     private _serverStartTime;
     private _lastErrorMsg;
     private _debugMsgRecv;
@@ -91,6 +93,8 @@ export declare class ResolveIOMainServer {
     private startAiAssistantCodexManagerIfEnabled;
     private startStandaloneNodeReaper;
     private reapStandaloneNodeProcesses;
+    private createStandaloneNodeReaperDetails;
+    private reportStandaloneNodeReaperAction;
     private listSystemProcesses;
     private parseProcessSnapshot;
     private execFileText;
@@ -174,6 +178,7 @@ export declare class ResolveIOMainServer {
     private resolveStandaloneNodeReaperHighMinAgeSeconds;
     private resolveStandaloneNodeReaperHighConsecutiveScans;
     private resolveStandaloneNodeReaperDryRun;
+    private resolveStandaloneNodeReaperAlertWindowMs;
     private isPm2ManagedRuntime;
     private isPrimaryPm2Instance;
     private isLocalRuntime;

package/server-app.js

@@ -1,4 +1,15 @@
 "use strict";
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -178,11 +189,13 @@ var ResolveIOMainServer = /** @class */ (function () {
         this._standaloneNodeReaperHighMinAgeSeconds = 60;
         this._standaloneNodeReaperHighConsecutiveScans = 2;
         this._standaloneNodeReaperDryRun = false;
+        this._standaloneNodeReaperAlertWindowMs = 60 * 60 * 1000;
         this._standaloneNodeReaperTimer = null;
         this._standaloneNodeReaperRunning = false;
         this._standaloneNodeReaperKillCounts = new Map();
         this._standaloneNodeReaperSuppressedSignatures = new Set();
         this._standaloneNodeReaperResourceHits = new Map();
+        this._standaloneNodeReaperAlertTimes = new Map();
         this._lastErrorMsg = null;
         this._debugMsgRecv = 0;
         this._debugMsgQueue = 0;
@@ -249,6 +262,7 @@ var ResolveIOMainServer = /** @class */ (function () {
                         this._standaloneNodeReaperHighMinAgeSeconds = this.resolveStandaloneNodeReaperHighMinAgeSeconds();
                         this._standaloneNodeReaperHighConsecutiveScans = this.resolveStandaloneNodeReaperHighConsecutiveScans();
                         this._standaloneNodeReaperDryRun = this.resolveStandaloneNodeReaperDryRun();
+                        this._standaloneNodeReaperAlertWindowMs = this.resolveStandaloneNodeReaperAlertWindowMs();
                         this._dynamicAppGatewayEnabled = this.resolveDynamicAppGatewayEnabled();
                         this._socketTier = this.resolveSocketTier();
                         this._maxClientSockets = this.resolveMaxClientSockets(this._socketTier);
@@ -596,13 +610,15 @@ var ResolveIOMainServer = /** @class */ (function () {
             highRssMb: this._standaloneNodeReaperHighRssMb,
             highMinAgeSeconds: this._standaloneNodeReaperHighMinAgeSeconds,
             highConsecutiveScans: this._standaloneNodeReaperHighConsecutiveScans,
+            alertWindowMs: this._standaloneNodeReaperAlertWindowMs,
             dryRun: this._standaloneNodeReaperDryRun
         });
     };
     ResolveIOMainServer.prototype.reapStandaloneNodeProcesses = function () {
         return __awaiter(this, void 0, void 0, function () {
-            var processes, protectedPids, _loop_1, this_1, processes_1, processes_1_1, proc;
+            var processes, protectedPids, _loop_1, this_1, processes_1, processes_1_1, proc, e_1_1;
             var e_1, _a;
+            var _this = this;
             var _b;
             return __generator(this, function (_c) {
                 switch (_c.label) {
@@ -613,88 +629,165 @@ var ResolveIOMainServer = /** @class */ (function () {
                         this._standaloneNodeReaperRunning = true;
                         _c.label = 1;
                     case 1:
-                        _c.trys.push([1, , 3, 4]);
+                        _c.trys.push([1, , 11, 12]);
                         return [4 /*yield*/, this.listSystemProcesses()];
                     case 2:
                         processes = _c.sent();
                         protectedPids = this.resolvePm2ProtectedPids(processes);
                         _loop_1 = function (proc) {
-                            if (!this_1.shouldReapStandaloneNodeProcess(proc, protectedPids)) {
-                                return "continue";
-                            }
-                            var signature = this_1.resolveStandaloneNodeProcessSignature(proc);
-                            if (!this_1.canReapStandaloneNodeSignature(signature)) {
-                                this_1.logStandaloneNodeReaperSuppressed(proc, signature);
-                                return "continue";
-                            }
-                            var reason = this_1.resolveStandaloneNodeReaperReason(proc);
-                            if (this_1._standaloneNodeReaperDryRun) {
-                                this_1.recordStandaloneNodeReaperKill(signature);
-                                console.warn(new Date(), '[Standalone Node Reaper] dry-run would terminate stale node process', {
-                                    pid: proc.pid,
-                                    ppid: proc.ppid,
-                                    ageSeconds: proc.ageSeconds,
-                                    cpuPct: proc.cpuPct,
-                                    rssMb: (0, common_1.round)(proc.rssKb / 1024),
-                                    reason: reason,
-                                    command: proc.args
-                                });
-                                return "continue";
-                            }
-                            try {
-                                process.kill(proc.pid, 'SIGTERM');
-                                this_1.recordStandaloneNodeReaperKill(signature);
-                                console.warn(new Date(), '[Standalone Node Reaper] terminated stale node process', {
-                                    pid: proc.pid,
-                                    ppid: proc.ppid,
-                                    ageSeconds: proc.ageSeconds,
-                                    cpuPct: proc.cpuPct,
-                                    rssMb: (0, common_1.round)(proc.rssKb / 1024),
-                                    reason: reason,
-                                    command: proc.args
-                                });
-                                var killTimer = setTimeout(function () {
-                                    try {
-                                        process.kill(proc.pid, 0);
-                                        process.kill(proc.pid, 'SIGKILL');
-                                        console.warn(new Date(), '[Standalone Node Reaper] force killed stale node process', {
-                                            pid: proc.pid,
-                                            reason: reason,
-                                            command: proc.args
-                                        });
-                                    }
-                                    catch (_a) { }
-                                }, 10000);
-                                (_b = killTimer.unref) === null || _b === void 0 ? void 0 : _b.call(killTimer);
-                            }
-                            catch (error) {
-                                if ((error === null || error === void 0 ? void 0 : error['code']) !== 'ESRCH') {
-                                    console.error(new Date(), '[Standalone Node Reaper] kill failed', {
-                                        pid: proc.pid,
-                                        command: proc.args,
-                                        error: (error === null || error === void 0 ? void 0 : error['message']) || error
-                                    });
+                            var signature, reason, details, details, killTimer, error_6;
+                            return __generator(this, function (_d) {
+                                switch (_d.label) {
+                                    case 0:
+                                        if (!this_1.shouldReapStandaloneNodeProcess(proc, protectedPids)) {
+                                            return [2 /*return*/, "continue"];
+                                        }
+                                        signature = this_1.resolveStandaloneNodeProcessSignature(proc);
+                                        if (!this_1.canReapStandaloneNodeSignature(signature)) {
+                                            this_1.logStandaloneNodeReaperSuppressed(proc, signature);
+                                            return [2 /*return*/, "continue"];
+                                        }
+                                        reason = this_1.resolveStandaloneNodeReaperReason(proc);
+                                        if (!this_1._standaloneNodeReaperDryRun) return [3 /*break*/, 2];
+                                        this_1.recordStandaloneNodeReaperKill(signature);
+                                        details = this_1.createStandaloneNodeReaperDetails(proc, reason, 'dry-run');
+                                        console.warn(new Date(), '[Standalone Node Reaper] dry-run would terminate stale node process', details);
+                                        return [4 /*yield*/, this_1.reportStandaloneNodeReaperAction(proc, reason, 'dry-run')];
+                                    case 1:
+                                        _d.sent();
+                                        return [2 /*return*/, "continue"];
+                                    case 2:
+                                        _d.trys.push([2, 4, , 5]);
+                                        process.kill(proc.pid, 'SIGTERM');
+                                        this_1.recordStandaloneNodeReaperKill(signature);
+                                        details = this_1.createStandaloneNodeReaperDetails(proc, reason, 'terminated');
+                                        console.warn(new Date(), '[Standalone Node Reaper] terminated stale node process', details);
+                                        return [4 /*yield*/, this_1.reportStandaloneNodeReaperAction(proc, reason, 'terminated')];
+                                    case 3:
+                                        _d.sent();
+                                        killTimer = setTimeout(function () { return __awaiter(_this, void 0, void 0, function () {
+                                            var _a;
+                                            return __generator(this, function (_b) {
+                                                switch (_b.label) {
+                                                    case 0:
+                                                        _b.trys.push([0, 2, , 3]);
+                                                        process.kill(proc.pid, 0);
+                                                        process.kill(proc.pid, 'SIGKILL');
+                                                        console.warn(new Date(), '[Standalone Node Reaper] force killed stale node process', {
+                                                            pid: proc.pid,
+                                                            reason: reason,
+                                                            command: proc.args
+                                                        });
+                                                        return [4 /*yield*/, this.reportStandaloneNodeReaperAction(proc, reason, 'force-killed')];
+                                                    case 1:
+                                                        _b.sent();
+                                                        return [3 /*break*/, 3];
+                                                    case 2:
+                                                        _a = _b.sent();
+                                                        return [3 /*break*/, 3];
+                                                    case 3: return [2 /*return*/];
+                                                }
+                                            });
+                                        }); }, 10000);
+                                        (_b = killTimer.unref) === null || _b === void 0 ? void 0 : _b.call(killTimer);
+                                        return [3 /*break*/, 5];
+                                    case 4:
+                                        error_6 = _d.sent();
+                                        if ((error_6 === null || error_6 === void 0 ? void 0 : error_6['code']) !== 'ESRCH') {
+                                            console.error(new Date(), '[Standalone Node Reaper] kill failed', {
+                                                pid: proc.pid,
+                                                command: proc.args,
+                                                error: (error_6 === null || error_6 === void 0 ? void 0 : error_6['message']) || error_6
+                                            });
+                                        }
+                                        return [3 /*break*/, 5];
+                                    case 5: return [2 /*return*/];
                                 }
-                            }
+                            });
                         };
                         this_1 = this;
+                        _c.label = 3;
+                    case 3:
+                        _c.trys.push([3, 8, 9, 10]);
+                        processes_1 = __values(processes), processes_1_1 = processes_1.next();
+                        _c.label = 4;
+                    case 4:
+                        if (!!processes_1_1.done) return [3 /*break*/, 7];
+                        proc = processes_1_1.value;
+                        return [5 /*yield**/, _loop_1(proc)];
+                    case 5:
+                        _c.sent();
+                        _c.label = 6;
+                    case 6:
+                        processes_1_1 = processes_1.next();
+                        return [3 /*break*/, 4];
+                    case 7: return [3 /*break*/, 10];
+                    case 8:
+                        e_1_1 = _c.sent();
+                        e_1 = { error: e_1_1 };
+                        return [3 /*break*/, 10];
+                    case 9:
                         try {
-                            for (processes_1 = __values(processes), processes_1_1 = processes_1.next(); !processes_1_1.done; processes_1_1 = processes_1.next()) {
-                                proc = processes_1_1.value;
-                                _loop_1(proc);
-                            }
+                            if (processes_1_1 && !processes_1_1.done && (_a = processes_1.return)) _a.call(processes_1);
                         }
-                        catch (e_1_1) { e_1 = { error: e_1_1 }; }
-                        finally {
-                            try {
-                                if (processes_1_1 && !processes_1_1.done && (_a = processes_1.return)) _a.call(processes_1);
-                            }
-                            finally { if (e_1) throw e_1.error; }
+                        finally { if (e_1) throw e_1.error; }
+                        return [7 /*endfinally*/];
+                    case 10: return [3 /*break*/, 12];
+                    case 11:
+                        this._standaloneNodeReaperRunning = false;
+                        return [7 /*endfinally*/];
+                    case 12: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    ResolveIOMainServer.prototype.createStandaloneNodeReaperDetails = function (proc, reason, action) {
+        return {
+            action: action,
+            pid: proc.pid,
+            ppid: proc.ppid,
+            ageSeconds: proc.ageSeconds,
+            cpuPct: proc.cpuPct,
+            rssMb: (0, common_1.round)(proc.rssKb / 1024),
+            reason: reason,
+            command: proc.args
+        };
+    };
+    ResolveIOMainServer.prototype.reportStandaloneNodeReaperAction = function (proc, reason, action) {
+        return __awaiter(this, void 0, void 0, function () {
+            var signature, alertKey, now, lastReportedAt, error_7;
+            var _a;
+            return __generator(this, function (_b) {
+                switch (_b.label) {
+                    case 0:
+                        signature = this.resolveStandaloneNodeProcessSignature(proc);
+                        alertKey = "".concat(action, "|").concat(signature);
+                        now = Date.now();
+                        lastReportedAt = this._standaloneNodeReaperAlertTimes.get(alertKey);
+                        if (lastReportedAt && (now - lastReportedAt) < this._standaloneNodeReaperAlertWindowMs) {
+                            return [2 /*return*/];
                         }
+                        _b.label = 1;
+                    case 1:
+                        _b.trys.push([1, 3, , 4]);
+                        return [4 /*yield*/, this.reportServerError("SERVER - Standalone Node Reaper ".concat(action, " process - ").concat(((_a = resolveio_server_app_1.ResolveIOServer.getServerConfig()) === null || _a === void 0 ? void 0 : _a['CLIENT_NAME']) || resolveio_server_app_1.ResolveIOServer.getClientName()), crypto.randomUUID(), __assign(__assign({}, this.createStandaloneNodeReaperDetails(proc, reason, action)), { host: os.hostname(), workerIndex: process.env.WORKER_INDEX || null, workerInstance: process.env.NODE_APP_INSTANCE || null, dryRun: this._standaloneNodeReaperDryRun, alertWindowMs: this._standaloneNodeReaperAlertWindowMs }), {
+                                context: 'standaloneNodeReaper',
+                                action: action,
+                                reason: reason,
+                                signature: signature
+                            }, 'warning')];
+                    case 2:
+                        _b.sent();
+                        this._standaloneNodeReaperAlertTimes.set(alertKey, now);
                         return [3 /*break*/, 4];
                     case 3:
-                        this._standaloneNodeReaperRunning = false;
-                        return [7 /*endfinally*/];
+                        error_7 = _b.sent();
+                        console.error(new Date(), '[Standalone Node Reaper] alert failed', {
+                            action: action,
+                            pid: proc.pid,
+                            error: (error_7 === null || error_7 === void 0 ? void 0 : error_7['message']) || error_7
+                        });
+                        return [3 /*break*/, 4];
                     case 4: return [2 /*return*/];
                 }
             });
@@ -1164,7 +1257,7 @@ var ResolveIOMainServer = /** @class */ (function () {
             return;
         }
         this._app.use(function (req, res, next) { return __awaiter(_this, void 0, void 0, function () {
-            var appId, appDoc, allowedHosts, controlHosts, requestHost, originHeader, originHost, expectedToken, providedToken, error_6;
+            var appId, appDoc, allowedHosts, controlHosts, requestHost, originHeader, originHost, expectedToken, providedToken, error_8;
             var _a, _b, _c;
             return __generator(this, function (_d) {
                 switch (_d.label) {
@@ -1247,11 +1340,11 @@ var ResolveIOMainServer = /** @class */ (function () {
                         next();
                         return [3 /*break*/, 4];
                     case 3:
-                        error_6 = _d.sent();
-                        console.error(new Date(), '[DynamicAppGateway] middleware failure', error_6);
+                        error_8 = _d.sent();
+                        console.error(new Date(), '[DynamicAppGateway] middleware failure', error_8);
                         res.status(500).send(JSON.stringify({
                             error: true,
-                            result: (error_6 === null || error_6 === void 0 ? void 0 : error_6.message) || 'Dynamic app gateway error.'
+                            result: (error_8 === null || error_8 === void 0 ? void 0 : error_8.message) || 'Dynamic app gateway error.'
                         }));
                         return [3 /*break*/, 4];
                     case 4: return [2 /*return*/];
@@ -2077,7 +2170,7 @@ var ResolveIOMainServer = /** @class */ (function () {
                         // Do not block message handler registration on DB write; this avoids losing
                         // very-early subscription messages sent immediately after websocket open.
                         setTimeout(function () { return __awaiter(_this, void 0, void 0, function () {
-                            var error_7;
+                            var error_9;
                             return __generator(this, function (_a) {
                                 switch (_a.label) {
                                     case 0:
@@ -2087,13 +2180,13 @@ var ResolveIOMainServer = /** @class */ (function () {
                                         _a.sent();
                                         return [3 /*break*/, 3];
                                     case 2:
-                                        error_7 = _a.sent();
-                                        console.error(new Date(), 'Error creating logged-in user', ws['id_socket'], error_7);
+                                        error_9 = _a.sent();
+                                        console.error(new Date(), 'Error creating logged-in user', ws['id_socket'], error_9);
                                         this.logConnectDebug('Create logged-in user failed', {
                                             id_socket: ws['id_socket'],
                                             id_user: ws['id_user'],
                                             user: ws['user'],
-                                            error: (error_7 === null || error_7 === void 0 ? void 0 : error_7.message) || error_7
+                                            error: (error_9 === null || error_9 === void 0 ? void 0 : error_9.message) || error_9
                                         });
                                         return [3 /*break*/, 3];
                                     case 3: return [2 /*return*/];
@@ -2420,6 +2513,12 @@ var ResolveIOMainServer = /** @class */ (function () {
         var raw = (_c = (_b = (_a = process.env.RESOLVEIO_STANDALONE_NODE_REAPER_DRY_RUN) !== null && _a !== void 0 ? _a : process.env.STANDALONE_NODE_REAPER_DRY_RUN) !== null && _b !== void 0 ? _b : config === null || config === void 0 ? void 0 : config['RESOLVEIO_STANDALONE_NODE_REAPER_DRY_RUN']) !== null && _c !== void 0 ? _c : config === null || config === void 0 ? void 0 : config['STANDALONE_NODE_REAPER_DRY_RUN'];
         return this.parseDebugFlag(raw);
     };
+    ResolveIOMainServer.prototype.resolveStandaloneNodeReaperAlertWindowMs = function () {
+        var _a, _b, _c;
+        var config = resolveio_server_app_1.ResolveIOServer.getServerConfig ? resolveio_server_app_1.ResolveIOServer.getServerConfig() : null;
+        var raw = (_c = (_b = (_a = process.env.RESOLVEIO_STANDALONE_NODE_REAPER_ALERT_WINDOW_MS) !== null && _a !== void 0 ? _a : process.env.STANDALONE_NODE_REAPER_ALERT_WINDOW_MS) !== null && _b !== void 0 ? _b : config === null || config === void 0 ? void 0 : config['RESOLVEIO_STANDALONE_NODE_REAPER_ALERT_WINDOW_MS']) !== null && _c !== void 0 ? _c : config === null || config === void 0 ? void 0 : config['STANDALONE_NODE_REAPER_ALERT_WINDOW_MS'];
+        return this.parsePositiveInt(raw, this._standaloneNodeReaperAlertWindowMs);
+    };
     ResolveIOMainServer.prototype.isPm2ManagedRuntime = function () {
         return Boolean(process.env.pm_id
             || process.env.PM2_HOME
@@ -3316,7 +3415,7 @@ var ResolveIOMainServer = /** @class */ (function () {
                             });
                         }
                         setTimeout(function () { return __awaiter(_this, void 0, void 0, function () {
-                            var error_8;
+                            var error_10;
                             return __generator(this, function (_a) {
                                 switch (_a.label) {
                                     case 0:
@@ -3326,8 +3425,8 @@ var ResolveIOMainServer = /** @class */ (function () {
                                         _a.sent();
                                         return [3 /*break*/, 3];
                                     case 2:
-                                        error_8 = _a.sent();
-                                        console.error(new Date(), 'AI execution run failed:', error_8);
+                                        error_10 = _a.sent();
+                                        console.error(new Date(), 'AI execution run failed:', error_10);
                                         return [3 /*break*/, 3];
                                     case 3: return [2 /*return*/];
                                 }