@resolveio/server-lib 22.2.32 → 22.2.34

package/src/managers/worker-server.manager.ts

@@ -0,0 +1,468 @@
+import { pack, unpack } from 'msgpackr';
+import * as WebSocket from 'ws';
+import { TaskPayload, TaskResponse } from '../models/server-message.model';
+import { dateReviver, round } from '../util/common';
+import { MethodManager } from './method.manager';
+
+export class WorkerServerManager {
+    private _methodManager: MethodManager;
+    private _serverConfig;
+    private _runningTasks = [];
+	private _taskLogThresholdMs = 200;
+
+    constructor() {}
+    
+    static create(methodManager: MethodManager, serverConfig) {
+        const workerServerManager = new WorkerServerManager();
+        workerServerManager.initialize(methodManager, serverConfig);
+        return workerServerManager;
+    }
+
+    public initialize(methodManager: MethodManager, serverConfig) {
+        this._methodManager = methodManager;
+        this._serverConfig = serverConfig;
+		if (this._methodManager && typeof this._methodManager.onReady === 'function') {
+			this._methodManager.onReady(() => this.startWorkerInstance());
+		}
+		else {
+			this.startWorkerInstance();
+		}
+    }
+
+    private startWorkerInstance() {
+        const workerIndex = encodeURIComponent(String(process.env.WORKER_INDEX || ''));
+        const workerInstance = encodeURIComponent(String(process.env.NODE_APP_INSTANCE || ''));
+        const workerToken = encodeURIComponent(String(this._serverConfig['WORKER_TOKEN'] || ''));
+        let wsUrl = this._serverConfig['SERVER_URL'] + '/websocket?workerToken=' + workerToken + '&workerIndex=' + workerIndex + '&workerInstance=' + workerInstance;
+        const ws = new WebSocket(wsUrl);
+    
+        let lastComm = new Date();
+        let interval = null;
+        let missedPongs = 0;
+        const heartbeatIntervalMs = 15000;
+        const maxMissedPongs = 2;
+        const maxSilenceMs = heartbeatIntervalMs * (maxMissedPongs + 1);
+        let openTimeout = null;
+        let opened = false;
+    
+        // Set timeout if the socket never opens
+        openTimeout = setTimeout(() => {
+            if (!opened) {
+                console.error(new Date(), 'WebSocket connection timeout. Retrying...');
+                ws.terminate(); // force close if still connecting
+            }
+        }, 10000); // 10 seconds timeout
+    
+        ws.on('open', () => {
+            opened = true;
+            clearTimeout(openTimeout);
+    
+            console.log(new Date(), 'Connected to main server as worker', process.env.WORKER_INDEX, process.env.NODE_APP_INSTANCE);
+            this.sendWorkerResponse(ws, 'ping');
+    
+            interval = setInterval(() => {
+                const now = Date.now();
+                const last = lastComm ? lastComm.getTime() : 0;
+                const silenceMs = last ? now - last : maxSilenceMs + 1;
+                if (silenceMs > maxSilenceMs || missedPongs > maxMissedPongs) {
+                    ws.close();
+                    return;
+                }
+                missedPongs += 1;
+                this.sendWorkerResponse(ws, 'ping');
+            }, heartbeatIntervalMs);
+        });
+    
+		ws.on('message', async (rawData: WebSocket.RawData) => {
+			lastComm = new Date();
+			if (typeof rawData === 'string') {
+				if (this._methodManager.getEnableDebug()) {
+					console.log(new Date(), 'Message Recv', rawData);
+				}
+
+				if (rawData === 'ping') {
+					if (this._methodManager.getEnableDebug()) {
+						console.log(new Date(), 'Recv Ping, Sending Pong');
+					}
+					
+					this.sendWorkerResponse(ws, 'pong');
+				}
+				else if (rawData === 'pong') {
+					if (this._methodManager.getEnableDebug()) {
+						console.log(new Date(), 'Recv Pong');
+					}
+					missedPongs = 0;
+				}
+				else {
+					let msg: any;
+					try {
+						msg = JSON.parse(rawData, dateReviver);
+					}
+					catch (e) {
+						console.error('Worker parse error', e);
+						return;
+					}
+	
+					if (msg.type === 'task') {
+						await this.handleIncomingTask(ws, msg);
+					}
+				}
+
+				return;
+			}
+
+			let messageBuffer: Buffer;
+
+			if (Buffer.isBuffer(rawData)) {
+				messageBuffer = rawData;
+			}
+			else if (Array.isArray(rawData)) {
+				const chunks = rawData as unknown as ReadonlyArray<Uint8Array>;
+				messageBuffer = Buffer.concat(chunks);
+			}
+			else if (rawData instanceof ArrayBuffer) {
+				messageBuffer = Buffer.from(rawData);
+			}
+			else if (ArrayBuffer.isView(rawData)) {
+				const view = rawData as NodeJS.ArrayBufferView;
+				messageBuffer = Buffer.from(view.buffer, view.byteOffset, view.byteLength);
+			}
+			else {
+				messageBuffer = Buffer.from(rawData as any);
+			}
+
+			if (this._methodManager.getEnableDebug()) {
+				console.log(new Date(), 'Message Recv (binary)', messageBuffer.length);
+			}
+
+			if (messageBuffer.length === 4) {
+				let heartbeat = messageBuffer.toString('utf8');
+
+				if (heartbeat === 'ping') {
+					if (this._methodManager.getEnableDebug()) {
+						console.log(new Date(), 'Recv Ping (binary), Sending Pong');
+					}
+
+					this.sendWorkerResponse(ws, 'pong');
+					return;
+				}
+				else if (heartbeat === 'pong') {
+					if (this._methodManager.getEnableDebug()) {
+						console.log(new Date(), 'Recv Pong (binary)');
+					}
+					missedPongs = 0;
+					return;
+				}
+			}
+
+			let msg: TaskPayload;
+
+			try {
+				msg = unpack(messageBuffer);
+			}
+			catch (e) {
+				console.error('Worker binary parse error', e);
+				return;
+			}
+
+			if (msg.type === 'task') {
+				await this.handleIncomingTask(ws, msg);
+			}
+		});
+    
+        ws.on('close', () => {
+            setTimeout(() => {
+                this.startWorkerInstance();
+            }, 1000);
+    
+            if (interval) {
+                clearInterval(interval);
+            }
+            clearTimeout(openTimeout);
+        });
+    
+		ws.on('error', (err) => {
+			const message = err?.message || '';
+			const statusMatch = message.match(/Unexpected server response:\s*(\d{3})/);
+			const statusCode = statusMatch ? statusMatch[1] : null;
+			const isRestartNoise = statusCode === '502';
+			if (statusCode && !isRestartNoise) {
+				console.error(new Date(), `Worker WS error: Failure ${statusCode}`);
+			}
+			else if (!isRestartNoise) {
+				console.error(new Date(), 'Worker WS error:', err);
+			}
+			else if (this._methodManager.getEnableDebug()) {
+				console.log(new Date(), 'Worker WS reconnecting after main restart', message);
+			}
+			ws.close();
+		});
+    }
+
+    private async handleIncomingTask(ws: WebSocket.WebSocket, data: TaskPayload) {
+        let { taskId, messageId, method, params, userContext } = data;
+		const receivedAt = Date.now();
+		const queuedAt = data.queuedAt;
+		const dispatchedAt = data.dispatchedAt;
+		const queueWaitMs = queuedAt && dispatchedAt ? round(dispatchedAt - queuedAt) : (queuedAt ? round(receivedAt - queuedAt) : null);
+		const dispatchDelayMs = dispatchedAt ? round(receivedAt - dispatchedAt) : null;
+        this._runningTasks.push(taskId);
+        const missingMethod = !method || !this._methodManager.getMethod(method);
+        if (!taskId || !method || missingMethod) {
+            if (missingMethod) {
+                console.error(new Date(), 'No method in method manager for handleIncomingTask worker server', method);
+				if (this._methodManager.getMethod('insertErrorLog')) {
+					try {
+						await this._methodManager.callMethod('insertErrorLog', `No Method in worker server handleIncomingTask: ${method} - ${JSON.stringify(data, null, 2)}`);
+					}
+					catch (error) {
+						console.error(new Date(), 'insertErrorLog failed in worker-server', error);
+					}
+				}
+            }
+
+            // console.log('Invalid task message received', data);
+			this.logWorkerEvent('workerTaskInvalid', {
+				taskId,
+				method,
+				messageId,
+				queueWaitMs,
+				dispatchDelayMs,
+				userContext,
+				params
+			});
+            this.sendWorkerResponse(ws, {
+                type: 'taskComplete',
+                taskId,
+                messageId,
+                error: true,
+                result: 'Invalid task'
+            });
+
+            return;
+        }
+
+        let timedOut = false;
+        let timeoutHandle = setTimeout(async () => {
+            timedOut = true;
+			const timeoutAt = Date.now();
+			const runMs = round(timeoutAt - receivedAt);
+			const totalMs = queuedAt ? round(timeoutAt - queuedAt) : null;
+            console.error(new Date(), 'Worker timed out on task:', taskId, 'Method:', method);
+
+			this.logWorkerEvent('workerTaskTimeout', {
+				taskId,
+				method,
+				messageId,
+				runMs,
+				queueWaitMs,
+				dispatchDelayMs,
+				totalMs,
+				userContext,
+				params
+			});
+
+            this.sendWorkerResponse(ws, {
+                type: 'taskComplete',
+                taskId,
+                messageId,
+                error: true,
+                result: 'Task timed out'
+            });
+
+            await this._methodManager.callMethod('insertErrorLog', `Timeout in Method: ${method} - ${JSON.stringify(data, null, 2)}`);
+        }, this._methodManager.getMethod(method)?.timeoutOverride || (1000 * 60 * 2));
+
+        try {
+            let managerThis = Object.assign({}, this._methodManager, MethodManager.prototype, {
+                id_user: userContext?.id_user || '',
+                user: userContext?.user || '',
+                id_ws: userContext?.id_ws || ''
+            });
+
+            if (this._methodManager.getEnableDebug()) {
+                console.log(new Date(), 'Running method', method);
+            }
+
+			let result = await this._methodManager.callMethod.call(managerThis, method, ...params);
+			let packedResult: Uint8Array | Buffer = null;
+			let meta: Record<string, any> = null;
+			let publicationPayload: any = undefined;
+			let publicationMeta: Record<string, any> = null;
+			const isPublicationResponse = !!(result && result.__publicationResponse);
+
+			try {
+				if (isPublicationResponse) {
+					publicationPayload = result.payload;
+					publicationMeta = {
+						type: 'publication',
+						publication: result.publication,
+						subscriptionData: result.subscriptionData,
+						userSpecific: result.userSpecific,
+						snapshot: result.snapshot
+					};
+				}
+
+				if (isPublicationResponse) {
+					meta = publicationMeta;
+					packedResult = <Buffer>pack(publicationPayload);
+					result = null;
+				}
+				else {
+					packedResult = <Buffer>pack(result);
+				}
+			}
+			catch (packErr) {
+				console.error(new Date(), 'Worker pack error', packErr);
+				if (isPublicationResponse) {
+					meta = publicationMeta;
+					result = publicationPayload;
+				}
+			}
+
+
+			if (this._methodManager.getEnableDebug()) {
+				console.log(new Date(), 'Finished method', method);
+			}
+
+			if (!timedOut) {
+				clearTimeout(timeoutHandle);
+				const payload: TaskResponse = {
+					type: 'taskComplete',
+					taskId,
+					messageId,
+					error: false,
+					result: packedResult ? null : result,
+					packedResult,
+					encoding: 'msgpack',
+					meta
+				};
+
+				this.sendWorkerResponse(ws, payload);
+			}
+
+            this._runningTasks = this._runningTasks.filter(a => a !== taskId);
+            
+            if (this._methodManager.getEnableDebug()) {
+                console.log(new Date(), 'Done with Task', JSON.stringify(this._runningTasks, null, 2));
+            }
+        }
+        catch (err) {
+            if (!timedOut) {
+                clearTimeout(timeoutHandle);
+				const errorAt = Date.now();
+				const runMs = round(errorAt - receivedAt);
+				const totalMs = queuedAt ? round(errorAt - queuedAt) : null;
+                console.error('Worker failed task:', taskId, 'Method:', method, err);
+                err.message = 'Worker failed task:' + taskId + ' Method:' + method + ' - ' + err.message;
+				this.logWorkerEvent('workerTaskError', {
+					taskId,
+					method,
+					messageId,
+					runMs,
+					queueWaitMs,
+					dispatchDelayMs,
+					totalMs,
+					userContext,
+					params,
+					error: err
+				});
+                this.sendWorkerResponse(ws, {
+                    type: 'taskComplete',
+                    taskId,
+                    messageId,
+                    error: true,
+                    result: err || 'Unknown error'
+                });
+                throw err;
+            }
+
+			const errorAt = Date.now();
+			const runMs = round(errorAt - receivedAt);
+			const totalMs = queuedAt ? round(errorAt - queuedAt) : null;
+			this.logWorkerEvent('workerTaskError', {
+				taskId,
+				method,
+				messageId,
+				runMs,
+				queueWaitMs,
+				dispatchDelayMs,
+				totalMs,
+				userContext,
+				params,
+				error: err
+			});
+
+            console.error(new Date(),  `Error in Method: ${method} - ${JSON.stringify(data, null, 2)}`);
+            
+            await this._methodManager.callMethod('insertErrorLog', `Error in Method: ${method} - ${JSON.stringify(data, null, 2)}`);
+            
+            this._runningTasks = this._runningTasks.filter(a => a !== taskId);
+        }
+    }
+
+	private logWorkerEvent(event: string, details: Record<string, any>) {
+		console.log(this.safeStringify({
+			ts: new Date().toISOString(),
+			event,
+			...details
+		}));
+	}
+
+	private safeStringify(value: any): string {
+		try {
+			const seen = new WeakSet();
+			return JSON.stringify(value, (key, val) => {
+				if (key === '__proto__') {
+					return undefined;
+				}
+				if (typeof val === 'bigint') {
+					return val.toString();
+				}
+				if (val instanceof Error) {
+					return {
+						name: val.name,
+						message: val.message,
+						stack: val.stack
+					};
+				}
+				if (val && typeof val === 'object') {
+					if (seen.has(val)) {
+						return '[Circular]';
+					}
+					seen.add(val);
+				}
+				return val;
+			});
+		}
+		catch (err) {
+			return JSON.stringify({
+				error: 'safeStringifyFailed',
+				message: err?.message || String(err)
+			});
+		}
+	}
+
+	public sendWorkerResponse(ws, payload: TaskResponse | string) {
+		if (this._methodManager.getEnableDebug()) {
+			console.log(new Date(), 'Sending', typeof payload === 'string' ? payload : '[binary]');
+		}
+
+		if (ws && ws.readyState === WebSocket.OPEN) {
+			try {
+				if (typeof payload === 'string') {
+					ws.send(payload);
+				}
+				else {
+					ws.send(pack(payload));
+				}
+			}
+			catch (err) {
+				console.error('Failed to send worker response:', err);
+			}
+		}
+		else if (ws) {
+			ws.close();
+		}
+	}
+}