@resolveio/server-lib 22.3.220 → 22.3.221

package/src/managers/worker-server.manager.ts

@@ -0,0 +1,536 @@
+import { pack, unpack } from 'msgpackr';
+import * as WebSocket from 'ws';
+import { TaskPayload, TaskResponse } from '../models/server-message.model';
+import { dateReviver, round } from '../util/common';
+import { MethodManager } from './method.manager';
+
+export interface WorkerServerConnectionHealth {
+	isConnected: boolean;
+	lastConnectionAttemptAt: number;
+	lastConnectedAt: number;
+	lastDisconnectedAt: number;
+	connectionLossStartedAt: number;
+	currentOutageMs: number;
+	lastDisconnectReason: string;
+}
+
+export class WorkerServerManager {
+	private _methodManager: MethodManager;
+	private _serverConfig;
+	private _runningTasks = [];
+	private _taskLogThresholdMs = 200;
+	private _isConnected = false;
+	private _lastConnectionAttemptAt = 0;
+	private _lastConnectedAt = 0;
+	private _lastDisconnectedAt = 0;
+	private _connectionLossStartedAt = 0;
+	private _lastDisconnectReason = '';
+
+	constructor() {}
+	
+	static create(methodManager: MethodManager, serverConfig) {
+		const workerServerManager = new WorkerServerManager();
+		workerServerManager.initialize(methodManager, serverConfig);
+		return workerServerManager;
+	}
+
+	public initialize(methodManager: MethodManager, serverConfig) {
+		this._methodManager = methodManager;
+		this._serverConfig = serverConfig;
+		if (this._methodManager && typeof this._methodManager.onReady === 'function') {
+			this._methodManager.onReady(() => this.startWorkerInstance());
+		}
+		else {
+			this.startWorkerInstance();
+		}
+	}
+
+	private markWorkerConnectionAttempt() {
+		const now = Date.now();
+		this._lastConnectionAttemptAt = now;
+		if (!this._isConnected && !this._connectionLossStartedAt) {
+			this._connectionLossStartedAt = now;
+		}
+	}
+
+	private markWorkerConnected() {
+		const now = Date.now();
+		this._isConnected = true;
+		this._lastConnectedAt = now;
+		this._lastDisconnectedAt = 0;
+		this._connectionLossStartedAt = 0;
+		this._lastDisconnectReason = '';
+	}
+
+	private markWorkerDisconnected(reason: string) {
+		const now = Date.now();
+		this._isConnected = false;
+		this._lastDisconnectedAt = now;
+		if (!this._connectionLossStartedAt) {
+			this._connectionLossStartedAt = now;
+		}
+		this._lastDisconnectReason = String(reason || 'Worker websocket disconnected.');
+	}
+
+	public getConnectionHealth(): WorkerServerConnectionHealth {
+		return {
+			isConnected: this._isConnected,
+			lastConnectionAttemptAt: this._lastConnectionAttemptAt,
+			lastConnectedAt: this._lastConnectedAt,
+			lastDisconnectedAt: this._lastDisconnectedAt,
+			connectionLossStartedAt: this._connectionLossStartedAt,
+			currentOutageMs: (!this._isConnected && this._connectionLossStartedAt)
+				? Math.max(0, Date.now() - this._connectionLossStartedAt)
+				: 0,
+			lastDisconnectReason: this._lastDisconnectReason
+		};
+	}
+
+	private startWorkerInstance() {
+		const workerIndex = encodeURIComponent(String(process.env.WORKER_INDEX || ''));
+		const workerInstance = encodeURIComponent(String(process.env.NODE_APP_INSTANCE || ''));
+		const workerToken = encodeURIComponent(String(this._serverConfig['WORKER_TOKEN'] || ''));
+		let wsUrl = this._serverConfig['SERVER_URL'] + '/websocket?workerToken=' + workerToken + '&workerIndex=' + workerIndex + '&workerInstance=' + workerInstance;
+		const ws = new WebSocket(wsUrl);
+
+		this.markWorkerConnectionAttempt();
+
+		let lastComm = new Date();
+		let interval = null;
+		let missedPongs = 0;
+		const heartbeatIntervalMs = 15000;
+		const maxMissedPongs = 2;
+		const maxSilenceMs = heartbeatIntervalMs * (maxMissedPongs + 1);
+		let openTimeout = null;
+		let opened = false;
+
+		// Set timeout if the socket never opens
+		openTimeout = setTimeout(() => {
+			if (!opened) {
+				this.markWorkerDisconnected('WebSocket connection timeout.');
+				console.error(new Date(), 'WebSocket connection timeout. Retrying...');
+				ws.terminate(); // force close if still connecting
+			}
+		}, 10000); // 10 seconds timeout
+
+		ws.on('open', () => {
+			opened = true;
+			clearTimeout(openTimeout);
+			this.markWorkerConnected();
+
+			console.log(new Date(), 'Connected to main server as worker', process.env.WORKER_INDEX, process.env.NODE_APP_INSTANCE);
+			this.sendWorkerResponse(ws, 'ping');
+
+			interval = setInterval(() => {
+				const now = Date.now();
+				const last = lastComm ? lastComm.getTime() : 0;
+				const silenceMs = last ? now - last : maxSilenceMs + 1;
+				if (silenceMs > maxSilenceMs || missedPongs > maxMissedPongs) {
+					this.markWorkerDisconnected('Worker websocket heartbeat timed out.');
+					ws.close();
+					return;
+				}
+				missedPongs += 1;
+				this.sendWorkerResponse(ws, 'ping');
+			}, heartbeatIntervalMs);
+		});
+
+		ws.on('message', async (rawData: WebSocket.RawData) => {
+			lastComm = new Date();
+			if (typeof rawData === 'string') {
+				if (this._methodManager.getEnableDebug()) {
+					console.log(new Date(), 'Message Recv', rawData);
+				}
+
+				if (rawData === 'ping') {
+					if (this._methodManager.getEnableDebug()) {
+						console.log(new Date(), 'Recv Ping, Sending Pong');
+					}
+					
+					this.sendWorkerResponse(ws, 'pong');
+				}
+				else if (rawData === 'pong') {
+					if (this._methodManager.getEnableDebug()) {
+						console.log(new Date(), 'Recv Pong');
+					}
+					missedPongs = 0;
+				}
+				else {
+					let msg: any;
+					try {
+						msg = JSON.parse(rawData, dateReviver);
+					}
+					catch (e) {
+						console.error('Worker parse error', e);
+						return;
+					}
+	
+					if (msg.type === 'task') {
+						await this.handleIncomingTask(ws, msg);
+					}
+				}
+
+				return;
+			}
+
+			let messageBuffer: Buffer;
+
+			if (Buffer.isBuffer(rawData)) {
+				messageBuffer = rawData;
+			}
+			else if (Array.isArray(rawData)) {
+				const chunks = rawData as unknown as ReadonlyArray<Uint8Array>;
+				messageBuffer = Buffer.concat(chunks);
+			}
+			else if (rawData instanceof ArrayBuffer) {
+				messageBuffer = Buffer.from(rawData);
+			}
+			else if (ArrayBuffer.isView(rawData)) {
+				const view = rawData as NodeJS.ArrayBufferView;
+				messageBuffer = Buffer.from(view.buffer, view.byteOffset, view.byteLength);
+			}
+			else {
+				messageBuffer = Buffer.from(rawData as any);
+			}
+
+			if (this._methodManager.getEnableDebug()) {
+				console.log(new Date(), 'Message Recv (binary)', messageBuffer.length);
+			}
+
+			if (messageBuffer.length === 4) {
+				let heartbeat = messageBuffer.toString('utf8');
+
+				if (heartbeat === 'ping') {
+					if (this._methodManager.getEnableDebug()) {
+						console.log(new Date(), 'Recv Ping (binary), Sending Pong');
+					}
+
+					this.sendWorkerResponse(ws, 'pong');
+					return;
+				}
+				else if (heartbeat === 'pong') {
+					if (this._methodManager.getEnableDebug()) {
+						console.log(new Date(), 'Recv Pong (binary)');
+					}
+					missedPongs = 0;
+					return;
+				}
+			}
+
+			let msg: TaskPayload;
+
+			try {
+				msg = unpack(messageBuffer);
+			}
+			catch (e) {
+				console.error('Worker binary parse error', e);
+				return;
+			}
+
+			if (msg.type === 'task') {
+				await this.handleIncomingTask(ws, msg);
+			}
+		});
+    
+		ws.on('close', (code, reason) => {
+			const closeReason = reason && reason.length
+				? `Worker websocket closed (${code}): ${reason.toString()}`
+				: `Worker websocket closed (${code})`;
+			this.markWorkerDisconnected(closeReason);
+
+			setTimeout(() => {
+				this.startWorkerInstance();
+			}, 1000);
+
+			if (interval) {
+				clearInterval(interval);
+			}
+			clearTimeout(openTimeout);
+		});
+
+		ws.on('error', (err) => {
+			const message = err?.message || '';
+			const statusMatch = message.match(/Unexpected server response:\s*(\d{3})/);
+			const statusCode = statusMatch ? statusMatch[1] : null;
+			const isRestartNoise = statusCode === '502';
+			if (statusCode && !isRestartNoise) {
+				console.error(new Date(), `Worker WS error: Failure ${statusCode}`);
+			}
+			else if (!isRestartNoise) {
+				console.error(new Date(), 'Worker WS error:', err);
+			}
+			else if (this._methodManager.getEnableDebug()) {
+				console.log(new Date(), 'Worker WS reconnecting after main restart', message);
+			}
+			this.markWorkerDisconnected(message || 'Worker websocket error.');
+			ws.close();
+		});
+	}
+
+    private async handleIncomingTask(ws: WebSocket.WebSocket, data: TaskPayload) {
+        let { taskId, messageId, method, params, userContext } = data;
+		const receivedAt = Date.now();
+		const queuedAt = data.queuedAt;
+		const dispatchedAt = data.dispatchedAt;
+		const queueWaitMs = queuedAt && dispatchedAt ? round(dispatchedAt - queuedAt) : (queuedAt ? round(receivedAt - queuedAt) : null);
+		const dispatchDelayMs = dispatchedAt ? round(receivedAt - dispatchedAt) : null;
+        this._runningTasks.push(taskId);
+        const missingMethod = !method || !this._methodManager.getMethod(method);
+        if (!taskId || !method || missingMethod) {
+            if (missingMethod) {
+                console.error(new Date(), 'No method in method manager for handleIncomingTask worker server', method);
+				if (this._methodManager.getMethod('insertErrorLog')) {
+					try {
+						await this._methodManager.callMethod('insertErrorLog', `No Method in worker server handleIncomingTask: ${method} - ${JSON.stringify(data, null, 2)}`);
+					}
+					catch (error) {
+						console.error(new Date(), 'insertErrorLog failed in worker-server', error);
+					}
+				}
+            }
+
+            // console.log('Invalid task message received', data);
+			this.logWorkerEvent('workerTaskInvalid', {
+				taskId,
+				method,
+				messageId,
+				queueWaitMs,
+				dispatchDelayMs,
+				userContext,
+				params
+			});
+            this.sendWorkerResponse(ws, {
+                type: 'taskComplete',
+                taskId,
+                messageId,
+                error: true,
+                result: 'Invalid task'
+            });
+
+            return;
+        }
+
+        let timedOut = false;
+        let timeoutHandle = setTimeout(async () => {
+            timedOut = true;
+			const timeoutAt = Date.now();
+			const runMs = round(timeoutAt - receivedAt);
+			const totalMs = queuedAt ? round(timeoutAt - queuedAt) : null;
+            console.error(new Date(), 'Worker timed out on task:', taskId, 'Method:', method);
+
+			this.logWorkerEvent('workerTaskTimeout', {
+				taskId,
+				method,
+				messageId,
+				runMs,
+				queueWaitMs,
+				dispatchDelayMs,
+				totalMs,
+				userContext,
+				params
+			});
+
+            this.sendWorkerResponse(ws, {
+                type: 'taskComplete',
+                taskId,
+                messageId,
+                error: true,
+                result: 'Task timed out'
+            });
+
+            await this._methodManager.callMethod('insertErrorLog', `Timeout in Method: ${method} - ${JSON.stringify(data, null, 2)}`);
+        }, this._methodManager.getMethod(method)?.timeoutOverride || (1000 * 60 * 2));
+
+        try {
+            let managerThis = Object.assign({}, this._methodManager, MethodManager.prototype, {
+                id_user: userContext?.id_user || '',
+                user: userContext?.user || '',
+                id_ws: userContext?.id_ws || ''
+            });
+
+            if (this._methodManager.getEnableDebug()) {
+                console.log(new Date(), 'Running method', method);
+            }
+
+			let result = await this._methodManager.callMethod.call(managerThis, method, ...params);
+			let packedResult: Uint8Array | Buffer = null;
+			let meta: Record<string, any> = null;
+			let publicationPayload: any = undefined;
+			let publicationMeta: Record<string, any> = null;
+			const isPublicationResponse = !!(result && result.__publicationResponse);
+
+			try {
+				if (isPublicationResponse) {
+					publicationPayload = result.payload;
+					publicationMeta = {
+						type: 'publication',
+						publication: result.publication,
+						subscriptionData: result.subscriptionData,
+						userSpecific: result.userSpecific,
+						snapshot: result.snapshot
+					};
+				}
+
+				if (isPublicationResponse) {
+					meta = publicationMeta;
+					packedResult = <Buffer>pack(publicationPayload);
+					result = null;
+				}
+				else {
+					packedResult = <Buffer>pack(result);
+				}
+			}
+			catch (packErr) {
+				console.error(new Date(), 'Worker pack error', packErr);
+				if (isPublicationResponse) {
+					meta = publicationMeta;
+					result = publicationPayload;
+				}
+			}
+
+
+			if (this._methodManager.getEnableDebug()) {
+				console.log(new Date(), 'Finished method', method);
+			}
+
+			if (!timedOut) {
+				clearTimeout(timeoutHandle);
+				const payload: TaskResponse = {
+					type: 'taskComplete',
+					taskId,
+					messageId,
+					error: false,
+					result: packedResult ? null : result,
+					packedResult,
+					encoding: 'msgpack',
+					meta
+				};
+
+				this.sendWorkerResponse(ws, payload);
+			}
+
+            this._runningTasks = this._runningTasks.filter(a => a !== taskId);
+            
+            if (this._methodManager.getEnableDebug()) {
+                console.log(new Date(), 'Done with Task', JSON.stringify(this._runningTasks, null, 2));
+            }
+        }
+        catch (err) {
+            if (!timedOut) {
+                clearTimeout(timeoutHandle);
+				const errorAt = Date.now();
+				const runMs = round(errorAt - receivedAt);
+				const totalMs = queuedAt ? round(errorAt - queuedAt) : null;
+                console.error('Worker failed task:', taskId, 'Method:', method, err);
+                err.message = 'Worker failed task:' + taskId + ' Method:' + method + ' - ' + err.message;
+				this.logWorkerEvent('workerTaskError', {
+					taskId,
+					method,
+					messageId,
+					runMs,
+					queueWaitMs,
+					dispatchDelayMs,
+					totalMs,
+					userContext,
+					params,
+					error: err
+				});
+                this.sendWorkerResponse(ws, {
+                    type: 'taskComplete',
+                    taskId,
+                    messageId,
+                    error: true,
+                    result: err || 'Unknown error'
+                });
+                throw err;
+            }
+
+			const errorAt = Date.now();
+			const runMs = round(errorAt - receivedAt);
+			const totalMs = queuedAt ? round(errorAt - queuedAt) : null;
+			this.logWorkerEvent('workerTaskError', {
+				taskId,
+				method,
+				messageId,
+				runMs,
+				queueWaitMs,
+				dispatchDelayMs,
+				totalMs,
+				userContext,
+				params,
+				error: err
+			});
+
+            console.error(new Date(),  `Error in Method: ${method} - ${JSON.stringify(data, null, 2)}`);
+            
+            await this._methodManager.callMethod('insertErrorLog', `Error in Method: ${method} - ${JSON.stringify(data, null, 2)}`);
+            
+            this._runningTasks = this._runningTasks.filter(a => a !== taskId);
+        }
+    }
+
+	private logWorkerEvent(event: string, details: Record<string, any>) {
+		console.log(this.safeStringify({
+			ts: new Date().toISOString(),
+			event,
+			...details
+		}));
+	}
+
+	private safeStringify(value: any): string {
+		try {
+			const seen = new WeakSet();
+			return JSON.stringify(value, (key, val) => {
+				if (key === '__proto__') {
+					return undefined;
+				}
+				if (typeof val === 'bigint') {
+					return val.toString();
+				}
+				if (val instanceof Error) {
+					return {
+						name: val.name,
+						message: val.message,
+						stack: val.stack
+					};
+				}
+				if (val && typeof val === 'object') {
+					if (seen.has(val)) {
+						return '[Circular]';
+					}
+					seen.add(val);
+				}
+				return val;
+			});
+		}
+		catch (err) {
+			return JSON.stringify({
+				error: 'safeStringifyFailed',
+				message: err?.message || String(err)
+			});
+		}
+	}
+
+	public sendWorkerResponse(ws, payload: TaskResponse | string) {
+		if (this._methodManager.getEnableDebug()) {
+			console.log(new Date(), 'Sending', typeof payload === 'string' ? payload : '[binary]');
+		}
+
+		if (ws && ws.readyState === WebSocket.OPEN) {
+			try {
+				if (typeof payload === 'string') {
+					ws.send(payload);
+				}
+				else {
+					ws.send(pack(payload));
+				}
+			}
+			catch (err) {
+				console.error('Failed to send worker response:', err);
+			}
+		}
+		else if (ws) {
+			ws.close();
+		}
+	}
+}