@resolveio/server-lib 22.3.221 → 22.3.222

package/src/managers/worker-server.manager.ts

@@ -1,536 +0,0 @@
-import { pack, unpack } from 'msgpackr';
-import * as WebSocket from 'ws';
-import { TaskPayload, TaskResponse } from '../models/server-message.model';
-import { dateReviver, round } from '../util/common';
-import { MethodManager } from './method.manager';
-
-export interface WorkerServerConnectionHealth {
-	isConnected: boolean;
-	lastConnectionAttemptAt: number;
-	lastConnectedAt: number;
-	lastDisconnectedAt: number;
-	connectionLossStartedAt: number;
-	currentOutageMs: number;
-	lastDisconnectReason: string;
-}
-
-export class WorkerServerManager {
-	private _methodManager: MethodManager;
-	private _serverConfig;
-	private _runningTasks = [];
-	private _taskLogThresholdMs = 200;
-	private _isConnected = false;
-	private _lastConnectionAttemptAt = 0;
-	private _lastConnectedAt = 0;
-	private _lastDisconnectedAt = 0;
-	private _connectionLossStartedAt = 0;
-	private _lastDisconnectReason = '';
-
-	constructor() {}
-	
-	static create(methodManager: MethodManager, serverConfig) {
-		const workerServerManager = new WorkerServerManager();
-		workerServerManager.initialize(methodManager, serverConfig);
-		return workerServerManager;
-	}
-
-	public initialize(methodManager: MethodManager, serverConfig) {
-		this._methodManager = methodManager;
-		this._serverConfig = serverConfig;
-		if (this._methodManager && typeof this._methodManager.onReady === 'function') {
-			this._methodManager.onReady(() => this.startWorkerInstance());
-		}
-		else {
-			this.startWorkerInstance();
-		}
-	}
-
-	private markWorkerConnectionAttempt() {
-		const now = Date.now();
-		this._lastConnectionAttemptAt = now;
-		if (!this._isConnected && !this._connectionLossStartedAt) {
-			this._connectionLossStartedAt = now;
-		}
-	}
-
-	private markWorkerConnected() {
-		const now = Date.now();
-		this._isConnected = true;
-		this._lastConnectedAt = now;
-		this._lastDisconnectedAt = 0;
-		this._connectionLossStartedAt = 0;
-		this._lastDisconnectReason = '';
-	}
-
-	private markWorkerDisconnected(reason: string) {
-		const now = Date.now();
-		this._isConnected = false;
-		this._lastDisconnectedAt = now;
-		if (!this._connectionLossStartedAt) {
-			this._connectionLossStartedAt = now;
-		}
-		this._lastDisconnectReason = String(reason || 'Worker websocket disconnected.');
-	}
-
-	public getConnectionHealth(): WorkerServerConnectionHealth {
-		return {
-			isConnected: this._isConnected,
-			lastConnectionAttemptAt: this._lastConnectionAttemptAt,
-			lastConnectedAt: this._lastConnectedAt,
-			lastDisconnectedAt: this._lastDisconnectedAt,
-			connectionLossStartedAt: this._connectionLossStartedAt,
-			currentOutageMs: (!this._isConnected && this._connectionLossStartedAt)
-				? Math.max(0, Date.now() - this._connectionLossStartedAt)
-				: 0,
-			lastDisconnectReason: this._lastDisconnectReason
-		};
-	}
-
-	private startWorkerInstance() {
-		const workerIndex = encodeURIComponent(String(process.env.WORKER_INDEX || ''));
-		const workerInstance = encodeURIComponent(String(process.env.NODE_APP_INSTANCE || ''));
-		const workerToken = encodeURIComponent(String(this._serverConfig['WORKER_TOKEN'] || ''));
-		let wsUrl = this._serverConfig['SERVER_URL'] + '/websocket?workerToken=' + workerToken + '&workerIndex=' + workerIndex + '&workerInstance=' + workerInstance;
-		const ws = new WebSocket(wsUrl);
-
-		this.markWorkerConnectionAttempt();
-
-		let lastComm = new Date();
-		let interval = null;
-		let missedPongs = 0;
-		const heartbeatIntervalMs = 15000;
-		const maxMissedPongs = 2;
-		const maxSilenceMs = heartbeatIntervalMs * (maxMissedPongs + 1);
-		let openTimeout = null;
-		let opened = false;
-
-		// Set timeout if the socket never opens
-		openTimeout = setTimeout(() => {
-			if (!opened) {
-				this.markWorkerDisconnected('WebSocket connection timeout.');
-				console.error(new Date(), 'WebSocket connection timeout. Retrying...');
-				ws.terminate(); // force close if still connecting
-			}
-		}, 10000); // 10 seconds timeout
-
-		ws.on('open', () => {
-			opened = true;
-			clearTimeout(openTimeout);
-			this.markWorkerConnected();
-
-			console.log(new Date(), 'Connected to main server as worker', process.env.WORKER_INDEX, process.env.NODE_APP_INSTANCE);
-			this.sendWorkerResponse(ws, 'ping');
-
-			interval = setInterval(() => {
-				const now = Date.now();
-				const last = lastComm ? lastComm.getTime() : 0;
-				const silenceMs = last ? now - last : maxSilenceMs + 1;
-				if (silenceMs > maxSilenceMs || missedPongs > maxMissedPongs) {
-					this.markWorkerDisconnected('Worker websocket heartbeat timed out.');
-					ws.close();
-					return;
-				}
-				missedPongs += 1;
-				this.sendWorkerResponse(ws, 'ping');
-			}, heartbeatIntervalMs);
-		});
-
-		ws.on('message', async (rawData: WebSocket.RawData) => {
-			lastComm = new Date();
-			if (typeof rawData === 'string') {
-				if (this._methodManager.getEnableDebug()) {
-					console.log(new Date(), 'Message Recv', rawData);
-				}
-
-				if (rawData === 'ping') {
-					if (this._methodManager.getEnableDebug()) {
-						console.log(new Date(), 'Recv Ping, Sending Pong');
-					}
-					
-					this.sendWorkerResponse(ws, 'pong');
-				}
-				else if (rawData === 'pong') {
-					if (this._methodManager.getEnableDebug()) {
-						console.log(new Date(), 'Recv Pong');
-					}
-					missedPongs = 0;
-				}
-				else {
-					let msg: any;
-					try {
-						msg = JSON.parse(rawData, dateReviver);
-					}
-					catch (e) {
-						console.error('Worker parse error', e);
-						return;
-					}
-	
-					if (msg.type === 'task') {
-						await this.handleIncomingTask(ws, msg);
-					}
-				}
-
-				return;
-			}
-
-			let messageBuffer: Buffer;
-
-			if (Buffer.isBuffer(rawData)) {
-				messageBuffer = rawData;
-			}
-			else if (Array.isArray(rawData)) {
-				const chunks = rawData as unknown as ReadonlyArray<Uint8Array>;
-				messageBuffer = Buffer.concat(chunks);
-			}
-			else if (rawData instanceof ArrayBuffer) {
-				messageBuffer = Buffer.from(rawData);
-			}
-			else if (ArrayBuffer.isView(rawData)) {
-				const view = rawData as NodeJS.ArrayBufferView;
-				messageBuffer = Buffer.from(view.buffer, view.byteOffset, view.byteLength);
-			}
-			else {
-				messageBuffer = Buffer.from(rawData as any);
-			}
-
-			if (this._methodManager.getEnableDebug()) {
-				console.log(new Date(), 'Message Recv (binary)', messageBuffer.length);
-			}
-
-			if (messageBuffer.length === 4) {
-				let heartbeat = messageBuffer.toString('utf8');
-
-				if (heartbeat === 'ping') {
-					if (this._methodManager.getEnableDebug()) {
-						console.log(new Date(), 'Recv Ping (binary), Sending Pong');
-					}
-
-					this.sendWorkerResponse(ws, 'pong');
-					return;
-				}
-				else if (heartbeat === 'pong') {
-					if (this._methodManager.getEnableDebug()) {
-						console.log(new Date(), 'Recv Pong (binary)');
-					}
-					missedPongs = 0;
-					return;
-				}
-			}
-
-			let msg: TaskPayload;
-
-			try {
-				msg = unpack(messageBuffer);
-			}
-			catch (e) {
-				console.error('Worker binary parse error', e);
-				return;
-			}
-
-			if (msg.type === 'task') {
-				await this.handleIncomingTask(ws, msg);
-			}
-		});
-    
-		ws.on('close', (code, reason) => {
-			const closeReason = reason && reason.length
-				? `Worker websocket closed (${code}): ${reason.toString()}`
-				: `Worker websocket closed (${code})`;
-			this.markWorkerDisconnected(closeReason);
-
-			setTimeout(() => {
-				this.startWorkerInstance();
-			}, 1000);
-
-			if (interval) {
-				clearInterval(interval);
-			}
-			clearTimeout(openTimeout);
-		});
-
-		ws.on('error', (err) => {
-			const message = err?.message || '';
-			const statusMatch = message.match(/Unexpected server response:\s*(\d{3})/);
-			const statusCode = statusMatch ? statusMatch[1] : null;
-			const isRestartNoise = statusCode === '502';
-			if (statusCode && !isRestartNoise) {
-				console.error(new Date(), `Worker WS error: Failure ${statusCode}`);
-			}
-			else if (!isRestartNoise) {
-				console.error(new Date(), 'Worker WS error:', err);
-			}
-			else if (this._methodManager.getEnableDebug()) {
-				console.log(new Date(), 'Worker WS reconnecting after main restart', message);
-			}
-			this.markWorkerDisconnected(message || 'Worker websocket error.');
-			ws.close();
-		});
-	}
-
-    private async handleIncomingTask(ws: WebSocket.WebSocket, data: TaskPayload) {
-        let { taskId, messageId, method, params, userContext } = data;
-		const receivedAt = Date.now();
-		const queuedAt = data.queuedAt;
-		const dispatchedAt = data.dispatchedAt;
-		const queueWaitMs = queuedAt && dispatchedAt ? round(dispatchedAt - queuedAt) : (queuedAt ? round(receivedAt - queuedAt) : null);
-		const dispatchDelayMs = dispatchedAt ? round(receivedAt - dispatchedAt) : null;
-        this._runningTasks.push(taskId);
-        const missingMethod = !method || !this._methodManager.getMethod(method);
-        if (!taskId || !method || missingMethod) {
-            if (missingMethod) {
-                console.error(new Date(), 'No method in method manager for handleIncomingTask worker server', method);
-				if (this._methodManager.getMethod('insertErrorLog')) {
-					try {
-						await this._methodManager.callMethod('insertErrorLog', `No Method in worker server handleIncomingTask: ${method} - ${JSON.stringify(data, null, 2)}`);
-					}
-					catch (error) {
-						console.error(new Date(), 'insertErrorLog failed in worker-server', error);
-					}
-				}
-            }
-
-            // console.log('Invalid task message received', data);
-			this.logWorkerEvent('workerTaskInvalid', {
-				taskId,
-				method,
-				messageId,
-				queueWaitMs,
-				dispatchDelayMs,
-				userContext,
-				params
-			});
-            this.sendWorkerResponse(ws, {
-                type: 'taskComplete',
-                taskId,
-                messageId,
-                error: true,
-                result: 'Invalid task'
-            });
-
-            return;
-        }
-
-        let timedOut = false;
-        let timeoutHandle = setTimeout(async () => {
-            timedOut = true;
-			const timeoutAt = Date.now();
-			const runMs = round(timeoutAt - receivedAt);
-			const totalMs = queuedAt ? round(timeoutAt - queuedAt) : null;
-            console.error(new Date(), 'Worker timed out on task:', taskId, 'Method:', method);
-
-			this.logWorkerEvent('workerTaskTimeout', {
-				taskId,
-				method,
-				messageId,
-				runMs,
-				queueWaitMs,
-				dispatchDelayMs,
-				totalMs,
-				userContext,
-				params
-			});
-
-            this.sendWorkerResponse(ws, {
-                type: 'taskComplete',
-                taskId,
-                messageId,
-                error: true,
-                result: 'Task timed out'
-            });
-
-            await this._methodManager.callMethod('insertErrorLog', `Timeout in Method: ${method} - ${JSON.stringify(data, null, 2)}`);
-        }, this._methodManager.getMethod(method)?.timeoutOverride || (1000 * 60 * 2));
-
-        try {
-            let managerThis = Object.assign({}, this._methodManager, MethodManager.prototype, {
-                id_user: userContext?.id_user || '',
-                user: userContext?.user || '',
-                id_ws: userContext?.id_ws || ''
-            });
-
-            if (this._methodManager.getEnableDebug()) {
-                console.log(new Date(), 'Running method', method);
-            }
-
-			let result = await this._methodManager.callMethod.call(managerThis, method, ...params);
-			let packedResult: Uint8Array | Buffer = null;
-			let meta: Record<string, any> = null;
-			let publicationPayload: any = undefined;
-			let publicationMeta: Record<string, any> = null;
-			const isPublicationResponse = !!(result && result.__publicationResponse);
-
-			try {
-				if (isPublicationResponse) {
-					publicationPayload = result.payload;
-					publicationMeta = {
-						type: 'publication',
-						publication: result.publication,
-						subscriptionData: result.subscriptionData,
-						userSpecific: result.userSpecific,
-						snapshot: result.snapshot
-					};
-				}
-
-				if (isPublicationResponse) {
-					meta = publicationMeta;
-					packedResult = <Buffer>pack(publicationPayload);
-					result = null;
-				}
-				else {
-					packedResult = <Buffer>pack(result);
-				}
-			}
-			catch (packErr) {
-				console.error(new Date(), 'Worker pack error', packErr);
-				if (isPublicationResponse) {
-					meta = publicationMeta;
-					result = publicationPayload;
-				}
-			}
-
-
-			if (this._methodManager.getEnableDebug()) {
-				console.log(new Date(), 'Finished method', method);
-			}
-
-			if (!timedOut) {
-				clearTimeout(timeoutHandle);
-				const payload: TaskResponse = {
-					type: 'taskComplete',
-					taskId,
-					messageId,
-					error: false,
-					result: packedResult ? null : result,
-					packedResult,
-					encoding: 'msgpack',
-					meta
-				};
-
-				this.sendWorkerResponse(ws, payload);
-			}
-
-            this._runningTasks = this._runningTasks.filter(a => a !== taskId);
-            
-            if (this._methodManager.getEnableDebug()) {
-                console.log(new Date(), 'Done with Task', JSON.stringify(this._runningTasks, null, 2));
-            }
-        }
-        catch (err) {
-            if (!timedOut) {
-                clearTimeout(timeoutHandle);
-				const errorAt = Date.now();
-				const runMs = round(errorAt - receivedAt);
-				const totalMs = queuedAt ? round(errorAt - queuedAt) : null;
-                console.error('Worker failed task:', taskId, 'Method:', method, err);
-                err.message = 'Worker failed task:' + taskId + ' Method:' + method + ' - ' + err.message;
-				this.logWorkerEvent('workerTaskError', {
-					taskId,
-					method,
-					messageId,
-					runMs,
-					queueWaitMs,
-					dispatchDelayMs,
-					totalMs,
-					userContext,
-					params,
-					error: err
-				});
-                this.sendWorkerResponse(ws, {
-                    type: 'taskComplete',
-                    taskId,
-                    messageId,
-                    error: true,
-                    result: err || 'Unknown error'
-                });
-                throw err;
-            }
-
-			const errorAt = Date.now();
-			const runMs = round(errorAt - receivedAt);
-			const totalMs = queuedAt ? round(errorAt - queuedAt) : null;
-			this.logWorkerEvent('workerTaskError', {
-				taskId,
-				method,
-				messageId,
-				runMs,
-				queueWaitMs,
-				dispatchDelayMs,
-				totalMs,
-				userContext,
-				params,
-				error: err
-			});
-
-            console.error(new Date(),  `Error in Method: ${method} - ${JSON.stringify(data, null, 2)}`);
-            
-            await this._methodManager.callMethod('insertErrorLog', `Error in Method: ${method} - ${JSON.stringify(data, null, 2)}`);
-            
-            this._runningTasks = this._runningTasks.filter(a => a !== taskId);
-        }
-    }
-
-	private logWorkerEvent(event: string, details: Record<string, any>) {
-		console.log(this.safeStringify({
-			ts: new Date().toISOString(),
-			event,
-			...details
-		}));
-	}
-
-	private safeStringify(value: any): string {
-		try {
-			const seen = new WeakSet();
-			return JSON.stringify(value, (key, val) => {
-				if (key === '__proto__') {
-					return undefined;
-				}
-				if (typeof val === 'bigint') {
-					return val.toString();
-				}
-				if (val instanceof Error) {
-					return {
-						name: val.name,
-						message: val.message,
-						stack: val.stack
-					};
-				}
-				if (val && typeof val === 'object') {
-					if (seen.has(val)) {
-						return '[Circular]';
-					}
-					seen.add(val);
-				}
-				return val;
-			});
-		}
-		catch (err) {
-			return JSON.stringify({
-				error: 'safeStringifyFailed',
-				message: err?.message || String(err)
-			});
-		}
-	}
-
-	public sendWorkerResponse(ws, payload: TaskResponse | string) {
-		if (this._methodManager.getEnableDebug()) {
-			console.log(new Date(), 'Sending', typeof payload === 'string' ? payload : '[binary]');
-		}
-
-		if (ws && ws.readyState === WebSocket.OPEN) {
-			try {
-				if (typeof payload === 'string') {
-					ws.send(payload);
-				}
-				else {
-					ws.send(pack(payload));
-				}
-			}
-			catch (err) {
-				console.error('Failed to send worker response:', err);
-			}
-		}
-		else if (ws) {
-			ws.close();
-		}
-	}
-}