@resolveio/server-lib 12.5.52 → 12.5.54

package/src/managers/subscription.manager.ts

@@ -0,0 +1,1196 @@
+import { ServerResponseModel } from '../models/server-response.model';
+import { ActiveSubscriptionModel, SubscriptionModel } from '../models/subscription.model';
+import * as WebSocket from 'ws';
+import { loadLogPublications } from '../publications/logs';
+import { loadAppStatusPublications } from '../publications/app-status';
+import { loadFilePublications } from '../publications/files';
+import { loadSuperAdminPublications } from '../publications/super-admin';
+import { LoggedInUsers } from '../collections/logged-in-users.collection';
+import { loadCronJobPublications } from '../publications/cron-jobs';
+import { loadFlagsPublications } from '../publications/flags';
+import { loadMethodResponsePublications } from '../publications/method-responses';
+import ResolveIOMainServer from '../server-app';
+import { ResolveIOServer } from '../index';
+import { loadNotificationPublications } from '../publications/notifications';
+import { loadReportBuilderReportPublications } from '../publications/report-builder-reports';
+import { LoggedInUserModel } from '../models/logged-in-users.model';
+
+import { ChangeStream, ChangeStreamDeleteDocument, ChangeStreamDocument, ChangeStreamInsertDocument, ChangeStreamReplaceDocument, ChangeStreamUpdateDocument, ResumeToken } from 'mongodb';
+import { loadReportBuilderLibraryPublications } from '../publications/report-builder-libraries';
+import { loadUserGroupPublications } from '../publications/user-groups';
+import { loadUserGuidePublications } from '../publications/user-guides';
+import { loadReportBuilderDashboardBuilderPublications } from '../publications/report-builder-dashboard-builders';
+import { objectIdHexString } from './mongo.manager';
+import { dateReviver, deepCopy } from '../util/common';
+import * as NodeCache from 'node-cache';
+import { MonitorManagerFunction } from './monitor.manager';
+import { Flags } from '../collections/flag.collection';
+import { cpus } from 'os';
+const numCPUs = cpus().length;
+const v8 = require('v8');
+
+// Performance Dependencies
+// import * as path from 'path';
+// import { Worker } from 'worker_threads';
+
+interface MongoQueueModel {
+	_id: number,
+	type: string;
+	collection: string;
+	subscription: ActiveSubscriptionModel;
+	running: boolean;
+	run_again: boolean;
+}
+
+// interface CurrentPerformanceMonitor {
+// 	_id: number;
+// 	function: string;
+// 	publication: string;
+// 	subscriptionData: any[];
+// 	date_start: Date;
+// 	date_end: Date;
+// 	duration: number;
+// 	result: string;
+// }
+
+export class SubscriptionManager {
+
+	private _mainServer: ResolveIOMainServer;
+	private _publications: SubscriptionModel = {};
+	private _subscriptions: ActiveSubscriptionModel[] = [];
+	private _wss: WebSocket.Server;
+	private _loggedInUsers: LoggedInUserModel[] = [];
+
+	private _mongoQueue: MongoQueueModel[] = [];
+	private _mongoQueueId = 0;
+
+	private _sendQueue = [];
+	private _runningQueue = false;
+	
+	private _oplog: ChangeStream;
+
+	private _nodeCache;
+	private _cacheId = 1;
+
+	private serverConfig;
+
+	private _heapStats = v8.getHeapStatistics();
+	private _heapLimit = this._heapStats.heap_size_limit / numCPUs / 8;
+
+	private _monitorManagerFunction: MonitorManagerFunction;
+
+	private _enableDebug = false;
+	private _debugOplogCollections = [];
+	private _debugOplogHits = 0;
+	private _debugSubCollections = [];
+	private _debugSubHits = 0;
+	private _debugUnSubHits = 0;
+	private _debugUnSubAllHits = 0;
+	private _debugMongoQueueHits = 0;
+	private _debugMongoQueueCollections = [];
+	private _debugSendQueueHits = 0;
+	private _debugRemoveCacheHits = 0;
+
+	private _oplogRetryCount = 0;
+
+	// private currentPerfomanceMonitor: CurrentPerformanceMonitor[] = [];
+	// private idPerformance: number = 0;
+	// private performanceThread;
+
+	constructor(mainServer, wss: WebSocket.Server, serverConfig, monitorManagerFunction: MonitorManagerFunction) {
+		this._mainServer = mainServer;
+		this._monitorManagerFunction = monitorManagerFunction;
+
+		this._nodeCache = new NodeCache( { stdTTL: 0, checkperiod: 0 } );
+
+		// setTimeout(() => {
+		// 	console.log('Setting up performance thread');
+
+		// 	this.performanceThread = new Worker(path.join(__dirname, './subscription.performance.js'));
+
+		// 	this.performanceThread.on('exit', code => {
+		// 		console.error(new Date(), 'THREAD EXITED!!!!!!!!!!!!!!!!!!', code);
+		// 	});
+
+		// 	this.performanceThread.on('error', code => {
+		// 		console.error(new Date(), 'THREAD RECV ERROR !!!!!!!!!!!!!!!!!!', code);
+		// 	});
+		// }, 5000);
+
+		// setInterval(() => {
+		// 	console.log('Post thread msg');
+		// 	this.performanceThread.postMessage(this.currentPerfomanceMonitor);
+		// 	this.currentPerfomanceMonitor = [];
+		// }, 10000);
+
+		this.serverConfig = serverConfig;
+		this._wss = wss;
+
+		// Publications
+		loadSuperAdminPublications(this);
+		loadAppStatusPublications(this);
+		loadLogPublications(this);
+		loadFilePublications(this);
+		loadCronJobPublications(this);
+		loadFlagsPublications(this);
+		loadMethodResponsePublications(this);
+		loadNotificationPublications(this);
+		loadReportBuilderReportPublications(this);
+		loadReportBuilderLibraryPublications(this);
+		loadUserGroupPublications(this);
+		loadUserGuidePublications(this);
+		loadReportBuilderDashboardBuilderPublications(this);
+
+		this.tailOpLog();
+
+		setInterval(() => {
+			this._oplogRetryCount = 0;
+		}, 15000);
+
+		setInterval(() => {
+			if (this.getEnableDebug()) {
+				console.log(new Date(), 'Sub Manager', 'Subs', this._subscriptions.length);
+				console.log(new Date(), 'Sub Manager', 'Logged In Users', this._loggedInUsers.length);
+				console.log(new Date(), 'Sub Manager', 'Mongo Queue', this._mongoQueue.length);
+				console.log(new Date(), 'Sub Manager', 'Mongo Queue Hits', this._debugMongoQueueHits);
+				console.log(new Date(), 'Sub Manager', 'Mongo Queue Collections', JSON.stringify(this._debugMongoQueueCollections.sort((a, b) => a.collection.localeCompare(b.collection) || a.publication.localeCompare(b.publication)), null, 2));
+				console.log(new Date(), 'Sub Manager', 'Oplog Hits', this._debugOplogHits);
+				console.log(new Date(), 'Sub Manager', 'Oplog Collections', JSON.stringify(this._debugOplogCollections.sort((a, b) => a.collection.localeCompare(b.collection) || a.type.localeCompare(b.type)), null, 2));
+				console.log(new Date(), 'Sub Manager', 'Send Queue Hits', this._debugSendQueueHits);
+				console.log(new Date(), 'Sub Manager', 'Sub Hits', this._debugSubHits);
+				console.log(new Date(), 'Sub Manager', 'Sub Collections', JSON.stringify(this._debugSubCollections.sort((a, b) => a.publication.localeCompare(b.publication)), null, 2));
+				console.log(new Date(), 'Sub Manager', 'Unsub Hits', this._debugUnSubHits);
+				console.log(new Date(), 'Sub Manager', 'Unsub All Hits', this._debugUnSubAllHits);
+				console.log(new Date(), 'Sub Manager', 'Cache Cleanup Hits', this._debugRemoveCacheHits);
+			}
+			
+			this._debugOplogHits = 0;
+			this._debugOplogCollections = [];
+			this._debugSubCollections = [];
+			this._debugMongoQueueHits = 0;
+			this._debugMongoQueueCollections = [];
+			this._debugSendQueueHits = 0;
+			this._debugSubHits = 0;
+			this._debugUnSubHits = 0;
+			this._debugUnSubAllHits = 0;
+			this._debugRemoveCacheHits = 0;
+		}, 60000);
+
+		setInterval(async () => {
+			if (!this._runningQueue) {
+				this._runningQueue = true;
+
+				if (this._sendQueue.length) {
+					// let startDate = new Date();
+					for (let i = this._sendQueue.length - 1; i >= 0; i--) {
+						this._debugSendQueueHits += 1;
+
+						let ws = this._mainServer.getWS(this._sendQueue[i].id_ws);
+						if (ws && ws.readyState === ws.OPEN) {
+							if (ws.bufferedAmount < 20480) {
+								let sendItem = this._sendQueue.pop();
+								ws.send(JSON.stringify(sendItem.data), (error) => {
+									if (error) {
+										console.log('Error on WS: ', error);
+										if (this.getEnableDebug()) {
+											console.log(new Date(), 'Sub Manager', 'Unsub WS', ws['user'], ws['id_socket'], 1);
+										}
+										this.unsubscribeAll(ws);
+									}
+								});
+							}
+						}
+						else {
+							this._sendQueue.splice(i, 1);
+						}
+					}
+
+					// let endDate = new Date();
+					// this.currentPerfomanceMonitor.push({
+					// 	_id: this.idPerformance++,
+					// 	function: 'sendInterval',
+					// 	publication: '',
+					// 	subscriptionData: [],
+					// 	date_start: startDate,
+					// 	date_end: endDate,
+					// 	duration: moment(endDate).diff(moment(startDate), 'milliseconds', false),
+					// 	result: 'Done'
+					// });
+				}
+
+				const queueArr = this._mongoQueue.filter(a => !a.running);
+				if (queueArr.length) {
+					queueArr.forEach(entry => {
+						entry.running = true;
+					});
+
+					for (let i = queueArr.length - 1; i >= 0; i--) {
+						this._debugMongoQueueHits += 1;
+						
+						let mongoQueue = queueArr[i];
+
+						if (!this._debugMongoQueueCollections.some(a => a.collection === mongoQueue.collection && a.publication === mongoQueue.subscription.publication)) {
+							this._debugMongoQueueCollections.push({
+								collection: mongoQueue.collection,
+								publication: mongoQueue.subscription.publication,
+								hits: 1
+							});
+						}
+						else {
+							this._debugMongoQueueCollections.filter(a => a.collection === mongoQueue.collection && a.publication === mongoQueue.subscription.publication)[0].hits += 1;
+						}
+
+						if (this._publications[mongoQueue.subscription.publication].user_specific) {
+							let promises = [];
+							for (let j = 0; j < mongoQueue.subscription.clients.length; j++) {
+								let client = mongoQueue.subscription.clients[j];
+								let ws = this._mainServer.getWS(client.id_socket);
+								if (ws && ws['id_socket'] === client.id_socket) {
+									promises.push(this.sendDataToOne(ws, client.messageId, mongoQueue.subscription, mongoQueue.type, mongoQueue.collection).catch((err) => {
+										this._mainServer.getMethodManager().sendEmail('dev@resolveio.com', 'SERVER - Error Detected - ' + this.serverConfig['CLIENT_NAME'], 'Error Detected During sendDataToOne - User Specific - Socket: ' + client.id_socket + ', User: ' + client.id_user + ', MessageId: ' + client.messageId + ', Pub: ' + mongoQueue.subscription.publication + ', Err: ' + JSON.stringify(err, null, 2));
+										return null;
+									}));
+								}
+							}
+
+							Promise.all(promises).then(() => {
+								if (mongoQueue.run_again) {
+									mongoQueue.running = false;
+									mongoQueue.run_again = false;
+								}
+								else {
+									if (this._mongoQueue.map(a => a._id).indexOf(mongoQueue._id) >= 0) {
+										this._mongoQueue.splice(this._mongoQueue.map(a => a._id).indexOf(mongoQueue._id), 1);
+									}
+								}
+							});
+						}
+						else {
+							this.sendDataToAll(mongoQueue).then(() => {
+								if (mongoQueue.run_again) {
+									mongoQueue.running = false;
+									mongoQueue.run_again = false;
+								}
+								else {
+									if (this._mongoQueue.map(a => a._id).indexOf(mongoQueue._id) >= 0) {
+										this._mongoQueue.splice(this._mongoQueue.map(a => a._id).indexOf(mongoQueue._id), 1);
+									}
+								}
+							}, () => {
+								if (this._mongoQueue.map(a => a._id).indexOf(mongoQueue._id) >= 0) {
+									this._mongoQueue.splice(this._mongoQueue.map(a => a._id).indexOf(mongoQueue._id), 1);
+								}
+							});
+						}
+					}
+
+					// let endDate = new Date();
+					
+					// this.currentPerfomanceMonitor.push({
+					// 	_id: this.idPerformance++,
+					// 	function: 'mongoQueue',
+					// 	publication: '',
+					// 	subscriptionData: [],
+					// 	date_start: startDate,
+					// 	date_end: endDate,
+					// 	duration: moment(endDate).diff(moment(startDate), 'milliseconds', false),
+					// 	result: 'Done'
+					// });
+				}	
+
+				this._runningQueue = false;
+			}
+		}, 25);
+
+		setInterval(async () => {
+			this._loggedInUsers = await LoggedInUsers.find();
+	
+			let userCopy = deepCopy(this._loggedInUsers);
+			for (let i = this._loggedInUsers.length - 1; i >= 0; i--) {
+				let loggedInUser = userCopy[i];
+
+				if (!loggedInUser.date || Date.now() - loggedInUser.date.getTime() > 120000) {
+					if (this._mainServer.getWS(loggedInUser.id_ws)) {
+						if (this.getEnableDebug()) {
+							console.log(new Date(), 'Sub Manager', 'Unsub WS', this._mainServer.getWS(loggedInUser.id_ws)['user'], this._mainServer.getWS(loggedInUser.id_ws)['id_socket'], 2);
+						}
+						this.unsubscribeAll(this._mainServer.getWS(loggedInUser.id_ws));
+					}
+					else {
+						this._subscriptions.forEach(sub => {
+							for (let j = sub.clients.length - 1; j >= 0; j--) {
+								let client = sub.clients[j];
+
+								if (client.id_socket === loggedInUser.id_ws) {
+									sub.clients.splice(j, 1);
+								}
+							}
+						});
+
+						LoggedInUsers.deleteOne({_id: loggedInUser._id});
+
+						if (this._loggedInUsers.findIndex(a => a._id === loggedInUser._id) >= 0) {
+							this._loggedInUsers.splice(this._loggedInUsers.findIndex(a => a._id === loggedInUser._id), 1);
+						}
+					}
+				}
+			}
+
+			for (let i = 0; i < this._subscriptions.length; i++) {
+				let sub = this._subscriptions[i];
+
+				for (let j = sub.clients.length - 1; j >= 0; j--) {
+					let client = sub.clients[j];
+
+					if (!this._loggedInUsers.some(a => a.id_ws === client.id_socket)) {
+						sub.clients.splice(j, 1);
+					}
+				}
+			}
+		}, 30000);
+
+		Flags.findOne({type: 'Enable Debug'}).then(flag => {
+			if (flag && flag.value) {
+				this._enableDebug = true;
+			}
+			else {
+				this._enableDebug = false;
+			}
+		});
+	}
+
+	private addToQueue(mongoQueue: MongoQueueModel) {
+		// let startDate = new Date();
+
+		let _mongoQueue = this._mongoQueue.filter(a => a.subscription.publication === mongoQueue.subscription.publication && JSON.stringify(a.subscription.subscriptionData) === JSON.stringify(mongoQueue.subscription.subscriptionData))[0];
+		
+		if (_mongoQueue) {
+			if (_mongoQueue.running) {
+				_mongoQueue.run_again = true;
+
+				// let endDate = new Date();
+				// this.currentPerfomanceMonitor.push({
+				// 	_id: this.idPerformance++,
+				// 	function: 'addToQueue',
+				// 	publication: mongoQueue.subscription.publication,
+				// // 	subscriptionData: mongoQueue.subscription.subscriptionData,
+				// 	subscriptionData: [],
+				// 	date_start: startDate,
+				// 	date_end: endDate,
+				// 	duration: moment(endDate).diff(moment(startDate), 'milliseconds', false),
+				// 	result: 'Run Again'
+				// });
+			}
+			// else {
+			// 	let endDate = new Date();
+			// 	this.currentPerfomanceMonitor.push({
+			// 		_id: this.idPerformance++,
+			// 		function: 'addToQueue',
+			// 		publication: mongoQueue.subscription.publication,
+			// // 	subscriptionData: mongoQueue.subscription.subscriptionData,
+			// 		subscriptionData: [],
+			// 		date_start: startDate,
+			// 		date_end: endDate,
+			// 		duration: moment(endDate).diff(moment(startDate), 'milliseconds', false),
+			// 		result: 'Not Running Yet'
+			// 	});
+			// }
+		}
+		else {
+			mongoQueue._id = this._mongoQueueId;
+			this._mongoQueueId += 1;
+
+			this._mongoQueue.splice(0, 0, mongoQueue);
+			
+			// let endDate = new Date();
+			// this.currentPerfomanceMonitor.push({
+			// 	_id: this.idPerformance++,
+			// 	function: 'addToQueue',
+			// 	publication: mongoQueue.subscription.publication,
+			// // 	subscriptionData: mongoQueue.subscription.subscriptionData,
+			// 	subscriptionData: [],
+			// 	date_start: startDate,
+			// 	date_end: endDate,
+			// 	duration: moment(endDate).diff(moment(startDate), 'milliseconds', false),
+			// 	result: 'Add to Queue'
+			// });
+		}
+	}
+
+	public invalidatePubsCache(collection, type) {
+		// let startDate = new Date();
+
+		ResolveIOServer.getMongoManager().invalidateQueryCache(collection);
+		
+		let collSubs = this._subscriptions.filter(a => a.collections.includes(collection));
+
+		for (let i = collSubs.length - 1; i >= 0; i--) {
+			let sub = collSubs[i];
+			
+			this.addToQueue({
+				_id: 0,
+				type: type,
+				collection: collection,
+				subscription: sub,
+				running: false,
+				run_again: false
+			});
+		}
+
+		// let endDate = new Date();
+		// this.currentPerfomanceMonitor.push({
+		// 	_id: this.idPerformance++,
+		// 	function: 'invalidatePubsCache',
+		// 	publication: collection,
+		// 	subscriptionData: [],
+		// 	date_start: startDate,
+		// 	date_end: endDate,
+		// 	duration: moment(endDate).diff(moment(startDate), 'milliseconds', false),
+		// 	result: 'Done'
+		// });
+	}
+
+	// Add all files to publications private object
+	public publications(method: SubscriptionModel) {
+		this._publications = Object.assign(this._publications, method);
+	}
+
+	public loggedInLatency(ws: WebSocket) {
+		let loggedInUser = this._loggedInUsers.filter(a => a.id_ws === ws['id_socket'])[0];
+
+		if (loggedInUser) {
+			loggedInUser.date = new Date();
+
+			LoggedInUsers.updateOne({id_ws: ws['id_socket']}, {$set: {latency: ws['latency'], date: loggedInUser.date}}).then(res => {
+				if (!res) {
+					this.unsubscribeAll(ws);
+					if (this.getEnableDebug()) {
+						console.log(new Date(), 'Sub Manager', 'Unsub WS', ws['user'], ws['id_socket'], 3);
+					}
+				}
+			}, () => {
+				this.unsubscribeAll(ws);
+				if (this.getEnableDebug()) {
+					console.log(new Date(), 'Sub Manager', 'Unsub WS', ws['user'], ws['id_socket'], 4);
+				}
+			});
+		}
+	}
+
+	// Subscribe to publication
+	public subscribe(messageRoute: string, messageDate: Date, ws: WebSocket, messageId: number, publication: string, subscriptionData: any[]) {
+		// let startDate = new Date();
+		this._debugSubHits += 1;
+
+		if (!this._debugSubCollections.some(a => a.publication === publication)) {
+			this._debugSubCollections.push({
+				publication: publication,
+				hits: 1
+			});
+		}
+		else {
+			this._debugSubCollections.filter(a => a.publication === publication)[0].hits += 1;
+		}
+
+		let pub = this._publications[publication];
+
+		if (!pub) {
+			console.error(new Date(), 'No Publication: ' + publication);
+			return;
+		}
+		else {
+			if (subscriptionData.length > 1 || subscriptionData[0]) {
+				if (!pub.check) {
+					console.error(new Date(), 'No Check Function For Pub ' + publication);
+					return;
+				}
+				else if (!pub.check._schema) {
+					console.error(new Date(), 'No Check Schema For Pub ' + publication);
+					return;
+				}
+				else {
+					let valObj = {};
+					let valKeys = Object.keys(pub.check._schema);
+
+					let rootKeys = valKeys.filter(a => !a.includes('.'));
+					
+					for (let i = 0; i < subscriptionData.length; i++) {
+						valObj[rootKeys[i]] = subscriptionData[i];
+					}
+
+					try {
+						pub.check.validate(valObj);
+					}
+					catch (errors) {
+						if (errors) {
+							console.error(new Date(), 'Error in Pub Check (' + publication + ')', errors);
+							return;
+						}
+					}
+				}
+			}
+
+			if (messageRoute !== 'Bypass') {
+				let urlData = messageRoute.split('/');
+				let urlModule = '';
+				let urlNext = urlData[0];
+				
+				if (urlData[0] === '') {
+					urlModule = '/';
+					urlNext = urlData[1];
+				}
+
+				urlModule += urlNext;
+
+				if (urlData.length > 1) {
+					urlModule += '/';
+				}
+
+				let otherRouteSubs = this._subscriptions.filter(a => a.clients.some(b => b.id_socket === ws['id_socket'] && b.messageRoute !== 'Bypass' && b.messageRoute !== '/' && b.messageRoute !== messageRoute && !b.messageRoute.startsWith(urlModule)));
+			
+				if (otherRouteSubs.length) {
+					this._mainServer.getMethodManager().sendEmail('dev@resolveio.com', 'SERVER - Detected Undestroyed Subscription - ' + this.serverConfig['CLIENT_NAME'], 'USER: ' + ws['user'] + ' (Socket: ' + ws['id_socket'] + ') ' + ' is on route: ' + messageRoute + ' but has the following subscriptions on other routes:' + JSON.stringify(otherRouteSubs, null, 2));
+
+					otherRouteSubs.forEach(otherSub => {
+						otherSub.clients.filter(a => a.id_socket === ws['id_socket']).forEach(client => {
+							this.unsubscribe(client.messageRoute, new Date(), ws, client.messageId, otherSub.publication, otherSub.subscriptionData);
+						});
+					});
+				}
+			}
+
+			let sub = this._subscriptions.filter(a => a.publication === publication && JSON.stringify(a.subscriptionData) === JSON.stringify(subscriptionData))[0];
+
+			// If sub found (another user watching same data), add client to same sub
+			if (sub) {
+				if (!sub.clients.some(a => a.id_socket === ws['id_socket'] && a.messageId === messageId)) {
+					sub.clients.push({
+						id_user: ws['id_user'],
+						messageId: messageId,
+						id_socket: ws['id_socket'],
+						messageRoute: messageRoute
+					});
+				}
+			}
+			// If sub not found, create new sub
+			else {
+				this._subscriptions.push({
+					publication: publication,
+					subscriptionData: subscriptionData,
+					collections: this.getPublicationCollections(publication),
+					clients: [{
+						id_user: ws['id_user'],
+						messageId: messageId,
+						id_socket: ws['id_socket'],
+						messageRoute: messageRoute
+					}],
+					cacheId: 0
+				});
+			}
+
+			if (!sub) {
+				sub = this._subscriptions.filter(a => a.publication === publication && JSON.stringify(a.subscriptionData) === JSON.stringify(subscriptionData))[0];
+			}
+
+			if (!sub.cacheId && !this._mongoQueue.some(a => a.subscription.publication === sub.publication && JSON.stringify(a.subscription.subscriptionData) === JSON.stringify(sub.subscriptionData))) {
+				this.addToQueue({
+					_id: 0,
+					type: 'newSub',
+					collection: '',
+					subscription: sub,
+					running: false,
+					run_again: false
+				});
+
+				// let endDate = new Date();
+				// this.currentPerfomanceMonitor.push({
+				// 	_id: this.idPerformance++,
+				// 	function: 'subscribe',
+				// 	publication: publication,
+				// // 	subscriptionData: subscriptionData,
+				// 	subscriptionData: [],
+				// 	date_start: startDate,
+				// 	date_end: endDate,
+				// 	duration: moment(endDate).diff(moment(startDate), 'milliseconds', false),
+				// 	result: 'Add to Queue'
+				// });
+			}
+			else if (sub.cacheId) {
+				this.sendDataToOne(ws, messageId, sub, 'newSub', publication);
+
+				// let endDate = new Date();
+				// this.currentPerfomanceMonitor.push({
+				// 	_id: this.idPerformance++,
+				// 	function: 'subscribe',
+				// 	publication: publication,
+				// // 	subscriptionData: subscriptionData,
+				// 	subscriptionData: [],
+				// 	date_start: startDate,
+				// 	date_end: endDate,
+				// 	duration: moment(endDate).diff(moment(startDate), 'milliseconds', false),
+				// 	result: 'Send Pub Data Once'
+				// });
+			}
+		}
+	}
+
+	public async createLoggedInUser(id_ws: string): Promise<LoggedInUserModel> {
+		return new Promise(async (resolve, reject) => {
+			let ws = this._mainServer.getWS(id_ws);
+
+			if (ws) {
+				let user = {
+					_id: objectIdHexString(),
+					__v: 0,
+					date: new Date(),
+					id_user: ws['id_user'],
+					user: ws['user'],
+					id_ws: ws['id_socket']
+				};
+				
+				this._loggedInUsers.push(user);
+				LoggedInUsers.insertOne(user);
+				
+				resolve(user);
+			}
+			else {
+				resolve(null);
+			}
+		});
+	}
+
+	// Unsubscribe from publication 
+	public unsubscribe(messageRoute: string, messageDate: Date, ws: WebSocket, messageId: number, publication: string, subscriptionData: any[]) {
+		this._debugUnSubHits += 1;
+		// let startDate = new Date();
+		// console.log('Before');
+		// console.dir(this._subscriptions.map(a => a.publication));
+
+		if (!this._publications[publication]) {
+			console.log('No Publication: ' + publication);
+			return;
+		}
+		else {
+			let sub = this._subscriptions.filter(a => a.publication === publication && JSON.stringify(a.subscriptionData) === JSON.stringify(subscriptionData))[0];
+
+			if (sub) {
+				for (let i = sub.clients.length - 1; i >= 0; i--) {
+					if (sub.clients[i].id_user === ws['id_user'] && sub.clients[i].messageId === messageId && sub.clients[i].id_socket === ws['id_socket']) {
+						sub.clients.splice(i, 1);
+					}
+				}
+			}
+		}
+
+		// let endDate = new Date();
+		// this.currentPerfomanceMonitor.push({
+		// 	_id: this.idPerformance++,
+		// 	function: 'unsubscribe',
+		// 	publication: publication,
+		// // 	subscriptionData: subscriptionData,
+		// 	subscriptionData: [],
+		// 	date_start: startDate,
+		// 	date_end: endDate,
+		// 	duration: moment(endDate).diff(moment(startDate), 'milliseconds', false),
+		// 	result: 'Done'
+		// });
+	}
+
+	// Unsubscribe from publication
+	public async unsubscribeAll(ws: WebSocket) {
+		this._debugUnSubAllHits += 1;
+		// console.log('Before');
+		// console.dir(this._subscriptions.map(a => a.publication));
+
+		// let startDate = new Date();
+
+		if (ws) {
+			let loggedInUser = await LoggedInUsers.findOne({id_ws: ws['id_socket']});
+
+			if (loggedInUser) {	
+				if (this._loggedInUsers.map(a => a._id).indexOf(loggedInUser._id) >= 0) {
+					this._loggedInUsers.splice(this._loggedInUsers.map(a => a._id).indexOf(loggedInUser._id), 1);
+				}
+				
+				LoggedInUsers.deleteOne({_id: loggedInUser._id});
+			}
+
+			let userSubs = this._subscriptions.filter(a => a.clients.some(b => b.id_user === ws['id_user'] && b.id_socket === ws['id_socket']));
+
+			for (let i = userSubs.length - 1; i >= 0; i--) {
+				let sub = userSubs[i];
+
+				for (let j = sub.clients.length - 1; j >= 0; j--) {
+					if (sub.clients[j].id_socket === ws['id_socket']) {
+						sub.clients.splice(j, 1);
+					}
+				}
+			}
+
+			ws.close();
+
+			process.nextTick(() => {
+				if ([ws.OPEN, ws.CLOSING].includes(<any>ws.readyState)) {
+					// Socket still hangs, hard close
+					ws.terminate();
+				}
+			});
+
+			// let endDate = new Date();
+			// this.currentPerfomanceMonitor.push({
+			// 	_id: this.idPerformance++,
+			// 	function: 'unsubscribeAll',
+			// 	publication: '',
+			// 	subscriptionData: [],
+			// 	date_start: startDate,
+			// 	date_end: endDate,
+			// 	duration: moment(endDate).diff(moment(startDate), 'milliseconds', false),
+			// 	result: 'Done'
+			// });
+		}
+
+		// console.log('After');
+		// console.dir(this._subscriptions.map(a => a.publication));
+	}
+
+	public getActiveSubscriptions() {
+		return this._subscriptions;
+	}
+
+	// Get publication collection
+	private getPublicationCollections(publication: string) {
+		return this._publications[publication].collections;
+	}
+
+	// Watch (tail) Mongo's operation log on the entire database (all insert/modify/delete will trigger this function)
+	private tailOpLog(resumeToken?: ResumeToken) {
+		if (!this._oplog || this._oplog.closed) {
+			this._oplogRetryCount += 1;
+
+			if (this._oplogRetryCount > 10) {
+				console.error('****************** TAIL OPLOG ERROR, RETRYING A BUNCH OF TIMES, KILLING PROCESS **************************')
+				process.exit(1);
+			}
+
+			let lastResumeToken: ResumeToken;
+
+			if (resumeToken) {
+				lastResumeToken = resumeToken;
+				try {
+					this._oplog = ResolveIOServer.getMainDB().watch([], {resumeAfter: resumeToken});
+				}
+				catch(errOp) {
+					if (this._oplog && !this._oplog.closed) {
+						this._oplog.close();
+					}
+					else {
+						this._oplog = ResolveIOServer.getMainDB().watch();
+					}
+				}
+			}
+			else {
+				this._oplog = ResolveIOServer.getMainDB().watch();
+			}
+
+			console.log(new Date(), 'oplog started');
+
+			this._oplog.on('change', (doc: ChangeStreamInsertDocument | ChangeStreamUpdateDocument | ChangeStreamReplaceDocument | ChangeStreamDeleteDocument) => {
+				// let startDate = new Date();
+
+				if (doc.ns) {
+					let collection = doc.ns.coll;
+
+					if (!this._debugOplogCollections.some(a => a.collection === doc.ns.coll && a.type === doc.operationType)) {
+						this._debugOplogCollections.push({
+							collection: doc.ns.coll,
+							type: doc.operationType,
+							hits: 1
+						});
+					}
+					else {
+						this._debugOplogCollections.filter(a => a.collection === doc.ns.coll && a.type === doc.operationType)[0].hits += 1;
+					}
+					
+					// console.log(new Date(), 'Op log doc', doc.operationType, collection);
+
+					if (collection && !collection.endsWith('.versions') && !collection.startsWith('monitor-') && collection !== 'logs' && collection !== 'log-method-latencies' && collection !== 'log-subscriptions') {
+						this._debugOplogHits += 1;
+						
+						if (doc.operationType === 'insert') {
+							if (collection === 'support-tickets') {
+								if (this.serverConfig['ROOT_URL'] === 'https://resolveio.com') {
+									this._mainServer.getMethodManager().callMethodInternal.call(this._mainServer.getMethodManager(), 'sendSupportTicketEmail', doc.documentKey['_id']);
+									this.invalidatePubsCache(collection, 'insert');
+								}
+							}
+							
+							if (collection !== 'method-responses') {
+								this.invalidatePubsCache(collection, 'insert');
+							}
+
+							// let endDate = new Date();
+							// this.currentPerfomanceMonitor.push({
+							// 	_id: this.idPerformance++,
+							// 	function: 'oplog',
+							// 	publication: collection,
+							// 	subscriptionData: [],
+							// 	date_start: startDate,
+							// 	date_end: endDate,
+							// 	duration: moment(endDate).diff(moment(startDate), 'milliseconds', false),
+							// 	result: 'Insert'
+							// });
+						}
+						else if (doc.operationType === 'update' || doc.operationType === 'replace') {
+							if (collection !== 'method-responses') {
+								this.invalidatePubsCache(collection, 'update');
+							}
+
+							// let endDate = new Date();
+							// this.currentPerfomanceMonitor.push({
+							// 	_id: this.idPerformance++,
+							// 	function: 'oplog',
+							// 	publication: collection,
+							// 	subscriptionData: [],
+							// 	date_start: startDate,
+							// 	date_end: endDate,
+							// 	duration: moment(endDate).diff(moment(startDate), 'milliseconds', false),
+							// 	result: 'Update'
+							// });
+						}
+						else if (doc.operationType === 'delete') {
+							if (collection !== 'method-responses') {
+								this.invalidatePubsCache(collection, 'delete');
+							}
+
+							// let endDate = new Date();
+							// this.currentPerfomanceMonitor.push({
+							// 	_id: this.idPerformance++,
+							// 	function: 'oplog',
+							// 	publication: collection,
+							// 	subscriptionData: [],
+							// 	date_start: startDate,
+							// 	date_end: endDate,
+							// 	duration: moment(endDate).diff(moment(startDate), 'milliseconds', false),
+							// 	result: 'Delete'
+							// });
+						}
+					}
+
+					if (collection === 'flags') {
+						Flags.findOne({type: 'Enable Debug'}).then(flag => {
+							if (flag && flag.value) {
+								this._enableDebug = true;
+							}
+							else {
+								this._enableDebug = false;
+							}
+						});
+					}
+
+					lastResumeToken = doc._id;
+				}
+			});
+			
+			this._oplog.on('error', error => {
+				console.log(new Date(), 'oplog error', error);
+				this._oplog.close();
+			});
+
+			this._oplog.on('end', () => {
+				console.log(new Date(), 'oplog end');
+				this._oplog.close();
+			});
+
+			this._oplog.on('close', () => {
+				this._oplog = null;
+
+				console.log(new Date(), 'oplog close');
+				this.tailOpLog(lastResumeToken);
+			});
+		}
+		else {
+			this._oplog.close();
+		}
+	}
+
+	// Fetch pub once, send to all clients linked to this pub
+	private sendDataToOne(ws: WebSocket, messageId: number, subscription: ActiveSubscriptionModel, type: string, collection: string) {
+		return new Promise(async (resolve, reject) => {
+			// let startDate = new Date();
+
+			if (!this._publications[subscription.publication].user_specific) { // Same pub for all users
+				if (subscription.cacheId) {
+					try {
+						let cacheData = JSON.parse(this._nodeCache.get(subscription.cacheId), dateReviver);
+
+						let serverRes: ServerResponseModel = {
+							messageId: messageId,
+							hasError: false,
+							data: cacheData
+						};
+	
+						this.sendWS(ws, serverRes);
+					}
+					catch(err) {
+						this._nodeCache.del(subscription.cacheId);
+						subscription.cacheId = 0;
+
+						this.addToQueue({
+							_id: 0,
+							type: 'newSub',
+							collection: '',
+							subscription: subscription,
+							running: false,
+							run_again: false
+						});
+					}
+
+					// let endDate = new Date();
+					// this.currentPerfomanceMonitor.push({
+					// 	_id: this.idPerformance++,
+					// 	function: 'sendDataToOne',
+					// 	publication: subscription.publication,
+					// // 	subscriptionData: subscription.subscriptionData,
+					// 	subscriptionData: [],
+					// 	date_start: startDate,
+					// 	date_end: endDate,
+					// 	duration: moment(endDate).diff(moment(startDate), 'milliseconds', false),
+					// 	result: 'Send cache'
+					// });
+
+					resolve(true);
+				}
+				else {
+					this._mainServer.getMethodManager().sendEmail('dev@resolveio.com', 'SERVER - Error Detected - ' + this.serverConfig['CLIENT_NAME'], 'Error Detected During sendDataToOne - No Cache! - ' + subscription.publication + '\n\nData \n' + JSON.stringify(subscription.subscriptionData, null, 2));
+
+					resolve(false);
+				}
+			}
+			else { // Pub is different for each user (Look at users publication) [MUCH SLOWER, RE-RUNS FOR EACH USER]
+				let monitor = this._monitorManagerFunction.startMonitorFunction('User Specific Publication', subscription.publication, '', '', subscription.subscriptionData);
+				this._publications[subscription.publication].function.call(Object.assign({}, this, SubscriptionManager.prototype), ws['id_user'], ...subscription.subscriptionData)
+				.then(res => {
+					this._monitorManagerFunction.finishMonitorFunction(monitor);
+					let serverRes: ServerResponseModel = {
+						messageId: messageId,
+						hasError: false,
+						data: res
+					};
+
+					// console.log('Send Pub Data Once 2', new Date(), 'Success', subscription);
+
+					this.sendWS(ws, serverRes);
+
+					// let endDate = new Date();
+					// this.currentPerfomanceMonitor.push({
+					// 	_id: this.idPerformance++,
+					// 	function: 'sendDataToOne',
+					// 	publication: subscription.publication,
+					// // 	subscriptionData: subscription.subscriptionData,
+					// 	subscriptionData: [],
+					// 	date_start: startDate,
+					// 	date_end: endDate,
+					// 	duration: moment(endDate).diff(moment(startDate), 'milliseconds', false),
+					// 	result: 'Send ws spec data'
+					// });
+
+					resolve(true);
+
+				}, err => {
+					this._monitorManagerFunction.finishMonitorFunction(monitor);
+					
+					let serverRes: ServerResponseModel = {
+						messageId: messageId,
+						hasError: true,
+						data: err
+					};
+
+					// console.log('Send Pub Data Once 2', new Date(), 'Error', subscription);
+
+					this.sendWS(ws, serverRes);
+
+					this._mainServer.getMethodManager().sendEmail('dev@resolveio.com', 'SERVER - Error Detected - ' + this.serverConfig['CLIENT_NAME'], 'Error Detected During Subscription ' + subscription.publication + ' - (sendDataToOne - WS)\n\nData \n' + JSON.stringify(subscription.subscriptionData, null, 2) + '\n\nErrors\n' + JSON.stringify(err, null, 2));
+
+					reject(err);
+				});
+			}
+		});
+	}
+
+	// Fetch pub once, send to all clients linked to this pub
+	private sendDataToAll(mongoQueue: MongoQueueModel) {
+		return new Promise(async (resolve, reject) => {
+			// let startDate = new Date();
+			if (!mongoQueue.subscription.clients.length) {				
+				if (mongoQueue.subscription.cacheId) {
+					this._nodeCache.del(mongoQueue.subscription.cacheId);
+				}
+
+				if (this._subscriptions.findIndex(a => a.publication === mongoQueue.subscription.publication && JSON.stringify(a.subscriptionData) === JSON.stringify(mongoQueue.subscription.subscriptionData)) >= 0) {
+					this._subscriptions.splice(this._subscriptions.findIndex(a => a.publication === mongoQueue.subscription.publication && JSON.stringify(a.subscriptionData) === JSON.stringify(mongoQueue.subscription.subscriptionData)), 1);
+				}
+
+				resolve(false);
+			}
+			else {
+				if (mongoQueue.subscription.publication !== 'superadminAPM' && mongoQueue.subscription.publication !== 'loggedInUsers') {
+					this._mainServer.getMethodManager().callMethodInternal.call(this._mainServer.getMethodManager(), 'insertSubscriptionLog', mongoQueue.type, mongoQueue.subscription.publication, mongoQueue.collection, JSON.stringify(mongoQueue.subscription.subscriptionData));
+				}
+
+				let monitor = this._monitorManagerFunction.startMonitorFunction('Publication', mongoQueue.subscription.publication, '', '', mongoQueue.subscription.subscriptionData);
+				this._publications[mongoQueue.subscription.publication].function.call(Object.assign({}, this, SubscriptionManager.prototype), ...mongoQueue.subscription.subscriptionData)
+				.then(res => {
+					this._monitorManagerFunction.finishMonitorFunction(monitor);
+					if (!mongoQueue.subscription.clients.length) {
+						let nodeCacheSize = this._nodeCache.getStats().vsize;
+						
+						if (nodeCacheSize > this._heapLimit) {
+							let deleteCount = 0;
+
+							const subArr = this._subscriptions.filter(a => a.cacheId && !a.clients.length);
+
+							for (let zz = 0; zz < subArr.length; zz++) {
+								this._debugRemoveCacheHits += 1;
+
+								this._nodeCache.del(subArr[zz].cacheId);
+								subArr[zz].cacheId = 0;
+
+								deleteCount += 1;
+
+								nodeCacheSize = this._nodeCache.getStats().vsize;
+
+								if (nodeCacheSize < this._heapLimit * 0.75) {
+									break;
+								}
+							}
+							
+							console.log('Sub Cache: ' + 'Too Big - ' + mongoQueue.subscription.publication + ' - Deleted: ' + deleteCount + ' - ' + nodeCacheSize);
+						}
+
+						if (mongoQueue.subscription.cacheId) {
+							this._nodeCache.del(mongoQueue.subscription.cacheId);
+						}
+
+						let cacheId = this._cacheId;
+						this._cacheId += 1;
+
+						this._nodeCache.set(cacheId, JSON.stringify(res));
+						mongoQueue.subscription.cacheId = cacheId;
+
+						// let endDate = new Date();
+						// this.currentPerfomanceMonitor.push({
+						// 	_id: this.idPerformance++,
+						// 	function: 'sendPubData',
+						// 	publication: subscription.publication,
+						// // 	subscriptionData: subscription.subscriptionData,
+						// 	subscriptionData: [],
+						// 	date_start: startDate,
+						// 	date_end: endDate,
+						// 	duration: moment(endDate).diff(moment(startDate), 'milliseconds', false),
+						// 	result: 'No clients'
+						// });
+
+						resolve(false);
+					}
+					else {
+						if (!mongoQueue.subscription.cacheId || this._nodeCache.get(mongoQueue.subscription.cacheId) !== JSON.stringify(res)) {
+							mongoQueue.subscription.clients.forEach(client => {
+								let ws = this._mainServer.getWS(client.id_socket);
+
+								if (ws) {
+									let serverRes: ServerResponseModel = {
+										messageId: client.messageId,
+										hasError: false,
+										data: res
+									};
+			
+									this.sendWS(ws, serverRes);
+								}
+							});
+	
+							let nodeCacheSize = this._nodeCache.getStats().vsize;
+						
+							if (nodeCacheSize > this._heapLimit) {
+								let deleteCount = 0;
+
+								const subArr = this._subscriptions.filter(a => a.cacheId && !a.clients.length);
+	
+								for (let zz = 0; zz < subArr.length; zz++) {
+									this._nodeCache.del(subArr[zz].cacheId);
+									subArr[zz].cacheId = 0;
+	
+									deleteCount += 1;
+
+									nodeCacheSize = this._nodeCache.getStats().vsize;
+	
+									if (nodeCacheSize < this._heapLimit * 0.75) {
+										break;
+									}
+								}
+								
+								console.log('Sub Cache: ' + 'Too Big - ' + mongoQueue.subscription.publication + ' - Deleted: ' + deleteCount + ' - ' + nodeCacheSize);
+							}
+
+							if (mongoQueue.subscription.cacheId) {
+								this._nodeCache.del(mongoQueue.subscription.cacheId);
+							}
+	
+							let cacheId = this._cacheId;
+							this._cacheId += 1;
+
+							this._nodeCache.set(cacheId, JSON.stringify(res));
+							mongoQueue.subscription.cacheId = cacheId;
+
+							// let endDate = new Date();
+							// this.currentPerfomanceMonitor.push({
+							// 	_id: this.idPerformance++,
+							// 	function: 'sendPubData',
+							// 	publication: subscription.publication,
+							// // 	subscriptionData: subscription.subscriptionData,
+							// 	subscriptionData: [],
+							// 	date_start: startDate,
+							// 	date_end: endDate,
+							// 	duration: moment(endDate).diff(moment(startDate), 'milliseconds', false),
+							// 	result: 'Same data'
+							// });
+						}
+
+						resolve(true);
+					}
+				}, err => {
+					this._monitorManagerFunction.finishMonitorFunction(monitor);
+					this._mainServer.getMethodManager().sendEmail('dev@resolveio.com', 'SERVER - Error Detected - ' + this.serverConfig['CLIENT_NAME'], 'Error Detected During Subscription ' + mongoQueue.subscription.publication + ' - (sendPubData)\n\nData \n' + JSON.stringify(mongoQueue.subscription.subscriptionData, null, 2) + '\n\nErrors\n' + JSON.stringify(err, null, 2));
+
+					mongoQueue.subscription.clients.forEach(client => {
+						let ws = this._mainServer.getWS(client.id_socket);
+
+						if (ws) {
+							if (ws['id_socket'] === client.id_socket) {
+								let serverRes: ServerResponseModel = {
+									messageId: client.messageId,
+									hasError: true,
+									data: err
+								};
+								
+								this.sendWS(ws, serverRes);
+							}
+						}
+					});
+
+					reject(err);
+				});
+			}
+		});
+	}
+
+	private sendWS(ws: WebSocket, data: ServerResponseModel) {
+		this._sendQueue.splice(0, 0, {
+			id_ws: ws['id_socket'],
+			data: data
+		});
+	}
+
+	public getEnableDebug() {
+		return this._enableDebug;
+	}
+}