@resolveio/server-lib 20.12.62 → 20.12.64

package/managers/subscription.manager.d.ts

@@ -51,7 +51,7 @@ export declare class SubscriptionManager {
     static create(wss: WebSocket.Server, serverConfig: any, monitorManagerFunction: MonitorManagerFunction): SubscriptionManager;
     private initialize;
     private setCacheLimit;
-    invalidatePubsCache(collection: string, type: string, documentId?: any): Promise<void>;
+    invalidatePubsCache(collection: string, type: string, documentId?: any, document?: any): Promise<void>;
     private _executeInvalidation;
     private subscriptionReferencesCollection;
     private delay;
@@ -84,6 +84,9 @@ export declare class SubscriptionManager {
     private updateSubscriptionDependencies;
     private normalizeDocumentId;
     private getDocumentIdQueryCandidates;
+    private isFilterSafeForInMemory;
+    private documentIdMatchesEvent;
+    private resolveFilterMatch;
     private documentMatchesFilter;
     private shouldInvalidateSubscription;
     private shouldInvalidateSubscriptionForEvents;

package/managers/subscription.manager.js

@@ -74,7 +74,6 @@ var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.SubscriptionManager = void 0;
 var NodeCache = require("node-cache");
-var os_1 = require("os");
 var flag_collection_1 = require("../collections/flag.collection");
 var logged_in_users_collection_1 = require("../collections/logged-in-users.collection");
 var app_status_1 = require("../publications/app-status");
@@ -95,8 +94,18 @@ var common_1 = require("../util/common");
 var error_reporter_1 = require("../util/error-reporter");
 var error_tracking_1 = require("../util/error-tracking");
 var subscription_dependency_context_1 = require("../util/subscription-dependency-context");
-var numCPUs = (0, os_1.cpus)().length;
 var v8 = require('v8');
+var sift = require('sift');
+function resolveHeapLimitBytes() {
+    var stats = v8.getHeapStatistics();
+    if (stats && typeof stats.heap_size_limit === 'number') {
+        return stats.heap_size_limit;
+    }
+    if (stats && typeof stats.total_available_size === 'number') {
+        return stats.total_available_size;
+    }
+    return 0;
+}
 // interface CurrentPerformanceMonitor {
 // 	_id: number;
 // 	function: string;
@@ -116,7 +125,7 @@ var SubscriptionManager = /** @class */ (function () {
         this._mongoQueue = [];
         this._mongoQueueId = 0;
         this._cacheId = 1;
-        this._heapSize = v8.getHeapStatistics() / numCPUs;
+        this._heapSize = resolveHeapLimitBytes();
         this._enableDebug = false;
         this._enableDependencyDebug = false;
         this._debugOplogCollections = [];
@@ -352,7 +361,7 @@ var SubscriptionManager = /** @class */ (function () {
             this._heapLimit = this._heapSize * 0.3; // Use 50% of total heap size
         }
     };
-    SubscriptionManager.prototype.invalidatePubsCache = function (collection, type, documentId) {
+    SubscriptionManager.prototype.invalidatePubsCache = function (collection, type, documentId, document) {
         return __awaiter(this, void 0, void 0, function () {
             var queue, debounceKey, now, firstInvalidationTime, waitedTooLong;
             var _this = this;
@@ -364,7 +373,7 @@ var SubscriptionManager = /** @class */ (function () {
                             queue = { events: [] };
                             this._pendingInvalidations.set(collection, queue);
                         }
-                        queue.events.push({ type: type, documentId: documentId });
+                        queue.events.push({ type: type, documentId: documentId, document: document });
                         debounceKey = collection;
                         now = Date.now();
                         firstInvalidationTime = this._invalidationPendingTimestamps.get(debounceKey) || now;
@@ -1239,7 +1248,7 @@ var SubscriptionManager = /** @class */ (function () {
                         _a.label = 4;
                     case 4:
                         _a.trys.push([4, 5, , 9]);
-                        this._oplog$ = resolveio_server_app_1.ResolveIOServer.getMongoConnection().watch(pipeline, { resumeAfter: resumeToken });
+                        this._oplog$ = resolveio_server_app_1.ResolveIOServer.getMongoConnection().watch(pipeline, { resumeAfter: resumeToken, fullDocument: 'updateLookup' });
                         startedWithResumeToken = true;
                         return [3 /*break*/, 9];
                     case 5:
@@ -1256,18 +1265,18 @@ var SubscriptionManager = /** @class */ (function () {
                         _a.sent();
                         lastResumeToken_1 = null;
                         console.log(new Date(), 'oplog resumeAfter failed, starting fresh', error_6);
-                        this._oplog$ = resolveio_server_app_1.ResolveIOServer.getMongoConnection().watch(pipeline);
+                        this._oplog$ = resolveio_server_app_1.ResolveIOServer.getMongoConnection().watch(pipeline, { fullDocument: 'updateLookup' });
                         startedWithResumeToken = false;
                         this.queueFullResync('oplog-resumeAfter-failed');
                         return [3 /*break*/, 9];
                     case 9: return [3 /*break*/, 11];
                     case 10:
-                        this._oplog$ = resolveio_server_app_1.ResolveIOServer.getMongoConnection().watch(pipeline);
+                        this._oplog$ = resolveio_server_app_1.ResolveIOServer.getMongoConnection().watch(pipeline, { fullDocument: 'updateLookup' });
                         _a.label = 11;
                     case 11:
                         console.log(new Date(), 'oplog started', startedWithResumeToken ? '(resumeAfter)' : '');
                         this._oplog$.on('change', function (doc) { return __awaiter(_this, void 0, void 0, function () {
-                            var collection, docId, flag, dependencyFlag;
+                            var collection, fullDocument, docId, flag, dependencyFlag;
                             return __generator(this, function (_a) {
                                 switch (_a.label) {
                                     case 0:
@@ -1288,18 +1297,19 @@ var SubscriptionManager = /** @class */ (function () {
                                         }
                                         if (!collection) return [3 /*break*/, 10];
                                         this._debugOplogHits += 1;
-                                        docId = doc.documentKey && doc.documentKey['_id'] !== undefined ? doc.documentKey['_id'] : (doc['fullDocument'] && doc['fullDocument']['_id'] !== undefined ? doc['fullDocument']['_id'] : null);
+                                        fullDocument = doc['fullDocument'];
+                                        docId = doc.documentKey && doc.documentKey['_id'] !== undefined ? doc.documentKey['_id'] : (fullDocument && fullDocument['_id'] !== undefined ? fullDocument['_id'] : null);
                                         if (!(doc.operationType === 'insert')) return [3 /*break*/, 5];
                                         if (!(collection === 'support-tickets')) return [3 /*break*/, 2];
                                         if (!(this.serverConfig['ROOT_URL'] === 'https://resolveio.com')) return [3 /*break*/, 2];
                                         resolveio_server_app_1.ResolveIOServer.getMainServer().getMethodManager().callMethod.call(resolveio_server_app_1.ResolveIOServer.getMainServer().getMethodManager(), 'sendSupportTicketEmail', doc.documentKey['_id']);
-                                        return [4 /*yield*/, this.invalidatePubsCache(collection, 'insert', docId)];
+                                        return [4 /*yield*/, this.invalidatePubsCache(collection, 'insert', docId, fullDocument)];
                                     case 1:
                                         _a.sent();
                                         _a.label = 2;
                                     case 2:
                                         if (!(collection !== 'method-responses')) return [3 /*break*/, 4];
-                                        return [4 /*yield*/, this.invalidatePubsCache(collection, 'insert', docId)];
+                                        return [4 /*yield*/, this.invalidatePubsCache(collection, 'insert', docId, fullDocument)];
                                     case 3:
                                         _a.sent();
                                         _a.label = 4;
@@ -1307,7 +1317,7 @@ var SubscriptionManager = /** @class */ (function () {
                                     case 5:
                                         if (!(doc.operationType === 'update' || doc.operationType === 'replace')) return [3 /*break*/, 8];
                                         if (!(collection !== 'method-responses')) return [3 /*break*/, 7];
-                                        return [4 /*yield*/, this.invalidatePubsCache(collection, 'update', docId)];
+                                        return [4 /*yield*/, this.invalidatePubsCache(collection, 'update', docId, fullDocument)];
                                     case 6:
                                         _a.sent();
                                         _a.label = 7;
@@ -1839,14 +1849,143 @@ var SubscriptionManager = /** @class */ (function () {
         }
         return candidates;
     };
-    SubscriptionManager.prototype.documentMatchesFilter = function (collection, documentId, filter) {
+    SubscriptionManager.prototype.isFilterSafeForInMemory = function (filter) {
+        if (!filter || typeof filter !== 'object') {
+            return false;
+        }
+        var allowedOperators = new Set([
+            '$and',
+            '$or',
+            '$nor',
+            '$in',
+            '$nin',
+            '$eq',
+            '$ne',
+            '$gt',
+            '$gte',
+            '$lt',
+            '$lte',
+            '$exists',
+            '$size',
+            '$all',
+            '$elemMatch',
+            '$regex',
+            '$options'
+        ]);
+        var disallowedOperators = new Set([
+            '$where',
+            '$expr',
+            '$text',
+            '$geoWithin',
+            '$geoIntersects',
+            '$near',
+            '$nearSphere',
+            '$jsonSchema',
+            '$mod',
+            '$type',
+            '$bitsAllSet',
+            '$bitsAllClear',
+            '$bitsAnySet',
+            '$bitsAnyClear'
+        ]);
+        var walk = function (value) {
+            var e_8, _a;
+            if (Array.isArray(value)) {
+                return value.every(function (item) { return walk(item); });
+            }
+            if (!value || typeof value !== 'object') {
+                return true;
+            }
+            try {
+                for (var _b = __values(Object.entries(value)), _c = _b.next(); !_c.done; _c = _b.next()) {
+                    var _d = __read(_c.value, 2), key = _d[0], entry = _d[1];
+                    if (key.startsWith('$')) {
+                        if (disallowedOperators.has(key)) {
+                            return false;
+                        }
+                        if (!allowedOperators.has(key)) {
+                            return false;
+                        }
+                    }
+                    if (!walk(entry)) {
+                        return false;
+                    }
+                }
+            }
+            catch (e_8_1) { e_8 = { error: e_8_1 }; }
+            finally {
+                try {
+                    if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
+                }
+                finally { if (e_8) throw e_8.error; }
+            }
+            return true;
+        };
+        return walk(filter);
+    };
+    SubscriptionManager.prototype.documentIdMatchesEvent = function (documentId, document) {
+        var e_9, _a;
+        var _this = this;
+        if (!document || documentId === undefined || documentId === null) {
+            return true;
+        }
+        var eventCandidates = this.getDocumentIdQueryCandidates(documentId)
+            .map(function (id) { return _this.normalizeDocumentId(id); })
+            .filter(Boolean);
+        var docCandidates = this.getDocumentIdQueryCandidates(document === null || document === void 0 ? void 0 : document._id)
+            .map(function (id) { return _this.normalizeDocumentId(id); })
+            .filter(Boolean);
+        if (!eventCandidates.length || !docCandidates.length) {
+            return true;
+        }
+        var docSet = new Set(docCandidates);
+        try {
+            for (var eventCandidates_1 = __values(eventCandidates), eventCandidates_1_1 = eventCandidates_1.next(); !eventCandidates_1_1.done; eventCandidates_1_1 = eventCandidates_1.next()) {
+                var candidate = eventCandidates_1_1.value;
+                if (docSet.has(candidate)) {
+                    return true;
+                }
+            }
+        }
+        catch (e_9_1) { e_9 = { error: e_9_1 }; }
+        finally {
+            try {
+                if (eventCandidates_1_1 && !eventCandidates_1_1.done && (_a = eventCandidates_1.return)) _a.call(eventCandidates_1);
+            }
+            finally { if (e_9) throw e_9.error; }
+        }
+        return false;
+    };
+    SubscriptionManager.prototype.resolveFilterMatch = function (filter, document, documentId) {
+        if (!document || !filter || typeof filter !== 'object') {
+            return null;
+        }
+        if (!this.isFilterSafeForInMemory(filter)) {
+            return null;
+        }
+        if (!this.documentIdMatchesEvent(documentId, document)) {
+            return false;
+        }
+        try {
+            var matcher = sift(filter);
+            return matcher(document);
+        }
+        catch (_a) {
+            return null;
+        }
+    };
+    SubscriptionManager.prototype.documentMatchesFilter = function (collection, documentId, filter, document) {
         return __awaiter(this, void 0, void 0, function () {
-            var db, filterCopy, idCandidates, idCandidates_1, idCandidates_1_1, idValue, combinedFilter, doc, e_8_1, _a;
-            var e_8, _b;
+            var inMemoryMatch, db, filterCopy, idCandidates, idCandidates_1, idCandidates_1_1, idValue, combinedFilter, doc, e_10_1, _a;
+            var e_10, _b;
             return __generator(this, function (_c) {
                 switch (_c.label) {
                     case 0:
                         _c.trys.push([0, 9, , 10]);
+                        inMemoryMatch = this.resolveFilterMatch(filter, document, documentId);
+                        if (inMemoryMatch !== null) {
+                            return [2 /*return*/, inMemoryMatch];
+                        }
                         db = resolveio_server_app_1.ResolveIOServer.getMainDB();
                         filterCopy = (0, common_1.deepCopy)(filter) || {};
                         idCandidates = this.getDocumentIdQueryCandidates(documentId);
@@ -1876,14 +2015,14 @@ var SubscriptionManager = /** @class */ (function () {
                         return [3 /*break*/, 2];
                     case 5: return [3 /*break*/, 8];
                     case 6:
-                        e_8_1 = _c.sent();
-                        e_8 = { error: e_8_1 };
+                        e_10_1 = _c.sent();
+                        e_10 = { error: e_10_1 };
                         return [3 /*break*/, 8];
                     case 7:
                         try {
                             if (idCandidates_1_1 && !idCandidates_1_1.done && (_b = idCandidates_1.return)) _b.call(idCandidates_1);
                         }
-                        finally { if (e_8) throw e_8.error; }
+                        finally { if (e_10) throw e_10.error; }
                         return [7 /*endfinally*/];
                     case 8: return [2 /*return*/, false];
                     case 9:
@@ -1894,10 +2033,10 @@ var SubscriptionManager = /** @class */ (function () {
             });
         });
     };
-    SubscriptionManager.prototype.shouldInvalidateSubscription = function (sub, collection, type, documentId) {
+    SubscriptionManager.prototype.shouldInvalidateSubscription = function (sub, collection, type, documentId, document) {
         return __awaiter(this, void 0, void 0, function () {
-            var normalizedDocumentId, hasDependencyData, trackedIds, filters, filters_1, filters_1_1, filter, e_9_1;
-            var e_9, _a;
+            var normalizedDocumentId, hasDependencyData, trackedIds, filters, filters_1, filters_1_1, filter, e_11_1;
+            var e_11, _a;
             var _b, _c;
             return __generator(this, function (_d) {
                 switch (_d.label) {
@@ -1942,7 +2081,7 @@ var SubscriptionManager = /** @class */ (function () {
                     case 2:
                         if (!!filters_1_1.done) return [3 /*break*/, 5];
                         filter = filters_1_1.value;
-                        return [4 /*yield*/, this.documentMatchesFilter(collection, documentId, filter)];
+                        return [4 /*yield*/, this.documentMatchesFilter(collection, documentId, filter, document)];
                     case 3:
                         if (_d.sent()) {
                             this.dependencyDebug('Invalidate due to filter match', { publication: sub.publication, collection: collection, type: type, documentId: normalizedDocumentId, filter: filter });
@@ -1954,14 +2093,14 @@ var SubscriptionManager = /** @class */ (function () {
                         return [3 /*break*/, 2];
                     case 5: return [3 /*break*/, 8];
                     case 6:
-                        e_9_1 = _d.sent();
-                        e_9 = { error: e_9_1 };
+                        e_11_1 = _d.sent();
+                        e_11 = { error: e_11_1 };
                         return [3 /*break*/, 8];
                     case 7:
                         try {
                             if (filters_1_1 && !filters_1_1.done && (_a = filters_1.return)) _a.call(filters_1);
                         }
-                        finally { if (e_9) throw e_9.error; }
+                        finally { if (e_11) throw e_11.error; }
                         return [7 /*endfinally*/];
                     case 8:
                         this.dependencyDebug('Skip invalidation after dependency checks', { publication: sub.publication, collection: collection, type: type, documentId: normalizedDocumentId });
@@ -1972,8 +2111,8 @@ var SubscriptionManager = /** @class */ (function () {
     };
     SubscriptionManager.prototype.shouldInvalidateSubscriptionForEvents = function (sub, collection, events) {
         return __awaiter(this, void 0, void 0, function () {
-            var sawInsert, sawDelete, events_1, events_1_1, event_1, e_10_1, paginationMeta, paginationReasons;
-            var e_10, _a;
+            var sawInsert, sawDelete, events_1, events_1_1, event_1, e_12_1, paginationMeta, paginationReasons;
+            var e_12, _a;
             return __generator(this, function (_b) {
                 switch (_b.label) {
                     case 0:
@@ -1993,7 +2132,7 @@ var SubscriptionManager = /** @class */ (function () {
                         else if (event_1.type === 'delete') {
                             sawDelete = true;
                         }
-                        return [4 /*yield*/, this.shouldInvalidateSubscription(sub, collection, event_1.type, event_1.documentId)];
+                        return [4 /*yield*/, this.shouldInvalidateSubscription(sub, collection, event_1.type, event_1.documentId, event_1.document)];
                     case 3:
                         if (_b.sent()) {
                             return [2 /*return*/, true];
@@ -2004,14 +2143,14 @@ var SubscriptionManager = /** @class */ (function () {
                         return [3 /*break*/, 2];
                     case 5: return [3 /*break*/, 8];
                     case 6:
-                        e_10_1 = _b.sent();
-                        e_10 = { error: e_10_1 };
+                        e_12_1 = _b.sent();
+                        e_12 = { error: e_12_1 };
                         return [3 /*break*/, 8];
                     case 7:
                         try {
                             if (events_1_1 && !events_1_1.done && (_a = events_1.return)) _a.call(events_1);
                         }
-                        finally { if (e_10) throw e_10.error; }
+                        finally { if (e_12) throw e_12.error; }
                         return [7 /*endfinally*/];
                     case 8:
                         paginationMeta = this.getPaginationMeta(sub, collection);