mosquito-transport 1.9.2 → 1.9.3

package/bin/extract_backup.js

@@ -1,6 +1,6 @@
 import { MongoClient } from "mongodb";
 import { serialize } from 'mongodb/lib/bson.js';
-import { BLOCKS_IDENTIFIERS, encryptData, isPath, isValidColName, isValidDbName, one_gb, resolvePath } from "./utils.js";
+import { BLOCKS_IDENTIFIERS, encryptData, isPath, isValidColName, isValidDbName, one_gb, resolvePath, wait } from "./utils.js";
 import { readdir, stat } from "fs/promises";
 import { createReadStream } from "fs";
 import { Validator } from "guard-object";
@@ -8,7 +8,7 @@ import { WritableBit } from "@deflexable/bit-stream";
 import { join } from "path";
 
 const BIT_SIZE = one_gb * .2;
-const DOC_LIMITER = 500;
+const DOC_LIMITER = 300;
 
 export const extractBackup = (config) => {
     let { database, storage, password, onMongodbOption } = { ...config };
@@ -90,6 +90,7 @@ export const extractBackup = (config) => {
                             pushBuffer(Buffer.from(`${thisCol}`, 'utf8'));
 
                             while (canLoadMore) {
+                                await wait(7); // pause for garbage collection
                                 const data = await dbNameInstance.collection(thisCol).find({})
                                     .skip(offset).limit(DOC_LIMITER).toArray();
                                 offset += DOC_LIMITER;
@@ -149,6 +150,7 @@ export const extractBackup = (config) => {
                                 reject(err);
                             });
                         });
+                        await wait(1); // pause for garbage collection
                     } else {
                         const files = await readdir(dir);
                         if (files.length) {

package/bin/install_backup.js

@@ -1,4 +1,4 @@
-import { BLOCKS_IDENTIFIERS, decryptData, resolvePath } from "./utils.js";
+import { BLOCKS_IDENTIFIERS, decryptData, resolvePath, wait } from "./utils.js";
 import { MongoClient } from "mongodb";
 import { deserialize } from 'mongodb/lib/bson.js';
 import { mkdir } from "fs/promises";
@@ -100,7 +100,9 @@ export const installBackup = (config) => new Promise((callResolve, callReject) =
                         { ...docRest },
                         { upsert: true }
                     );
-                    ++installionStats.totalWrittenDocuments;
+                    if (!(++installionStats.totalWrittenDocuments % 300)) {
+                        await wait(7); // pause for garbage collection
+                    }
                 } else {
                     lastBlocks.database = INIT_BLOCKS.database;
 
@@ -135,7 +137,9 @@ export const installBackup = (config) => new Promise((callResolve, callReject) =
                             const writeStream = createWriteStream(lastBlocks.storage.path);
                             writeStream.write(thisElem);
                             lastBlocks.storage.file = writeStream;
-                            ++installionStats.totalWrittenFiles;
+                            if (!(++installionStats.totalWrittenFiles % 50)) {
+                                await wait(3); // pause for garbage collection
+                            };
                         };
                     } else throw `unknown block identifier "${prevHeader}" at block_id ${BLOCK_ID}`;
                 }

package/bin/utils.js

@@ -4,6 +4,11 @@ import { createCipheriv, createDecipheriv, createHash } from 'node:crypto';
 export const one_mb = 1024 * 1024,
     one_gb = one_mb * 1024;
 
+export const wait = (ms = 1000) =>
+    new Promise(resolve => {
+        setTimeout(resolve, ms);
+    });
+
 export const BLOCKS_IDENTIFIERS = {
     DB_URL: '--->[DB_URL]:',
     DB_NAME: '--->[DB_NAME]:',

package/lib/helpers/variables.js

@@ -22,5 +22,9 @@ export const Scoped = {
      * }}
      */
     InstancesData: {},
-    BlacklistedTokens: {}
+    BlacklistedTokens: {},
+    /**
+     * @type {{ [key: string]: { callers: Map; destroy: () => void; }}}
+     */
+    AccumulatedDatabaseEmittions: {}
 };

package/lib/index.d.ts

@@ -1,4 +1,4 @@
-import { Db, Document, MongoClient, MongoClientOptions, SortDirection, UpdateDescription } from "mongodb";
+import { ChangeStreamOptions, Db, Document, MongoClient, MongoClientOptions, SortDirection, UpdateDescription } from "mongodb";
 import express from "express";
 import { CorsOptions } from "cors";
 import { Sort } from "mongodb";
@@ -882,22 +882,24 @@ interface MosquitoHttpOptions {
     allowDisabledAuth?: boolean;
 }
 
-interface DatabaseListenerOption {
-    includeBeforeData?: boolean;
-    includeAfterData?: boolean;
+interface DatabaseListenerOption extends ChangeStreamOptions {
+    /**
+     * An array of {@link https://www.mongodb.com/docs/manual/reference/operator/aggregation-pipeline/|aggregation pipeline stages} through which to pass change stream documents. This allows for filtering (using $match) and manipulating the change stream documents.
+     */
     pipeline?: { pipeline?: Document[] }
 }
 
 interface DatabaseListenerCallbackData {
     insertion?: { _id: string };
+    update?: UpdateDescription;
+    replacement?: { _id: string };
     deletion?: string;
-    update?: UpdateDescription,
-    before?: Document,
-    after?: Document,
-    timestamp: number,
-    auth?: AuthData | undefined,
-    operation: 'insert' | 'delete' | 'update';
+    before?: Document;
+    after?: Document;
+    timestamp: number;
+    operation: 'insert' | 'update' | 'replace' | 'delete';
     documentKey: string;
+    extras: any
 }
 
 interface StorageSnapshot {

package/lib/index.js

@@ -799,13 +799,14 @@ export default class MosquitoTransportServer {
             if (normalizeRoute(route) === normalizeRoute(e))
                 throw `"${e}" is a reserved route used internally`;
         });
+        const { logger } = this.config;
+        const hasLogger = logger.includes('all') || logger.includes('external-requests'),
+            hasErrorLogger = logger.includes('all') || logger.includes('error');
+
         Scoped.expressInstances[this.port].use(
             express.Router({ caseSensitive: true }).all(`/${normalizeRoute(route)}`, async (req, res) => {
                 const { mtoken, uglified } = req.headers;
-                const { logger } = this.config;
-                const hasLogger = logger.includes('all') || logger.includes('external-requests'),
-                    hasErrorLogger = logger.includes('all') || logger.includes('error'),
-                    now = hasLogger && Date.now();
+                const now = hasLogger && Date.now();
 
                 if (hasLogger) console.log(`started route: /${req.url}`);
                 res.set(NO_CACHE_HEADER);
@@ -917,13 +918,13 @@ export default class MosquitoTransportServer {
 
     listenDatabase = (path, callback, options) => {
         if (typeof path !== 'string') throw `listenDatabase first argument must be a string but got ${path}`;
-        const { dbName, dbUrl } = options || {},
-            { logger } = this.config;
+        const { dbName, dbUrl } = options || {};
+        const { logger } = this.config;
+        const hasLogger = logger.includes('all') || logger.includes('database-snapshot'),
+            hasErrorLogger = logger.includes('all') || logger.includes('error');
 
         return emitDatabase(path, async function () {
-            const hasLogger = logger.includes('all') || logger.includes('database-snapshot'),
-                hasErrorLogger = logger.includes('all') || logger.includes('error'),
-                now = hasLogger && Date.now();
+            const now = hasLogger && Date.now();
             if (hasLogger) console.log(`db-snapshot ${path}: `, arguments[0]);
             try {
                 await callback?.(...arguments);
@@ -1006,11 +1007,11 @@ export default class MosquitoTransportServer {
 
     listenStorage = (callback) => {
         const { logger } = this.config;
+        const hasLogger = logger.includes('all') || logger.includes('storage'),
+            hasErrorLogger = logger.includes('all') || logger.includes('error');
 
         return StorageListener.listenTo(this.projectName, async ({ dest, ...rest }) => {
-            const hasLogger = logger.includes('all') || logger.includes('storage'),
-                hasErrorLogger = logger.includes('all') || logger.includes('error'),
-                now = hasLogger && Date.now();
+            const now = hasLogger && Date.now();
 
             if (hasLogger) console.log(`started listenStorage ${dest}:`);
             try {
@@ -1022,18 +1023,20 @@ export default class MosquitoTransportServer {
         });
     };
 
-    listenNewUser = (callback) => emitDatabase(EnginePath.userAcct, s => {
-        if (s.insertion) {
-            const j = { ...s.insertion };
-            j.uid = j._id;
-            if (j._id) delete j._id;
-            callback?.(j);
-        }
-    }, this.projectName, ADMIN_DB_NAME, ADMIN_DB_URL);
+    listenNewUser = (callback) =>
+        emitDatabase(EnginePath.userAcct, s => {
+            if (s.insertion) {
+                const j = { ...s.insertion };
+                j.uid = j._id;
+                if (j._id) delete j._id;
+                callback?.(j);
+            }
+        }, this.projectName, ADMIN_DB_NAME, ADMIN_DB_URL);
 
-    listenDeletedUser = (callback) => emitDatabase(EnginePath.userAcct, s => {
-        if (s.deletion) callback?.(s.deletion);
-    }, this.projectName, ADMIN_DB_NAME, ADMIN_DB_URL);
+    listenDeletedUser = (callback) =>
+        emitDatabase(EnginePath.userAcct, s => {
+            if (s.deletion) callback?.(s.deletion);
+        }, this.projectName, ADMIN_DB_NAME, ADMIN_DB_URL);
 
     updateUserProfile = async (uid, profile) => {
         if (!Validator.OBJECT(profile)) throw 'updateUserProfile() second argument must be an object';

package/lib/products/database/base.js

@@ -6,19 +6,22 @@ import { Scoped } from "../../helpers/variables.js";
  */
 export const getDB = (projectName, name, url = DEFAULT_DB) => {
     if (!projectName) throw 'expected projectName in getDb()';
-    const { defaultName: dbName, instance } = getDbInstance(projectName, url) || {};
-
-    if (name === ADMIN_DB_NAME) name = dbName;
-    if (!instance) throw `no MongoClient was found for database with dbRef "${url}"`;
-    // if (!name && !dbName) throw `no dbName found for database with dbRef "${dbUrl}"`;
-
-    return instance.db(name || dbName);
+    const { dbName, instance } = getDbNaming(projectName, name, url) || {};
+    return instance.instance.db(dbName);
 };
 
-export const getDbInstance = (projectName, dbUrl = DEFAULT_DB) => {
+export const getDbNaming = (projectName, name, dbRef = DEFAULT_DB) => {
     if (!projectName) throw 'expected projectName in getDb()';
-    if (dbUrl === 'admin' || dbUrl === 'default') throw `reserved keyword dbRef: "${dbUrl}"`;
+    if (dbRef === 'admin' || dbRef === 'default') throw `reserved keyword dbRef: "${dbRef}"`;
+
+    dbRef = dbRef === ADMIN_DB_URL ? 'admin' : dbRef === DEFAULT_DB ? 'default' : dbRef;
+    const instance = Scoped.InstancesData[projectName].mongoInstances[dbRef];
+
+    if (!instance) throw `no MongoClient was found for database with dbRef "${dbRef}"`;
 
-    dbUrl = dbUrl === ADMIN_DB_URL ? 'admin' : dbUrl === DEFAULT_DB ? 'default' : dbUrl;
-    return Scoped.InstancesData[projectName].mongoInstances[dbUrl];
+    return {
+        dbRef,
+        instance,
+        dbName: name === ADMIN_DB_NAME ? instance.defaultName : (name || instance.defaultName)
+    };
 }

package/lib/products/database/index.js

@@ -1,6 +1,6 @@
 import express from "express";
 import { deserializeE2E, encodeBinary, niceTry, serializeE2E } from "../../helpers/utils.js";
-import { getDB, getDbInstance } from "./base.js";
+import { getDB, getDbNaming } from "./base.js";
 import { validateJWT } from "../auth/tokenizer.js";
 import { Scoped } from "../../helpers/variables.js";
 import { EngineRoutes, ERRORS, NO_CACHE_HEADER } from "../../helpers/values.js";
@@ -79,12 +79,11 @@ const deserializeWriteValue = (value) => {
     } else return value;
 };
 
-const cleanseFind = (path, find, projectName, dbName, dbUrl) => {
-    const { defaultName, instance } = getDbInstance(projectName, dbUrl);
-    dbName = dbName || defaultName;
+const cleanseFind = (path, find, projectName, name, dbUrl) => {
+    const { instance, dbName } = getDbNaming(projectName, name, dbUrl);
 
-    if (instance.__intercepted) {
-        const d = instance.interceptMap?.map?.[dbName]?.[path];
+    if (instance.instance.__intercepted) {
+        const d = instance.instance.interceptMap?.map?.[dbName]?.[path];
         if (d?.fulltext) return find;
     }
     return cleanseFindCore(find);
@@ -289,33 +288,95 @@ const extractDocField = async (d, commands, projectName, dbName, dbUrl, doc_hold
 };
 
 export const emitDatabase = (path, callback, projectName, dbName, dbUrl, options) => {
-    const { includeBeforeData, includeAfterData, pipeline } = options || {};
+    const naming = getDbNaming(projectName, dbName, dbUrl);
 
-    const col = getDB(projectName, dbName, dbUrl).collection(path),
-        stream = col.watch(pipeline, {
-            fullDocument: includeAfterData ? 'whenAvailable' : undefined,
-            fullDocumentBeforeChange: includeBeforeData ? 'whenAvailable' : undefined
-        });
+    const nodeId = `${path}:${projectName}:${naming.dbName}:${naming.dbRef}:${options && serializeToBase64(options)}`;
+    let instance = Scoped.AccumulatedDatabaseEmittions[nodeId];
+
+    if (!instance) {
+        const callers = new Map();
+        const destroy = internalEmitDatabase(path, (...args) => {
+            callers.forEach(value => {
+                value(...args);
+            });
+        }, projectName, dbName, dbUrl, options);
+
+        Scoped.AccumulatedDatabaseEmittions[nodeId] = (instance = { callers, destroy });
+    }
+
+    const ref = {};
+    instance.callers.set(ref, callback);
+
+    return () => {
+        if (!instance.callers.has(ref)) return;
+        instance.callers.delete(ref);
+        if (!instance.callers.size) {
+            instance.destroy();
+            delete Scoped.AccumulatedDatabaseEmittions[nodeId];
+        }
+    }
+}
+
+const requiredEvent = ['insert', 'update', 'replace', 'delete'];
+
+const internalEmitDatabase = (path, callback, projectName, dbName, dbUrl, options, resumeAfter) => {
+    const { pipeline, ...restOptions } = options || {};
+
+    const col = getDB(projectName, dbName, dbUrl).collection(path);
+    const stream = col.watch(pipeline || [
+        {
+            $match: {
+                operationType: { $in: requiredEvent }
+            }
+        }
+    ], {
+        ...restOptions,
+        resumeAfter
+    });
 
     stream.on('change', l => {
-        const { operationType: ops, fullDocument, fullDocumentBeforeChange, documentKey, updateDescription, clusterTime } = l;
+        const { operationType: ops, fullDocument, fullDocumentBeforeChange, documentKey, updateDescription, clusterTime, ...rest } = l;
+
+        if (!requiredEvent.includes(ops)) return;
 
-        if (ops !== 'insert' && ops !== 'delete' && ops !== 'update') return;
         callback?.({
             documentKey: documentKey._id,
             insertion: ops === 'insert' ? fullDocument : undefined,
-            deletion: ops === 'delete' ? documentKey._id : undefined,
             update: ops === 'update' ? { ...updateDescription } : undefined,
-            before: includeBeforeData ? fullDocumentBeforeChange : undefined,
-            after: includeAfterData ? fullDocument : undefined,
-            timestamp: clusterTime?.toNumber?.(),
-            auth: undefined,
-            operation: ops
+            replacement: ops === 'replace' ? fullDocument : undefined,
+            deletion: ops === 'delete' ? documentKey._id : undefined,
+            before: fullDocumentBeforeChange,
+            after: fullDocument,
+            timestamp: clusterTime,
+            operation: ops,
+            extras: rest
         });
     });
 
+    stream.on('resumeTokenChanged', (token) => {
+        resumeAfter = token;
+    });
+
+    let closure = null;
+
+    stream.on('error', async (error) => {
+        await stream.close();
+        setTimeout(() => {
+            if (closure === null)
+                closure = internalEmitDatabase(path, callback, projectName, dbName, dbUrl, options, resumeAfter);
+        }, 7000);
+        process.emit('uncaughtException', `emitDatabase error: ${error}`);
+    });
+
     return () => {
-        stream.close();
+        const thisClosure = closure;
+        closure = undefined;
+
+        if (thisClosure === null) {
+            stream.close();
+        } else if (typeof thisClosure === 'function') {
+            thisClosure();
+        }
     }
 };
 
@@ -658,7 +719,7 @@ export const databaseLiveRoutesHandler = ({
                     } catch (e) {
                         socket.emit('mSnapshot', [simplifyCaughtError(e), undefined]);
                     }
-                }, projectName, dbName, dbUrl, { pipeline: { ...commands.find } });
+                }, projectName, dbName, dbUrl);
             } catch (e) {
                 if (hasErrorLoger) console.error(`errRoute /${route} err:`, e);
                 socket.emit('mSnapshot', [simplifyCaughtError(e), undefined]);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mosquito-transport",
-  "version": "1.9.2",
+  "version": "1.9.3",
   "description": "Quickly spawn server infrastructure along robust authentication, database, storage, and cross-platform compatibility",
   "main": "lib/index.js",
   "type": "module",