mongo-realtime 1.2.0 → 2.0.0

package/README.md

@@ -58,6 +58,44 @@ server.listen(3000, () => {
 });
 ```
 
+## Breaking Changes since v2.x.x
+
+In older versions, streams receive all documents and send them to sockets. This can lead to performance issues with large collections.\
+From version 2.x.x, streams now require a limit (default 50 documents) from the client side to avoid overloading the server and network.\
+Receiving streamed docs now works like this:
+
+```javascript
+socket.emit("realtime", {
+  limit: 100,
+  streamId: "users",
+  reverse: true,
+  registerId: "random-client-generated-id",
+});
+/*
+ * Request 100 documents from 'users' stream (streamId defined server-side or collection name if autoStream)
+ * 'reverse' to get the latest documents first
+ * 'registerId' is a unique id generated by the client to identify this request. It will be used to listen to the response event.
+ * The response return an object {results: [...], count: number, total:number, remaining:number, coll:string}
+ */
+
+socket.on("realtime:users:random-client-generated-id", (data) => {
+  console.log("Received streamed documents:", data.results); // 100 max per
+});
+
+socket.emit("realtime", {
+  limit: 50,
+  streamId: "usersWithEmail", // make sure this stream is defined server-side
+  registerId: "specific-request-id",
+});
+
+socket.on("realtime:usersWithEmail:specific-request-id", (data) => {
+  console.log("Received streamed documents with email:", data.results); // 50 max per
+});
+
+// Make sure to listen before emitting the request to avoid missing the response
+// To stop receiving streamed documents for a specific request just use socket.off
+```
+
 ## 📋 API
 
 ### `MongoRealtime.init(options)`
@@ -68,24 +106,25 @@ Initializes the socket system and MongoDB Change Streams.
 
 \* means required
 
-| Parameter                | Type              | Description                                                                      |
-| ------------------------ | ----------------- | -------------------------------------------------------------------------------- |
-| `options.dbUri`          | `String`\*        | Database URI                                                                     |
-| `options.dbOptions`      | `Object`          | Mongoose connection options                                                      |
-| `options.onDbConnect`    | `Function`        | Callback on successful database connection                                       |
-| `options.onDbError`      | `Function`        | Callback on database connection error                                            |
-| `options.server`         | `http.Server`\*   | HTTP server to attach Socket.IO                                                  |
-| `options.authentify`     | `Function`        | Function to authenticate socket connections. Should return true if authenticated |
-| `options.middlewares`    | `Array[Function]` | Array of Socket.IO middlewares                                                   |
-| `options.onSocket`       | `Function`        | Callback on socket connection                                                    |
-| `options.offSocket`      | `Function`        | Callback on socket disconnection                                                 |
-| `options.watch`          | `Array[String]`   | Collections to only watch. Listen to all when is empty                           |
-| `options.ignore`         | `Array[String]`   | Collections to only ignore. Overrides watch array                                |
-| `options.autoListStream` | `Array[String]`   | Collections to automatically stream to clients. Default is all                   |
+| Parameter             | Type              | Description                                                                      |
+| --------------------- | ----------------- | -------------------------------------------------------------------------------- |
+| `options.dbUri`       | `String`\*        | Database URI                                                                     |
+| `options.dbOptions`   | `Object`          | Mongoose connection options                                                      |
+| `options.onDbConnect` | `Function`        | Callback on successful database connection                                       |
+| `options.onDbError`   | `Function`        | Callback on database connection error                                            |
+| `options.server`      | `http.Server`\*   | HTTP server to attach Socket.IO                                                  |
+| `options.authentify`  | `Function`        | Function to authenticate socket connections. Should return true if authenticated |
+| `options.middlewares` | `Array[Function]` | Array of Socket.IO middlewares                                                   |
+| `options.onSocket`    | `Function`        | Callback on socket connection                                                    |
+| `options.offSocket`   | `Function`        | Callback on socket disconnection                                                 |
+| `options.watch`       | `Array[String]`   | Collections to only watch. Listen to all when is empty                           |
+| `options.ignore`      | `Array[String]`   | Collections to only ignore. Overrides watch array                                |
+| `options.autoStream`  | `Array[String]`   | Collections to automatically stream to clients. Default is all                   |
+| `options.debug`       | `Boolean`         | Enable debug logs                                                                |
 
 #### Static Properties and Methods
 
-- `MongoRealtime.addListStream(streamId, collection, filter)`: Manually add a list stream for a specific collection and filter
+- `MongoRealtime.addStream(streamId, collection, filter)`: Manually add a list stream for a specific collection and filter
 - `MongoRealtime.connection`: MongoDB connection
 - `MongoRealtime.collections`: Array of database collections
 - `MongoRealtime.io`: Socket.IO server instance
@@ -103,15 +142,15 @@ The package automatically emits six types of events for each database change:
 
 ### Event Types
 
-| Event                         | Description       | Example                                    |
-| ----------------------------- | ----------------- | ------------------------------------------ |
-| `db:change`                   | All changes       | Any collection change                      |
-| `db:{type}`                   | By operation type | `db:insert`, `db:update`, `db:delete`      |
-| `db:change:{collection}`      | By collection     | `db:change:users`, `db:change:posts`       |
-| `db:{type}:{collection}`      | Type + collection | `db:insert:users`, `db:update:posts`       |
-| `db:change:{collection}:{id}` | Specific document | `db:change:users:507f1f77bcf86cd799439011` |
-| `db:{type}:{collection}:{id}` | Type + document   | `db:insert:users:507f1f77bcf86cd799439011` |
-| `db:stream:{streamId}`        | By stream         | `db:stream:myStreamId`                     |
+| Event                              | Description       | Example                                    |
+| ---------------------------------- | ----------------- | ------------------------------------------ |
+| `db:change`                        | All changes       | Any collection change                      |
+| `db:{type}`                        | By operation type | `db:insert`, `db:update`, `db:delete`      |
+| `db:change:{collection}`           | By collection     | `db:change:users`, `db:change:posts`       |
+| `db:{type}:{collection}`           | Type + collection | `db:insert:users`, `db:update:posts`       |
+| `db:change:{collection}:{id}`      | Specific document | `db:change:users:507f1f77bcf86cd799439011` |
+| `db:{type}:{collection}:{id}`      | Type + document   | `db:insert:users:507f1f77bcf86cd799439011` |
+| `realtime:{streamId}:{registerId}` | By stream         | `realtime:myStreamId:registerId`           |
 
 ### Event listeners
 
@@ -267,57 +306,47 @@ MongoRealtime.init({
 });
 ```
 
-### Setup list streams
+### Setup streams
 
 The server will automatically emit a list of filtered documents from the specified collections after each change.\
 Each list stream requires an unique `streamId`, the `collection` name, and an optional `filter` function that returns a boolean or a promise resolving to a boolean.
 Clients receive the list on the event `db:stream:{streamId}`.\
 
-On init, when `safeListStream` is `true`(default), two list streams can't have the same `streamId` or else an error will be thrown. This will prevent accidental overrides. You can still remove a stream with `MongoRealtime.removeListStream(streamId)` and add it again or use inside a `try-catch` scope.
-
 ```javascript
 MongoRealtime.init({
   dbUri: "mongodb://localhost:27017/mydb",
   server: server,
-  autoListStream: ["users"], // automatically stream users collection only
+  autoStream: ["users"], // automatically stream users collection only
 });
 
-MongoRealtime.addListStream("users", "users", (doc) => !!doc.email); // will throw an error as streamId 'users' already exists
+MongoRealtime.addStream("users", "users", (doc) => !!doc.email); // will throw an error as streamId 'users' already exists
 
-MongoRealtime.removeListStream("users"); // remove the previous stream
-MongoRealtime.addListStream("users", "users", (doc) => !!doc.email); // client can listen to db:stream:users
+MongoRealtime.removeStream("users"); // remove the previous stream
+MongoRealtime.addStream("users", "users", (doc) => !!doc.email); // client can listen to db:stream:users
 
-MongoRealtime.addListStream("usersWithEmail", "users", (doc) => !!doc.email); // client can listen to db:stream:usersWithEmail
+MongoRealtime.addStream("usersWithEmail", "users", (doc) => !!doc.email); // client can listen to db:stream:usersWithEmail
 ```
 
 #### ⚠️ NOTICE
 
-When `autoListStream` is not set, all collections are automatically streamed and WITHOUT any filter.\
+When `autoStream` is not set, all collections are automatically streamed and WITHOUT any filter.\
 That means that if you have a `posts` collection, all documents from this collection will be sent to the clients on each change.\
 
-Also, `safeListStream` is enabled by default. So, you can't add a list stream with id `posts` if `autoListStream` contains `"posts"` or is not set.\
-
-If you want to add a filtered list stream with id `posts`, you must set `autoListStream` to an array NOT containing `"posts"`or call `MongoRealtime.removeListStream("posts")` after initialization.\
-Therefore, if you want to add a filtered list stream for all collections, you must set `autoListStream` to an empty array.
-
-To avoid all these issues, you can set `safeListStream` to `false` in the init options but be careful as this will allow you to override existing streams.
-
 ```javascript
 MongoRealtime.init({
   dbUri: "mongodb://localhost:27017/mydb",
   server: server,
-  autoListStream: [], // stream no collection automatically (you can add your own filtered streams later)
+  autoStream: [], // stream no collection automatically (you can add your own filtered streams later)
 });
 // or
 MongoRealtime.init({
   dbUri: "mongodb://localhost:27017/mydb",
   server: server,
-  safeListStream: false, // disable safe mode (you can override existing streams)
-  // Still stream all collections automatically but you can override them
+  // Stream all collections automatically but you can override them
 }):
 
-MongoRealtime.addListStream("posts", "posts", (doc) => !!doc.title); // client can listen to db:stream:posts
-MongoRealtime.addListStream("users", "users", (doc) => !!doc.email); // will not throw an error
+MongoRealtime.addStream("postsWithTitle", "posts", (doc) => !!doc.title); // client can listen to db:stream:posts
+MongoRealtime.addStream("users", "users", (doc) => !!doc.email); // will not throw an error cause 'users' is already streamed but with no filter
 ```
 
 #### Usecase for id based streams
@@ -336,7 +365,7 @@ MongoRealtime.init({
   },
   onSocket: (socket) => {
     // setup a personal stream for each connected user
-    MongoRealtime.addListStream(
+    MongoRealtime.addStream(
       `userPost:${socket.uid}`,
       "posts",
       (doc) => doc._id == socket.uid
@@ -344,7 +373,7 @@ MongoRealtime.init({
   },
   offSocket: (socket) => {
     // clean up when user disconnects
-    MongoRealtime.removeListStream(`userPost:${socket.uid}`);
+    MongoRealtime.removeStream(`userPost:${socket.uid}`);
   },
 });
 
@@ -352,31 +381,27 @@ MongoRealtime.init({
 // or activate stream from a controller or middleware
 app.get("/my-posts", (req, res) => {
   const { user } = req;
-  try {
-    MongoRealtime.addListStream(
-      `userPosts:${user._id}`,
-      "posts",
-      (doc) => doc.authorId === user._id
-    );
-  } catch (e) {
-    // stream already exists
-  }
+  MongoRealtime.addStream(
+    `userPosts:${user._id}`,
+    "posts",
+    (doc) => doc.authorId === user._id
+  );
 
   res.send("Stream activated");
 });
 ```
 
-#### Usecase with async filter
+### Usecase with async filter
 
 ```javascript
 // MongoRealtime.init({...});
 
-MongoRealtime.addListStream("authorizedUsers", "users", async (doc) => {
+MongoRealtime.addStream("authorizedUsers", "users", async (doc) => {
   const isAdmin = await UserService.isAdmin(doc._id);
   return isAdmin && doc.email.endsWith("@mydomain.com");
 });
 
-MongoRealtime.addListStream(
+MongoRealtime.addStream(
   "bestPosts",
   "posts",
   async (doc) => doc.likes > (await PostService.getLikesThreshold())
@@ -400,6 +425,8 @@ mongod --replSet rs0
 rs.initiate()
 ```
 
+For any other issues, open an issue on the GitHub repository.
+
 ## 📄 License
 
 MIT

package/index.js

@@ -35,17 +35,19 @@ class MongoRealtime {
     mongoose.connection;
   /** @type {Record<String, [(change:ChangeStreamDocument)=>void]>} */ static #listeners =
     {};
-  /** @type {Record<String, [Object]>} */
-  static #cache = {};
+  
   static sockets = () => [...this.io.sockets.sockets.values()];
 
   /**@type {Record<String, {collection:String,filter: (doc:Object)=>Promise<boolean>}>} */
   static #streams = {};
 
+  /**@type {Record<String, {expiration:Date,result:Record<String,{}> }>} */
+  static #data = {};
+
   /** @type {[String]} - All DB collections */
   static collections = [];
 
-  static #safeListStream = true;
+  static #debug = false;
 
   static version = version;
 
@@ -69,7 +71,7 @@ class MongoRealtime {
     const text = `[REALTIME] ${message}`;
     switch (type) {
       case 1:
-        console.log(chalk.bold.hex('#11AA60FF')(text));
+        console.log(chalk.bold.hex("#11AA60FF")(text));
         break;
       case 2:
         console.log(chalk.bold.bgHex("#257993")(text));
@@ -81,12 +83,20 @@ class MongoRealtime {
         console.log(chalk.bold.red(text));
         break;
 
+      case 5:
+        console.log(chalk.italic(text));
+        break;
+
       default:
         console.log(text);
         break;
     }
   }
 
+  static #debugLog(message) {
+    if (this.#debug) this.#log("[DEBUG] " + message, 5);
+  }
+
   /**
    * Initializes the socket system.
    *
@@ -100,10 +110,10 @@ class MongoRealtime {
    * @param {(socket: import("socket.io").Socket) => void} options.onSocket - Callback triggered when a socket connects
    * @param {(socket: import("socket.io").Socket, reason: import("socket.io").DisconnectReason) => void} options.offSocket - Callback triggered when a socket disconnects
    * @param {import("http").Server} options.server - HTTP server to attach Socket.IO to
-   * @param {[String]} options.autoListStream - Collections to stream automatically. If empty, will stream no collection. If null, will stream all collections.
+   * @param {[String]} options.autoStream - Collections to stream automatically. If empty, will stream no collection. If null, will stream all collections.
    * @param {[String]} options.watch - Collections to watch. If empty, will watch all collections
    * @param {[String]} options.ignore - Collections to ignore. Can override `watch`
-   * @param {bool} options.safeListStream -  If true(default), declaring an existing streamId will throw an error
+   * @param {bool} options.debug -  Enable debug mode
    *
    */
   static async init({
@@ -115,23 +125,29 @@ class MongoRealtime {
     authentify,
     onSocket,
     offSocket,
-    safeListStream = true,
-    autoListStream,
+    debug = false,
+    autoStream,
     middlewares = [],
     watch = [],
     ignore = [],
   }) {
-    console.clear();
     this.#log(`MongoRealtime version (${this.version})`, 2);
+
     if (this.io) this.io.close();
     this.#check(() => dbUri);
     this.#check(() => server);
+    this.#debug = debug;
 
     this.io = new Server(server);
     this.connection.once("open", async () => {
       this.collections = (await this.connection.listCollections()).map(
         (c) => c.name
       );
+      this.#debugLog(
+        `${this.collections.length} collections found : ${this.collections.join(
+          ", "
+        )}`
+      );
 
       let pipeline = [];
       if (watch.length !== 0 && ignore.length === 0) {
@@ -158,70 +174,56 @@ class MongoRealtime {
 
       /** Setup main streams */
       let collectionsToStream = [];
-      if (autoListStream == null) collectionsToStream = this.collections;
+      if (autoStream == null) collectionsToStream = this.collections;
       else
         collectionsToStream = this.collections.filter((c) =>
-          autoListStream.includes(c)
+          autoStream.includes(c)
         );
-      for (let col of collectionsToStream) this.addListStream(col, col);
+      for (let col of collectionsToStream) this.addStream(col, col);
+      this.#debugLog(
+        `Auto stream on collections  : ${collectionsToStream.join(", ")}`
+      );
 
-      /** Emit streams on change */
+      /** Emit listen events on change */
       changeStream.on("change", async (change) => {
         const coll = change.ns.coll;
+        const colName = coll.toLowerCase();
+        const doc = change.fullDocument;
 
-        if (!this.#cache[coll]) {
-          this.#cache[coll] = await this.connection.db
-            .collection(coll)
-            .find({})
-            .sort({ _id: -1 })
-            .toArray();
-        } else
-          switch (change.operationType) {
-            case "insert":
-              this.#cache[coll].unshift(change.fullDocument); // add to top;
-              break;
+        this.#debugLog(`Collection '${colName}' changed`);
 
-            case "update":
-            case "replace":
-              this.#cache[coll] = this.#cache[coll].map((doc) =>
-                doc._id.toString() === change.documentKey._id.toString()
-                  ? change.fullDocument
-                  : doc
-              );
-              break;
+        change.col = colName;
 
+        const type = change.operationType;
+        const id = change.documentKey?._id.toString();
+
+        for (const k in this.#streams) {
+          const stream = this.#streams[k];
+          if (stream.collection != coll) continue;
+
+          Promise.resolve(stream.filter(doc)).then((ok) => {
+            if (ok) {
+              this.io.emit(`realtime:${k}`, {
+                results: [doc],
+                coll: coll,
+                count: 1,
+                total: 1,
+                remaining: 0,
+              });
+            }
+          });
+        }
+        for (let k in this.#data) {
+          if (!k.startsWith(`${coll}-`) || !this.#data[k].result[id]) continue;
+          doc._id = doc._id.toString();
+          switch (change.operationType) {
             case "delete":
-              this.#cache[coll] = this.#cache[coll].filter(
-                (doc) =>
-                  doc._id.toString() !== change.documentKey._id.toString()
-              );
+              delete this.#data[k].result[id];
               break;
+            default:
+              this.#data[k].result[id] = doc;
           }
-
-        Object.entries(this.#streams).forEach(async (e) => {
-          const key = e[0];
-          const value = e[1];
-          if (value.collection != coll) return;
-          const filterResults = await Promise.allSettled(
-            this.#cache[coll].map((doc) => value.filter(doc))
-          );
-
-          const filtered = this.#cache[coll].filter(
-            (_, i) => filterResults[i] && filterResults[i].value
-          );
-
-          this.io.emit(`db:stream:${key}`, filtered);
-          this.notifyListeners(`db:stream:${key}`, filtered);
-        });
-      });
-
-      /** Emit listen events on change */
-      changeStream.on("change", async (change) => {
-        const colName = change.ns.coll.toLowerCase();
-        change.col = colName;
-
-        const type = change.operationType;
-        const id = change.documentKey?._id;
+        }
 
         const e_change = "db:change";
         const e_change_type = `db:${type}`;
@@ -254,14 +256,12 @@ class MongoRealtime {
       onDbConnect?.call(this, mongoose.connection);
     } catch (error) {
       onDbError?.call(this, error);
-      this.#log("Failed to init",4);
+      this.#log("Failed to init", 4);
       return;
     }
 
     this.#check(() => mongoose.connection.db, "No database found");
 
-    this.#safeListStream = !!safeListStream;
-
     watch = watch.map((s) => s.toLowerCase());
     ignore = ignore.map((s) => s.toLowerCase());
 
@@ -291,37 +291,121 @@ class MongoRealtime {
 
     this.io.on("connection", (socket) => {
       socket.emit("version", version);
-      socket.on("db:stream[register]", async (streamId, registerId) => {
-        const stream = this.#streams[streamId];
-        if (!stream) return;
-        const coll = stream.collection;
-        if (!this.#cache[coll]) {
-          this.#cache[coll] = await this.connection.db
-            .collection(coll)
-            .find({})
-            .sort({ _id: -1 })
-            .toArray();
-        }
-        const filterResults = await Promise.allSettled(
-          this.#cache[coll].map((doc) => stream.filter(doc))
-        );
 
-        const filtered = this.#cache[coll].filter(
-          (_, i) => filterResults[i] && filterResults[i].value
-        );
-        this.io.emit(`db:stream[register][${registerId}]`, filtered);
-      });
+      socket.on(
+        "realtime",
+        async ({ streamId, limit, reverse, registerId }) => {
+          if (!streamId || !this.#streams[streamId]) return;
+
+          const stream = this.#streams[streamId];
+          const coll = stream.collection;
+
+          const default_limit = 50;
+          limit ??= default_limit;
+          try {
+            limit = parseInt(limit);
+          } catch (_) {
+            limit = default_limit;
+          }
+
+          reverse = reverse == true;
+          registerId ??= "";
+          this.#debugLog(
+            `Socket ${socket.id} registred for realtime '${coll}:${registerId}'. Limit ${limit}. Reversed ${reverse}`
+          );
+
+          let total;
+          const ids = [];
+
+          do {
+            total = await this.connection.db
+              .collection(coll)
+              .estimatedDocumentCount();
+
+            const length = ids.length;
+            const range = [length, Math.min(total, length + limit)];
+            const now = new Date();
+
+            let cachedKey = `${coll}-${range}`;
+
+            let cachedResult = this.#data[cachedKey];
+            if (cachedResult && cachedResult.expiration < now) {
+              delete this.#data[cachedKey];
+            }
+
+            cachedResult = this.#data[cachedKey];
+
+            const result = cachedResult
+              ? Object.values(cachedResult.result)
+              : await this.connection.db
+                  .collection(coll)
+                  .find({
+                    _id: {
+                      $nin: ids,
+                    },
+                  })
+                  .limit(limit)
+                  .sort({ _id: reverse ? -1 : 1 })
+                  .toArray();
+
+            ids.push(...result.map((d) => d._id));
+
+            const delayInMin = 1;
+            const expiration = new Date(now.getTime() + delayInMin * 60 * 1000);
+            const resultMap = result.reduce((acc, item) => {
+              item._id = item._id.toString();
+              acc[item._id] = item;
+              return acc;
+            }, {});
+
+            if (!cachedResult) {
+              this.#data[cachedKey] = {
+                expiration,
+                result: resultMap,
+              };
+            }
+
+            const filtered = (
+              await Promise.all(
+                result.map(async (doc) => {
+                  try {
+                    return {
+                      doc,
+                      ok: await stream.filter(doc),
+                    };
+                  } catch (e) {
+                    return {
+                      doc,
+                      ok: false,
+                    };
+                  }
+                })
+              )
+            )
+              .filter((item) => item.ok)
+              .map((item) => item.doc);
+
+            const data = {
+              results: filtered,
+              coll,
+              count: filtered.length,
+              total,
+              remaining: total - ids.length,
+            };
+
+            socket.emit(`realtime:${streamId}:${registerId}`, data);
+          } while (ids.length < total);
+        }
+      );
 
       socket.on("disconnect", (r) => {
         if (offSocket) offSocket(socket, r);
       });
 
       if (onSocket) onSocket(socket);
-
     });
-    
 
-    this.#log(`Initialized`,1);
+    this.#log(`Initialized`, 1);
   }
 
   /**
@@ -357,16 +441,14 @@ class MongoRealtime {
    *
    * Register a new list stream to listen
    */
-  static addListStream(streamId, collection, filter) {
+  static addStream(streamId, collection, filter) {
     if (!streamId) throw new Error("Stream id is required");
     if (!collection) throw new Error("Collection is required");
+    if (this.#streams[streamId] && this.collections.includes(streamId))
+      throw new Error(`streamId '${streamId}' cannot be a collection`);
 
     filter ??= (_, __) => true;
-    if (this.#safeListStream && this.#streams[streamId]) {
-      throw new Error(
-        `Stream '${streamId}' already registered or is reserved.`
-      );
-    }
+
     this.#streams[streamId] = {
       collection,
       filter,
@@ -378,7 +460,7 @@ class MongoRealtime {
    *
    * Delete a registered stream
    */
-  static removeListStream(streamId) {
+  static removeStream(streamId) {
     delete this.#streams[streamId];
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mongo-realtime",
-  "version": "1.2.0",
+  "version": "2.0.0",
   "main": "index.js",
   "scripts": {},
   "keywords": [