mongo-realtime 1.1.3 → 2.0.0

package/README.md

@@ -29,18 +29,19 @@ npm install mongo-realtime
 ```javascript
 const express = require("express");
 const http = require("http");
-const mongoose = require("mongoose");
 const MongoRealtime = require("mongo-realtime");
 
 const app = express();
 const server = http.createServer(app);
 
-mongoose.connect("mongodb://localhost:27017/mydb").then((c) => {
-  console.log("Connected to db", c.connection.name);
-});
-
 MongoRealtime.init({
-  connection: mongoose.connection,
+  dbUri: "mongodb://localhost:27017/mydb",
+  onDbConnect: (conn) => {
+    console.log("Connected to db", conn.name);
+  },
+  onDbError: (err) => {
+    console.log(err.message);
+  },
   server: server,
   ignore: ["posts"], // ignore 'posts' collection
   onSocket: (socket) => {
@@ -57,6 +58,44 @@ server.listen(3000, () => {
 });
 ```
 
+## Breaking Changes since v2.x.x
+
+In older versions, streams receive all documents and send them to sockets. This can lead to performance issues with large collections.\
+From version 2.x.x, streams now require a limit (default 50 documents) from the client side to avoid overloading the server and network.\
+Receiving streamed docs now works like this:
+
+```javascript
+socket.emit("realtime", {
+  limit: 100,
+  streamId: "users",
+  reverse: true,
+  registerId: "random-client-generated-id",
+});
+/*
+ * Request 100 documents from 'users' stream (streamId defined server-side or collection name if autoStream)
+ * 'reverse' to get the latest documents first
+ * 'registerId' is a unique id generated by the client to identify this request. It will be used to listen to the response event.
+ * The response return an object {results: [...], count: number, total:number, remaining:number, coll:string}
+ */
+
+socket.on("realtime:users:random-client-generated-id", (data) => {
+  console.log("Received streamed documents:", data.results); // 100 max per
+});
+
+socket.emit("realtime", {
+  limit: 50,
+  streamId: "usersWithEmail", // make sure this stream is defined server-side
+  registerId: "specific-request-id",
+});
+
+socket.on("realtime:usersWithEmail:specific-request-id", (data) => {
+  console.log("Received streamed documents with email:", data.results); // 50 max per
+});
+
+// Make sure to listen before emitting the request to avoid missing the response
+// To stop receiving streamed documents for a specific request just use socket.off
+```
+
 ## 📋 API
 
 ### `MongoRealtime.init(options)`
@@ -67,21 +106,25 @@ Initializes the socket system and MongoDB Change Streams.
 
 \* means required
 
-| Parameter                | Type                    | Description                                                                      |
-| ------------------------ | ----------------------- | -------------------------------------------------------------------------------- |
-| `options.connection`     | `mongoose.Connection`\* | Active Mongoose connection                                                       |
-| `options.server`         | `http.Server`\*         | HTTP server to attach Socket.IO                                                  |
-| `options.authentify`     | `Function`              | Function to authenticate socket connections. Should return true if authenticated |
-| `options.middlewares`    | `Array[Function]`       | Array of Socket.IO middlewares                                                   |
-| `options.onSocket`       | `Function`              | Callback on socket connection                                                    |
-| `options.offSocket`      | `Function`              | Callback on socket disconnection                                                 |
-| `options.watch`          | `Array[String]`         | Collections to only watch. Listen to all when is empty                           |
-| `options.ignore`         | `Array[String]`         | Collections to only ignore. Overrides watch array                                |
-| `options.autoListStream` | `Array[String]`         | Collections to automatically stream to clients. Default is all                   |
+| Parameter             | Type              | Description                                                                      |
+| --------------------- | ----------------- | -------------------------------------------------------------------------------- |
+| `options.dbUri`       | `String`\*        | Database URI                                                                     |
+| `options.dbOptions`   | `Object`          | Mongoose connection options                                                      |
+| `options.onDbConnect` | `Function`        | Callback on successful database connection                                       |
+| `options.onDbError`   | `Function`        | Callback on database connection error                                            |
+| `options.server`      | `http.Server`\*   | HTTP server to attach Socket.IO                                                  |
+| `options.authentify`  | `Function`        | Function to authenticate socket connections. Should return true if authenticated |
+| `options.middlewares` | `Array[Function]` | Array of Socket.IO middlewares                                                   |
+| `options.onSocket`    | `Function`        | Callback on socket connection                                                    |
+| `options.offSocket`   | `Function`        | Callback on socket disconnection                                                 |
+| `options.watch`       | `Array[String]`   | Collections to only watch. Listen to all when is empty                           |
+| `options.ignore`      | `Array[String]`   | Collections to only ignore. Overrides watch array                                |
+| `options.autoStream`  | `Array[String]`   | Collections to automatically stream to clients. Default is all                   |
+| `options.debug`       | `Boolean`         | Enable debug logs                                                                |
 
 #### Static Properties and Methods
 
-- `MongoRealtime.addListStream(streamId, collection, filter)`: Manually add a list stream for a specific collection and filter
+- `MongoRealtime.addStream(streamId, collection, filter)`: Manually add a list stream for a specific collection and filter
 - `MongoRealtime.connection`: MongoDB connection
 - `MongoRealtime.collections`: Array of database collections
 - `MongoRealtime.io`: Socket.IO server instance
@@ -99,15 +142,15 @@ The package automatically emits six types of events for each database change:
 
 ### Event Types
 
-| Event                         | Description       | Example                                    |
-| ----------------------------- | ----------------- | ------------------------------------------ |
-| `db:change`                   | All changes       | Any collection change                      |
-| `db:{type}`                   | By operation type | `db:insert`, `db:update`, `db:delete`      |
-| `db:change:{collection}`      | By collection     | `db:change:users`, `db:change:posts`       |
-| `db:{type}:{collection}`      | Type + collection | `db:insert:users`, `db:update:posts`       |
-| `db:change:{collection}:{id}` | Specific document | `db:change:users:507f1f77bcf86cd799439011` |
-| `db:{type}:{collection}:{id}` | Type + document   | `db:insert:users:507f1f77bcf86cd799439011` |
-| `db:stream:{streamId}`        | By stream         | `db:stream:myStreamId`                     |
+| Event                              | Description       | Example                                    |
+| ---------------------------------- | ----------------- | ------------------------------------------ |
+| `db:change`                        | All changes       | Any collection change                      |
+| `db:{type}`                        | By operation type | `db:insert`, `db:update`, `db:delete`      |
+| `db:change:{collection}`           | By collection     | `db:change:users`, `db:change:posts`       |
+| `db:{type}:{collection}`           | Type + collection | `db:insert:users`, `db:update:posts`       |
+| `db:change:{collection}:{id}`      | Specific document | `db:change:users:507f1f77bcf86cd799439011` |
+| `db:{type}:{collection}:{id}`      | Type + document   | `db:insert:users:507f1f77bcf86cd799439011` |
+| `realtime:{streamId}:{registerId}` | By stream         | `realtime:myStreamId:registerId`           |
 
 ### Event listeners
 
@@ -160,7 +203,7 @@ Each event contains the full MongoDB change object:
 
 ```javascript
 MongoRealtime.init({
-  connection: connection,
+  dbUri: "mongodb://localhost:27017/mydb",
   server: server,
   onSocket: (socket) => {
     socket.on("subscribe:users", () => {
@@ -209,7 +252,7 @@ MongoRealtime.io.to("users-room").emit("custom-event", data);
 
 ```javascript
 MongoRealtime.init({
-  connection: mongoose.connection,
+  dbUri: "mongodb://localhost:27017/mydb",
   server: server,
   onSocket: (socket) => {
     socket.on("error", (error) => {
@@ -228,6 +271,15 @@ MongoRealtime.init({
 
 ### Socket Authentication
 
+You can provide an `authentify` function in the init options to authenticate socket connections.\
+The function receives the token (from `socket.handshake.auth.token` or `socket.handshake.headers.authorization`) and the socket object.\
+When setted, it rejects connections based on this logic:
+
+- Token not provided -> error `NO_TOKEN_PROVIDED`
+- Token invalid or returns `false` -> error `UNAUTHORIZED`
+- Any other error -> error `AUTH_ERROR`
+- Return `true` to accept the connection
+
 ```javascript
 function authenticateSocket(token, socket) {
   const verify = AuthService.verifyToken(token);
@@ -239,7 +291,7 @@ function authenticateSocket(token, socket) {
 }
 
 MongoRealtime.init({
-  connection: mongoose.connection,
+  dbUri: "mongodb://localhost:27017/mydb",
   server: server,
   authentify: authenticateSocket,
   middlewares: [
@@ -254,64 +306,54 @@ MongoRealtime.init({
 });
 ```
 
-### Setup list streams
+### Setup streams
 
 The server will automatically emit a list of filtered documents from the specified collections after each change.\
 Each list stream requires an unique `streamId`, the `collection` name, and an optional `filter` function that returns a boolean or a promise resolving to a boolean.
 Clients receive the list on the event `db:stream:{streamId}`.\
 
-On init, when `safeListStream` is `true`(default), two list streams can't have the same `streamId` or else an error will be thrown. This will prevent accidental overrides. You can still remove a stream with `MongoRealtime.removeListStream(streamId)` and add it again or use inside a `try-catch` scope.
-
 ```javascript
 MongoRealtime.init({
-  connection: mongoose.connection,
+  dbUri: "mongodb://localhost:27017/mydb",
   server: server,
-  autoListStream: ["users"], // automatically stream users collection only
+  autoStream: ["users"], // automatically stream users collection only
 });
 
-MongoRealtime.addListStream("users", "users", (doc) => !!doc.email); // will throw an error as streamId 'users' already exists
+MongoRealtime.addStream("users", "users", (doc) => !!doc.email); // will throw an error as streamId 'users' already exists
 
-MongoRealtime.removeListStream("users"); // remove the previous stream
-MongoRealtime.addListStream("users", "users", (doc) => !!doc.email); // client can listen to db:stream:users
+MongoRealtime.removeStream("users"); // remove the previous stream
+MongoRealtime.addStream("users", "users", (doc) => !!doc.email); // client can listen to db:stream:users
 
-MongoRealtime.addListStream("usersWithEmail", "users", (doc) => !!doc.email); // client can listen to db:stream:usersWithEmail
+MongoRealtime.addStream("usersWithEmail", "users", (doc) => !!doc.email); // client can listen to db:stream:usersWithEmail
 ```
 
 #### ⚠️ NOTICE
 
-When `autoListStream` is not set, all collections are automatically streamed and WITHOUT any filter.\
+When `autoStream` is not set, all collections are automatically streamed and WITHOUT any filter.\
 That means that if you have a `posts` collection, all documents from this collection will be sent to the clients on each change.\
 
-Also, `safeListStream` is enabled by default. So, you can't add a list stream with id `posts` if `autoListStream` contains `"posts"` or is not set.\
-
-If you want to add a filtered list stream with id `posts`, you must set `autoListStream` to an array NOT containing `"posts"`or call `MongoRealtime.removeListStream("posts")` after initialization.\
-Therefore, if you want to add a filtered list stream for all collections, you must set `autoListStream` to an empty array.
-
-To avoid all these issues, you can set `safeListStream` to `false` in the init options but be careful as this will allow you to override existing streams.
-
 ```javascript
 MongoRealtime.init({
-  connection: mongoose.connection,
+  dbUri: "mongodb://localhost:27017/mydb",
   server: server,
-  autoListStream: [], // stream no collection automatically (you can add your own filtered streams later)
+  autoStream: [], // stream no collection automatically (you can add your own filtered streams later)
 });
 // or
 MongoRealtime.init({
-  connection: mongoose.connection,
+  dbUri: "mongodb://localhost:27017/mydb",
   server: server,
-  safeListStream: false, // disable safe mode (you can override existing streams)
-  // Still stream all collections automatically but you can override them 
+  // Stream all collections automatically but you can override them
 }):
 
-MongoRealtime.addListStream("posts", "posts", (doc) => !!doc.title); // client can listen to db:stream:posts
-MongoRealtime.addListStream("users", "users", (doc) => !!doc.email); // will not throw an error
+MongoRealtime.addStream("postsWithTitle", "posts", (doc) => !!doc.title); // client can listen to db:stream:posts
+MongoRealtime.addStream("users", "users", (doc) => !!doc.email); // will not throw an error cause 'users' is already streamed but with no filter
 ```
 
 #### Usecase for id based streams
 
 ```javascript
 MongoRealtime.init({
-  connection: mongoose.connection,
+  dbUri: "mongodb://localhost:27017/mydb",
   server: server,
   authentify: (token, socket) => {
     try {
@@ -323,7 +365,7 @@ MongoRealtime.init({
   },
   onSocket: (socket) => {
     // setup a personal stream for each connected user
-    MongoRealtime.addListStream(
+    MongoRealtime.addStream(
       `userPost:${socket.uid}`,
       "posts",
       (doc) => doc._id == socket.uid
@@ -331,7 +373,7 @@ MongoRealtime.init({
   },
   offSocket: (socket) => {
     // clean up when user disconnects
-    MongoRealtime.removeListStream(`userPost:${socket.uid}`);
+    MongoRealtime.removeStream(`userPost:${socket.uid}`);
   },
 });
 
@@ -339,31 +381,27 @@ MongoRealtime.init({
 // or activate stream from a controller or middleware
 app.get("/my-posts", (req, res) => {
   const { user } = req;
-  try {
-    MongoRealtime.addListStream(
-      `userPosts:${user._id}`,
-      "posts",
-      (doc) => doc.authorId === user._id
-    );
-  } catch (e) {
-    // stream already exists
-  }
+  MongoRealtime.addStream(
+    `userPosts:${user._id}`,
+    "posts",
+    (doc) => doc.authorId === user._id
+  );
 
   res.send("Stream activated");
 });
 ```
 
-#### Usecase with async filter
+### Usecase with async filter
 
 ```javascript
 // MongoRealtime.init({...});
 
-MongoRealtime.addListStream("authorizedUsers", "users", async (doc) => {
+MongoRealtime.addStream("authorizedUsers", "users", async (doc) => {
   const isAdmin = await UserService.isAdmin(doc._id);
   return isAdmin && doc.email.endsWith("@mydomain.com");
 });
 
-MongoRealtime.addListStream(
+MongoRealtime.addStream(
   "bestPosts",
   "posts",
   async (doc) => doc.likes > (await PostService.getLikesThreshold())
@@ -379,12 +417,16 @@ MongoRealtime.addListStream(
 
 ### MongoDB must be in Replica Set mode
 
+To use Change Streams, MongoDB must be running as a replica set. For local development, you can initiate a single-node replica set:
+
 ```bash
 mongod --replSet rs0
 
 rs.initiate()
 ```
 
+For any other issues, open an issue on the GitHub repository.
+
 ## 📄 License
 
 MIT

package/index.js

@@ -1,14 +1,7 @@
+const mongoose = require("mongoose");
 const { Server } = require("socket.io");
-
-function sortObj(obj = {}) {
-  const out = {};
-  for (let k of Object.keys(obj).sort()) {
-    const v = obj[k];
-    out[k] = typeof v == "object" && !Array.isArray(v) ? sortObj(v) : v;
-  }
-
-  return out;
-}
+const { version } = require("./package.json");
+const chalk = require("chalk");
 
 /**
  * @typedef {Object} ChangeStreamDocument
@@ -38,114 +31,123 @@ function sortObj(obj = {}) {
 
 class MongoRealtime {
   /** @type {import("socket.io").Server} */ static io;
-  /** @type {import("mongoose").Connection} */ static connection;
+  /** @type {import("mongoose").Connection} */ static connection =
+    mongoose.connection;
   /** @type {Record<String, [(change:ChangeStreamDocument)=>void]>} */ static #listeners =
     {};
-  /** @type {Record<String, [Object]>} */
-  static #cache = {};
+  
   static sockets = () => [...this.io.sockets.sockets.values()];
 
   /**@type {Record<String, {collection:String,filter: (doc:Object)=>Promise<boolean>}>} */
   static #streams = {};
 
+  /**@type {Record<String, {expiration:Date,result:Record<String,{}> }>} */
+  static #data = {};
+
   /** @type {[String]} - All DB collections */
   static collections = [];
 
-  static #safeListStream = true;
+  static #debug = false;
+
+  static version = version;
+
+  static #check(fn, err) {
+    const result = fn();
+    if (!result) {
+      let src = fn.toString().trim();
+
+      let match =
+        src.match(/=>\s*([^{};]+)$/) ||
+        src.match(/return\s+([^;}]*)/) ||
+        src.match(/{([^}]*)}/);
+
+      const expr = err ?? (match ? match[1].trim() : src);
+
+      throw new Error(`MongoRealtime failed to check "${expr}"`);
+    }
+  }
+
+  static #log(message, type = 0) {
+    const text = `[REALTIME] ${message}`;
+    switch (type) {
+      case 1:
+        console.log(chalk.bold.hex("#11AA60FF")(text));
+        break;
+      case 2:
+        console.log(chalk.bold.bgHex("#257993")(text));
+        break;
+      case 3:
+        console.log(chalk.bold.yellow(text));
+        break;
+      case 4:
+        console.log(chalk.bold.red(text));
+        break;
+
+      case 5:
+        console.log(chalk.italic(text));
+        break;
+
+      default:
+        console.log(text);
+        break;
+    }
+  }
+
+  static #debugLog(message) {
+    if (this.#debug) this.#log("[DEBUG] " + message, 5);
+  }
 
   /**
    * Initializes the socket system.
    *
    * @param {Object} options
-   * @param {import("mongoose").Connection} options.connection - Active Mongoose connection
+   * @param {String} options.dbUri - Database URI
+   * @param {mongoose.ConnectOptions | undefined} options.dbOptions - Database connect options
    * @param {(token:String, socket: import("socket.io").Socket) => boolean | Promise<boolean>} options.authentify - Auth function that should return true if `token` is valid
    * @param {[( socket: import("socket.io").Socket, next: (err?: ExtendedError) => void) => void]} options.middlewares - Register mmiddlewares on incoming socket
+   * @param {(conn:mongoose.Connection) => void} options.onDbConnect - Callback triggered when a socket connects
+   * @param {(err:Error) => void} options.onDbError - Callback triggered when a socket connects
    * @param {(socket: import("socket.io").Socket) => void} options.onSocket - Callback triggered when a socket connects
    * @param {(socket: import("socket.io").Socket, reason: import("socket.io").DisconnectReason) => void} options.offSocket - Callback triggered when a socket disconnects
    * @param {import("http").Server} options.server - HTTP server to attach Socket.IO to
-   * @param {[String]} options.autoListStream - Collections to stream automatically. If empty, will stream no collection. If null, will stream all collections.
+   * @param {[String]} options.autoStream - Collections to stream automatically. If empty, will stream no collection. If null, will stream all collections.
    * @param {[String]} options.watch - Collections to watch. If empty, will watch all collections
    * @param {[String]} options.ignore - Collections to ignore. Can override `watch`
-   * @param {bool} options.safeListStream -  If true(default), declaring an existing streamId will throw an error
+   * @param {bool} options.debug -  Enable debug mode
    *
    */
-  static init({
-    connection,
+  static async init({
+    dbUri,
+    dbOptions,
     server,
+    onDbConnect,
+    onDbError,
     authentify,
-    middlewares = [],
-    autoListStream,
     onSocket,
     offSocket,
-    safeListStream = true,
+    debug = false,
+    autoStream,
+    middlewares = [],
     watch = [],
     ignore = [],
   }) {
-    if (this.io) this.io.close();
-    this.io = new Server(server);
-    this.connection = connection;
-    this.#safeListStream = !!safeListStream;
-
-    watch = watch.map((s) => s.toLowerCase());
-    ignore = ignore.map((s) => s.toLowerCase());
-
-    this.io.use(async (socket, next) => {
-      if (!!authentify) {
-        try {
-          const token =
-            socket.handshake.auth.token ||
-            socket.handshake.headers.authorization;
-          if (!token) return next(new Error("NO_TOKEN_PROVIDED"));
-
-          const authorized = await authentify(token, socket);
-          if (authorized === true) return next(); // exactly returns true
-
-          return next(new Error("UNAUTHORIZED"));
-        } catch (error) {
-          return next(new Error("AUTH_ERROR"));
-        }
-      } else {
-        return next();
-      }
-    });
-
-    for (let middleware of middlewares) {
-      this.io.use(middleware);
-    }
-
-    this.io.on("connection", (socket) => {
-      if (onSocket) onSocket(socket);
+    this.#log(`MongoRealtime version (${this.version})`, 2);
 
-      socket.on("db:stream[register]", async (streamId, registerId) => {
-        const stream = this.#streams[streamId];
-        if (!stream) return;
-        const coll = stream.collection;
-
-        if (!this.#cache[coll]) {
-          this.#cache[coll] = await connection.db
-            .collection(coll)
-            .find({})
-            .toArray();
-        }
-        const filterResults = await Promise.allSettled(
-          this.#cache[coll].map((doc) => stream.filter(doc))
-        );
-
-        const filtered = this.#cache[coll].filter(
-          (_, i) => filterResults[i] && filterResults[i].value
-        );
-        this.io.emit(`db:stream[register][${registerId}]`, filtered);
-      });
-
-      socket.on("disconnect", (r) => {
-        if (offSocket) offSocket(socket, r);
-      });
-    });
+    if (this.io) this.io.close();
+    this.#check(() => dbUri);
+    this.#check(() => server);
+    this.#debug = debug;
 
-    connection.once("open", async () => {
-      this.collections = (await connection.listCollections()).map(
+    this.io = new Server(server);
+    this.connection.once("open", async () => {
+      this.collections = (await this.connection.listCollections()).map(
         (c) => c.name
       );
+      this.#debugLog(
+        `${this.collections.length} collections found : ${this.collections.join(
+          ", "
+        )}`
+      );
 
       let pipeline = [];
       if (watch.length !== 0 && ignore.length === 0) {
@@ -165,76 +167,63 @@ class MongoRealtime {
         ];
       }
 
-      const changeStream = connection.watch(pipeline, {
+      const changeStream = this.connection.watch(pipeline, {
         fullDocument: "updateLookup",
         fullDocumentBeforeChange: "whenAvailable",
       });
 
       /** Setup main streams */
       let collectionsToStream = [];
-      if (autoListStream == null) collectionsToStream = this.collections;
+      if (autoStream == null) collectionsToStream = this.collections;
       else
         collectionsToStream = this.collections.filter((c) =>
-          autoListStream.includes(c)
+          autoStream.includes(c)
         );
-      for (let col of collectionsToStream) this.addListStream(col, col);
+      for (let col of collectionsToStream) this.addStream(col, col);
+      this.#debugLog(
+        `Auto stream on collections  : ${collectionsToStream.join(", ")}`
+      );
 
-      /** Emit streams on change */
+      /** Emit listen events on change */
       changeStream.on("change", async (change) => {
         const coll = change.ns.coll;
+        const colName = coll.toLowerCase();
+        const doc = change.fullDocument;
 
-        if (!this.#cache[coll]) {
-          this.#cache[coll] = await connection.db
-            .collection(coll)
-            .find({})
-            .toArray();
-        } else
-          switch (change.operationType) {
-            case "insert":
-              this.#cache[coll].push(change.fullDocument);
-              break;
+        this.#debugLog(`Collection '${colName}' changed`);
 
-            case "update":
-            case "replace":
-              this.#cache[coll] = this.#cache[coll].map((doc) =>
-                doc._id.toString() === change.documentKey._id.toString()
-                  ? change.fullDocument
-                  : doc
-              );
-              break;
+        change.col = colName;
 
+        const type = change.operationType;
+        const id = change.documentKey?._id.toString();
+
+        for (const k in this.#streams) {
+          const stream = this.#streams[k];
+          if (stream.collection != coll) continue;
+
+          Promise.resolve(stream.filter(doc)).then((ok) => {
+            if (ok) {
+              this.io.emit(`realtime:${k}`, {
+                results: [doc],
+                coll: coll,
+                count: 1,
+                total: 1,
+                remaining: 0,
+              });
+            }
+          });
+        }
+        for (let k in this.#data) {
+          if (!k.startsWith(`${coll}-`) || !this.#data[k].result[id]) continue;
+          doc._id = doc._id.toString();
+          switch (change.operationType) {
             case "delete":
-              this.#cache[coll] = this.#cache[coll].filter(
-                (doc) =>
-                  doc._id.toString() !== change.documentKey._id.toString()
-              );
+              delete this.#data[k].result[id];
               break;
+            default:
+              this.#data[k].result[id] = doc;
           }
-
-        Object.entries(this.#streams).forEach(async (e) => {
-          const key = e[0];
-          const value = e[1];
-          if (value.collection != coll) return;
-          const filterResults = await Promise.allSettled(
-            this.#cache[coll].map((doc) => value.filter(doc))
-          );
-
-          const filtered = this.#cache[coll].filter(
-            (_, i) => filterResults[i] && filterResults[i].value
-          );
-
-          this.io.emit(`db:stream:${key}`, filtered);
-          this.notifyListeners(`db:stream:${key}`, filtered);
-        });
-      });
-
-      /** Emit listen events on change */
-      changeStream.on("change", async (change) => {
-        const colName = change.ns.coll.toLowerCase();
-        change.col = colName;
-
-        const type = change.operationType;
-        const id = change.documentKey?._id;
+        }
 
         const e_change = "db:change";
         const e_change_type = `db:${type}`;
@@ -260,6 +249,163 @@ class MongoRealtime {
         }
       });
     });
+
+    try {
+      await mongoose.connect(dbUri, dbOptions);
+      this.#log(`Connected to db '${mongoose.connection.name}'`, 1);
+      onDbConnect?.call(this, mongoose.connection);
+    } catch (error) {
+      onDbError?.call(this, error);
+      this.#log("Failed to init", 4);
+      return;
+    }
+
+    this.#check(() => mongoose.connection.db, "No database found");
+
+    watch = watch.map((s) => s.toLowerCase());
+    ignore = ignore.map((s) => s.toLowerCase());
+
+    this.io.use(async (socket, next) => {
+      if (!!authentify) {
+        try {
+          const token =
+            socket.handshake.auth.token ||
+            socket.handshake.headers.authorization;
+          if (!token) return next(new Error("NO_TOKEN_PROVIDED"));
+
+          const authorized = await authentify(token, socket);
+          if (authorized === true) return next(); // exactly returns true
+
+          return next(new Error("UNAUTHORIZED"));
+        } catch (error) {
+          return next(new Error("AUTH_ERROR"));
+        }
+      } else {
+        return next();
+      }
+    });
+
+    for (let middleware of middlewares) {
+      this.io.use(middleware);
+    }
+
+    this.io.on("connection", (socket) => {
+      socket.emit("version", version);
+
+      socket.on(
+        "realtime",
+        async ({ streamId, limit, reverse, registerId }) => {
+          if (!streamId || !this.#streams[streamId]) return;
+
+          const stream = this.#streams[streamId];
+          const coll = stream.collection;
+
+          const default_limit = 50;
+          limit ??= default_limit;
+          try {
+            limit = parseInt(limit);
+          } catch (_) {
+            limit = default_limit;
+          }
+
+          reverse = reverse == true;
+          registerId ??= "";
+          this.#debugLog(
+            `Socket ${socket.id} registred for realtime '${coll}:${registerId}'. Limit ${limit}. Reversed ${reverse}`
+          );
+
+          let total;
+          const ids = [];
+
+          do {
+            total = await this.connection.db
+              .collection(coll)
+              .estimatedDocumentCount();
+
+            const length = ids.length;
+            const range = [length, Math.min(total, length + limit)];
+            const now = new Date();
+
+            let cachedKey = `${coll}-${range}`;
+
+            let cachedResult = this.#data[cachedKey];
+            if (cachedResult && cachedResult.expiration < now) {
+              delete this.#data[cachedKey];
+            }
+
+            cachedResult = this.#data[cachedKey];
+
+            const result = cachedResult
+              ? Object.values(cachedResult.result)
+              : await this.connection.db
+                  .collection(coll)
+                  .find({
+                    _id: {
+                      $nin: ids,
+                    },
+                  })
+                  .limit(limit)
+                  .sort({ _id: reverse ? -1 : 1 })
+                  .toArray();
+
+            ids.push(...result.map((d) => d._id));
+
+            const delayInMin = 1;
+            const expiration = new Date(now.getTime() + delayInMin * 60 * 1000);
+            const resultMap = result.reduce((acc, item) => {
+              item._id = item._id.toString();
+              acc[item._id] = item;
+              return acc;
+            }, {});
+
+            if (!cachedResult) {
+              this.#data[cachedKey] = {
+                expiration,
+                result: resultMap,
+              };
+            }
+
+            const filtered = (
+              await Promise.all(
+                result.map(async (doc) => {
+                  try {
+                    return {
+                      doc,
+                      ok: await stream.filter(doc),
+                    };
+                  } catch (e) {
+                    return {
+                      doc,
+                      ok: false,
+                    };
+                  }
+                })
+              )
+            )
+              .filter((item) => item.ok)
+              .map((item) => item.doc);
+
+            const data = {
+              results: filtered,
+              coll,
+              count: filtered.length,
+              total,
+              remaining: total - ids.length,
+            };
+
+            socket.emit(`realtime:${streamId}:${registerId}`, data);
+          } while (ids.length < total);
+        }
+      );
+
+      socket.on("disconnect", (r) => {
+        if (offSocket) offSocket(socket, r);
+      });
+
+      if (onSocket) onSocket(socket);
+    });
+
+    this.#log(`Initialized`, 1);
   }
 
   /**
@@ -295,16 +441,14 @@ class MongoRealtime {
    *
    * Register a new list stream to listen
    */
-  static addListStream(streamId, collection, filter) {
+  static addStream(streamId, collection, filter) {
     if (!streamId) throw new Error("Stream id is required");
     if (!collection) throw new Error("Collection is required");
+    if (this.#streams[streamId] && this.collections.includes(streamId))
+      throw new Error(`streamId '${streamId}' cannot be a collection`);
 
     filter ??= (_, __) => true;
-    if (this.#safeListStream && this.#streams[streamId]) {
-      throw new Error(
-        `Stream '${streamId}' already registered or is reserved.`
-      );
-    }
+
     this.#streams[streamId] = {
       collection,
       filter,
@@ -316,7 +460,7 @@ class MongoRealtime {
    *
    * Delete a registered stream
    */
-  static removeListStream(streamId) {
+  static removeStream(streamId) {
     delete this.#streams[streamId];
   }
 

package/package.json

@@ -1,26 +1,27 @@
-{
-  "name": "mongo-realtime",
-  "version": "1.1.3",
-  "main": "index.js",
-  "scripts": {},
-  "keywords": [
-    "mongo",
-    "mongoose",
-    "replica",
-    "stream",
-    "realtime",
-    "mongodb",
-    "socket",
-    "db"
-  ],
-  "author": "D3R50N",
-  "license": "MIT",
-  "repository": {
-    "url": "git+https://github.com/D3R50N/mongo-realtime.git"
-  },
-  "description": "A Node.js package that combines Socket.IO and MongoDB Change Streams to deliver real-time database updates to your WebSocket clients.",
-  "dependencies": {
-    "mongoose": "^8.17.0",
-    "socket.io": "^4.8.1"
-  }
-}
+{
+  "name": "mongo-realtime",
+  "version": "2.0.0",
+  "main": "index.js",
+  "scripts": {},
+  "keywords": [
+    "mongo",
+    "mongoose",
+    "replica",
+    "stream",
+    "realtime",
+    "mongodb",
+    "socket",
+    "db"
+  ],
+  "author": "D3R50N",
+  "license": "MIT",
+  "repository": {
+    "url": "git+https://github.com/D3R50N/mongo-realtime.git"
+  },
+  "description": "A Node.js package that combines Socket.IO and MongoDB Change Streams to deliver real-time database updates to your WebSocket clients.",
+  "dependencies": {
+    "chalk": "^4.1.2",
+    "mongoose": "^8.17.0",
+    "socket.io": "^4.8.1"
+  }
+}