mongo-realtime 1.0.3 → 1.1.0

package/README.md

@@ -27,16 +27,16 @@ npm install mongo-realtime
 ### Example setup
 
 ```javascript
-const express = require('express');
-const http = require('http');
-const mongoose = require('mongoose');
-const MongoRealtime = require('mongo-realtime');
+const express = require("express");
+const http = require("http");
+const mongoose = require("mongoose");
+const MongoRealtime = require("mongo-realtime");
 
 const app = express();
 const server = http.createServer(app);
 
-mongoose.connect('mongodb://localhost:27017/mydb').then((c) => {
-    console.log("Connected to db",c.connection.name);
+mongoose.connect("mongodb://localhost:27017/mydb").then((c) => {
+  console.log("Connected to db", c.connection.name);
 });
 
 MongoRealtime.init({
@@ -45,15 +45,15 @@ MongoRealtime.init({
   ignore: ["posts"], // ignore 'posts' collection
   onSocket: (socket) => {
     console.log(`Client connected: ${socket.id}`);
-    socket.emit('welcome', { message: 'Connection successful!' });
+    socket.emit("welcome", { message: "Connection successful!" });
   },
   offSocket: (socket, reason) => {
     console.log(`Client disconnected: ${socket.id}, reason: ${reason}`);
-  }
+  },
 });
 
 server.listen(3000, () => {
-  console.log('Server started on port 3000');
+  console.log("Server started on port 3000");
 });
 ```
 
@@ -67,22 +67,31 @@ Initializes the socket system and MongoDB Change Streams.
 
 \* means required
 
-| Parameter | Type | Description |
-|-----------|------|-------------|
-| `options.connection` | `mongoose.Connection`* | Active Mongoose connection |
-| `options.server` | `http.Server`* | HTTP server to attach Socket.IO |
-| `options.authentify` | `Function` | Function to authenticate socket connections. Should return true if authenticated |
-| `options.middlewares` | `Array[Function]` | Array of Socket.IO middlewares |
-| `options.onSocket` | `Function` | Callback on socket connection |
-| `options.offSocket` | `Function` | Callback on socket disconnection |
-| `options.watch` | `Array[String]` | Collections to only watch. Listen to all when is empty |
-| `options.ignore` | `Array[String]` | Collections to only ignore. Overrides watch array |
-
-#### Static Properties
-
-- `MongoRealtime.io`: Socket.IO server instance
+| Parameter                | Type                    | Description                                                                      |
+| ------------------------ | ----------------------- | -------------------------------------------------------------------------------- |
+| `options.connection`     | `mongoose.Connection`\* | Active Mongoose connection                                                       |
+| `options.server`         | `http.Server`\*         | HTTP server to attach Socket.IO                                                  |
+| `options.authentify`     | `Function`              | Function to authenticate socket connections. Should return true if authenticated |
+| `options.middlewares`    | `Array[Function]`       | Array of Socket.IO middlewares                                                   |
+| `options.onSocket`       | `Function`              | Callback on socket connection                                                    |
+| `options.offSocket`      | `Function`              | Callback on socket disconnection                                                 |
+| `options.watch`          | `Array[String]`         | Collections to only watch. Listen to all when is empty                           |
+| `options.ignore`         | `Array[String]`         | Collections to only ignore. Overrides watch array                                |
+| `options.autoListStream` | `Array[String]`         | Collections to automatically stream to clients. Default is all                   |
+
+#### Static Properties and Methods
+
+- `MongoRealtime.addListStream(streamId, collection, filter)`: Manually add a list stream for a specific collection and filter
 - `MongoRealtime.connection`: MongoDB connection
-- `MongoRealtime.sockets`: Array of connected sockets
+- `MongoRealtime.collections`: Array of database collections
+- `MongoRealtime.io`: Socket.IO server instance
+- `MongoRealtime.init(options)`: Initialize the package with options
+- `MongoRealtime.listen(event, callback)`: Add an event listener for database changes
+- `MongoRealtime.notifyListeners(event, data)`: Manually emit an event to all listeners
+- `MongoRealtime.removeStream(streamId)`: Remove a previously added stream by id
+- `MongoRealtime.removeListener(event, callback)`: Remove a specific event listener or all listeners for an event
+- `MongoRealtime.removeAllListeners()`: Remove all event listeners
+- `MongoRealtime.sockets()`: Returns an array of connected sockets
 
 ## 🎯 Emitted Events
 
@@ -90,42 +99,42 @@ The package automatically emits six types of events for each database change:
 
 ### Event Types
 
-| Event | Description | Example |
-|-------|-------------|---------|
-| `db:change` | All changes | Any collection change |
-| `db:{type}` | By operation type | `db:insert`, `db:update`, `db:delete` |
-| `db:change:{collection}` | By collection | `db:change:users`, `db:change:posts` |
-| `db:{type}:{collection}` | Type + collection | `db:insert:users`, `db:update:posts` |
+| Event                         | Description       | Example                                    |
+| ----------------------------- | ----------------- | ------------------------------------------ |
+| `db:change`                   | All changes       | Any collection change                      |
+| `db:{type}`                   | By operation type | `db:insert`, `db:update`, `db:delete`      |
+| `db:change:{collection}`      | By collection     | `db:change:users`, `db:change:posts`       |
+| `db:{type}:{collection}`      | Type + collection | `db:insert:users`, `db:update:posts`       |
 | `db:change:{collection}:{id}` | Specific document | `db:change:users:507f1f77bcf86cd799439011` |
-| `db:{type}:{collection}:{id}` | Type + document | `db:insert:users:507f1f77bcf86cd799439011` |
+| `db:{type}:{collection}:{id}` | Type + document   | `db:insert:users:507f1f77bcf86cd799439011` |
+| `db:stream:{streamId}`        | By stream         | `db:stream:myStreamId`                     |
 
 ### Event listeners
 
 You can add serverside listeners to those db events to trigger specific actions on the server:
 
 ```js
-function sendNotification(change){
+function sendNotification(change) {
   const userId = change.docId; // or change.documentKey._id
-  NotificationService.send(userId,"Welcome to DB"); 
+  NotificationService.send(userId, "Welcome to DB");
 }
 
-MongoRealtime.listen("db:insert:users",sendNotification);
+MongoRealtime.listen("db:insert:users", sendNotification);
 ```
 
 #### Adding many callback to one event
 
 ```js
-MongoRealtime.listen("db:insert:users",anotherAction);
-MongoRealtime.listen("db:insert:users",anotherAction2);
+MongoRealtime.listen("db:insert:users", anotherAction);
+MongoRealtime.listen("db:insert:users", anotherAction2);
 ```
 
 #### Removing event listeners
 
 ```js
-MongoRealtime.removeListener("db:insert:users",sendNotification); // remove this specific action from this event
+MongoRealtime.removeListener("db:insert:users", sendNotification); // remove this specific action from this event
 MongoRealtime.removeListener("db:insert:users"); // remove all actions from this event
 MongoRealtime.removeAllListeners(); // remove all listeners
-
 ```
 
 ### Event Payload Structure
@@ -136,7 +145,7 @@ Each event contains the full MongoDB change object:
 {
   "_id": {...},
   "col":"users", // same as ns.coll
-  "docId":"...", // same as documentKey._id 
+  "docId":"...", // same as documentKey._id
   "operationType": "insert|update|delete|replace",
   "documentKey": { "_id": "..." },
   "ns": { "db": "mydb", "coll": "users" },
@@ -154,14 +163,13 @@ MongoRealtime.init({
   connection: connection,
   server: server,
   onSocket: (socket) => {
-     socket.on('subscribe:users', () => {
-        socket.join('users-room');
+    socket.on("subscribe:users", () => {
+      socket.join("users-room");
     });
   },
-
 });
 
-MongoRealtime.io.to('users-room').emit('custom-event', data);
+MongoRealtime.io.to("users-room").emit("custom-event", data);
 ```
 
 ### Client-side - Receiving updates
@@ -169,31 +177,31 @@ MongoRealtime.io.to('users-room').emit('custom-event', data);
 ```html
 <!DOCTYPE html>
 <html>
-<head>
-  <script src="/socket.io/socket.io.js"></script>
-</head>
-<body>
-  <script>
-    const socket = io();
-
-    socket.on('db:change', (change) => {
-      console.log('Detected change:', change);
-    });
-
-    socket.on('db:insert:users', (change) => {
-      console.log('New user:', change.fullDocument);
-    });
-
-    const userId = '507f1f77bcf86cd799439011';
-    socket.on(`db:update:users:${userId}`, (change) => {
-      console.log('Updated user:', change.fullDocument);
-    });
-
-    socket.on('db:delete', (change) => {
-      console.log('Deleted document:', change.documentKey);
-    });
-  </script>
-</body>
+  <head>
+    <script src="/socket.io/socket.io.js"></script>
+  </head>
+  <body>
+    <script>
+      const socket = io();
+
+      socket.on("db:change", (change) => {
+        console.log("Detected change:", change);
+      });
+
+      socket.on("db:insert:users", (change) => {
+        console.log("New user:", change.fullDocument);
+      });
+
+      const userId = "507f1f77bcf86cd799439011";
+      socket.on(`db:update:users:${userId}`, (change) => {
+        console.log("Updated user:", change.fullDocument);
+      });
+
+      socket.on("db:delete", (change) => {
+        console.log("Deleted document:", change.documentKey);
+      });
+    </script>
+  </body>
 </html>
 ```
 
@@ -204,15 +212,15 @@ MongoRealtime.init({
   connection: mongoose.connection,
   server: server,
   onSocket: (socket) => {
-    socket.on('error', (error) => {
-      console.error('Socket error:', error);
+    socket.on("error", (error) => {
+      console.error("Socket error:", error);
     });
   },
   offSocket: (socket, reason) => {
-    if (reason === 'transport error') {
-      console.log('Transport error detected');
+    if (reason === "transport error") {
+      console.log("Transport error detected");
     }
-  }
+  },
 });
 ```
 
@@ -221,13 +229,13 @@ MongoRealtime.init({
 ### Socket Authentication
 
 ```javascript
-function authenticateSocket(token, socket){
-   const verify = AuthService.verifyToken(token);
-   if(verify){
-     socket.user = verify.user; // attach user info to socket
-     return true; // should return true to accept the connection
-   }
-   return false;
+function authenticateSocket(token, socket) {
+  const verify = AuthService.verifyToken(token);
+  if (verify) {
+    socket.user = verify.user; // attach user info to socket
+    return true; // should return true to accept the connection
+  }
+  return false;
 }
 
 MongoRealtime.init({
@@ -236,14 +244,116 @@ MongoRealtime.init({
   authentify: authenticateSocket,
   middlewares: [
     (socket, next) => {
-     console.log(`User is authenticated: ${socket.user.email}`);
-     next();
-    }
+      console.log(`User is authenticated: ${socket.user.email}`);
+      next();
+    },
   ],
   offSocket: (socket, reason) => {
     console.log(`Socket ${socket.id} disconnected: ${reason}`);
+  },
+});
+```
+
+### Setup list streams
+
+The server will automatically emit a list of filtered documents from the specified collections after each change.\
+Each list stream requires an unique `streamId`, the `collection` name, and an optional `filter` function that returns a boolean or a promise resolving to a boolean.
+Clients receive the list on the event `db:stream:{streamId}`.\
+For best practice, two list streams can't have the same `streamId` or else an error will be thrown. This will prevent accidental overrides. You can still remove a stream with `MongoRealtime.removeListStream(streamId)` and add it again or use inside a `try-catch` scope.
+
+```javascript
+MongoRealtime.init({
+  connection: mongoose.connection,
+  server: server,
+  autoListStream: ["users"], // automatically stream users collection only
+});
+
+MongoRealtime.addListStream("users", "users", (doc) => !!doc.email); // will throw an error as streamId 'users' already exists
+
+MongoRealtime.removeListStream("users"); // remove the previous stream
+MongoRealtime.addListStream("users", "users", (doc) => !!doc.email); // client can listen to db:stream:users
+
+MongoRealtime.addListStream("usersWithEmail", "users", (doc) => !!doc.email); // client can listen to db:stream:usersWithEmail
+```
+
+#### ⚠️ NOTICE
+
+When `autoListStream` is not set, all collections are automatically streamed and WITHOUT any filter.\
+That means that if you have a `posts` collection, all documents from this collection will be sent to the clients on each change.\
+And as two list streams can't have the same `streamId`, if you want to add a filtered list stream with id `posts`, you must set `autoListStream` to an array NOT containing `"posts"`or `MongoRealtime.removeListStream("posts")` after initialization.\
+Therefore, if you want to add a filtered list stream for all collections, you must set `autoListStream` to an empty array.
+
+```javascript
+MongoRealtime.init({
+  connection: mongoose.connection,
+  server: server,
+  autoListStream: [], // stream no collection automatically (you can add your own filtered streams later)
+});
+MongoRealtime.addListStream("posts", "posts", (doc) => !!doc.title); // client can listen to db:stream:posts
+MongoRealtime.addListStream("users", "users", (doc) => !!doc.email); // will not throw an error
+```
+
+#### Usecase for id based streams
+
+```javascript
+MongoRealtime.init({
+  connection: mongoose.connection,
+  server: server,
+  authentify: (token, socket) => {
+    try {
+      socket.uid = decodeToken(token).uid; // setup user id from token
+      return true;
+    } catch (error) {
+      return false;
+    }
+  },
+  onSocket: (socket) => {
+    // setup a personal stream for each connected user
+    MongoRealtime.addListStream(
+      `userPost:${socket.uid}`,
+      "posts",
+      (doc) => doc._id == socket.uid
+    );
+  },
+  offSocket: (socket) => {
+    // clean up when user disconnects
+    MongoRealtime.removeListStream(`userPost:${socket.uid}`);
+  },
+});
+
+// ...
+// or activate stream from a controller or middleware
+app.get("/my-posts", (req, res) => {
+  const { user } = req;
+  try {
+    MongoRealtime.addListStream(
+      `userPosts:${user._id}`,
+      "posts",
+      (doc) => doc.authorId === user._id
+    );
+  } catch (e) {
+    // stream already exists
   }
+
+  res.send("Stream activated");
+});
+```
+
+#### Usecase with async filter
+
+```javascript
+// MongoRealtime.init({...});
+
+MongoRealtime.addListStream("authorizedUsers", "users", async (doc) => {
+  const isAdmin = await UserService.isAdmin(doc._id);
+  return isAdmin && doc.email.endsWith("@mydomain.com");
 });
+
+MongoRealtime.addListStream(
+  "bestPosts",
+  "posts",
+  async (doc) => doc.likes > (await PostService.getLikesThreshold())
+);
 ```
 
 ## 📚 Dependencies
@@ -267,4 +377,4 @@ MIT
 
 ## 🤝 Contributing
 
-Contributions are welcome! Feel free to open an issue or submit a pull request.****
+Contributions are welcome! Feel free to open an issue or submit a pull request.

package/index.js

@@ -1,11 +1,55 @@
 const { Server } = require("socket.io");
 
+function sortObj(obj = {}) {
+  const out = {};
+  for (let k of Object.keys(obj).sort()) {
+    const v = obj[k];
+    out[k] = typeof v == "object" && !Array.isArray(v) ? sortObj(v) : v;
+  }
+
+  return out;
+}
+
+/**
+ * @typedef {Object} ChangeStreamDocument
+ * @property {"insert"|"update"|"replace"|"delete"|"invalidate"|"drop"|"dropDatabase"|"rename"} operationType
+ *   The type of operation that triggered the event.
+ *
+ * @property {Object} ns
+ * @property {string} ns.db - Database name
+ * @property {string} ns.coll - Collection name
+ *
+ * @property {Object} documentKey
+ * @property {import("bson").ObjectId|string} documentKey._id - The document’s identifier
+ *
+ * @property {Object} [fullDocument]
+ *   The full document after the change (only present if `fullDocument: "updateLookup"` is enabled).
+ *
+ * @property {Object} [updateDescription]
+ * @property {Object.<string, any>} [updateDescription.updatedFields]
+ *   Fields that were updated during an update operation.
+ * @property {string[]} [updateDescription.removedFields]
+ *   Fields that were removed during an update operation.
+ *
+ * @property {Object} [rename] - Info about the collection rename (if operationType is "rename").
+ *
+ * @property {Date} [clusterTime] - Logical timestamp of the event.
+ */
+
 class MongoRealtime {
   /** @type {import("socket.io").Server} */ static io;
   /** @type {import("mongoose").Connection} */ static connection;
-  /** @type {[import("socket.io").Socket]} */ static sockets = [];
   /** @type {Record<String, [(change:ChangeStreamDocument)=>void]>} */ static #listeners =
     {};
+  /** @type {Record<String, [Object]>} */
+  static #cache = {};
+  static sockets = () => [...this.io.sockets.sockets.values()];
+
+  /**@type {Record<String, {collection:String,filter: (doc:Object)=>Promise<boolean>}>} */
+  static #streams = {};
+
+  /** @type {[String]} - All DB collections */
+  static collections = [];
 
   /**
    * Initializes the socket system.
@@ -17,6 +61,7 @@ class MongoRealtime {
    * @param {(socket: import("socket.io").Socket) => void} options.onSocket - Callback triggered when a socket connects
    * @param {(socket: import("socket.io").Socket, reason: import("socket.io").DisconnectReason) => void} options.offSocket - Callback triggered when a socket disconnects
    * @param {import("http").Server} options.server - HTTP server to attach Socket.IO to
+   * @param {[String]} options.autoListStream - Collections to stream automatically. If empty, will stream no collection. If null, will stream all collections.
    * @param {[String]} options.watch - Collections to watch. If empty, will watch all collections
    * @param {[String]} options.ignore - Collections to ignore. Can override `watch`
    *
@@ -25,16 +70,14 @@ class MongoRealtime {
     connection,
     server,
     authentify,
-    middlewares=[],
+    middlewares = [],
+    autoListStream,
     onSocket,
     offSocket,
     watch = [],
     ignore = [],
   }) {
-    if (this.io)
-      this.io.close(() => {
-        this.sockets = [];
-      });
+    if (this.io) this.io.close();
     this.io = new Server(server);
     this.connection = connection;
 
@@ -44,20 +87,21 @@ class MongoRealtime {
     this.io.use(async (socket, next) => {
       if (!!authentify) {
         try {
-          const token = socket.handshake.auth.token;
-          if (!token) return next(new Error("No token provided"));
+          const token =
+            socket.handshake.auth.token ||
+            socket.handshake.headers.authorization;
+          if (!token) return next(new Error("NO_TOKEN_PROVIDED"));
 
-          const authorized =await authentify(token, socket);
-          if (authorized===true) return next(); // exactly returns true
+          const authorized = await authentify(token, socket);
+          if (authorized === true) return next(); // exactly returns true
 
-          return next(new Error("Unauthorized"));
+          return next(new Error("UNAUTHORIZED"));
         } catch (error) {
-          return next(new Error("Authentication error"));
+          return next(new Error("AUTH_ERROR"));
         }
       } else {
         return next();
       }
-      
     });
 
     for (let middleware of middlewares) {
@@ -65,16 +109,18 @@ class MongoRealtime {
     }
 
     this.io.on("connection", (socket) => {
-      this.sockets = [...this.io.sockets.sockets.values()];
       if (onSocket) onSocket(socket);
 
       socket.on("disconnect", (r) => {
-        this.sockets = [...this.io.sockets.sockets.values()];
         if (offSocket) offSocket(socket, r);
       });
     });
 
-    connection.once("open", () => {
+    connection.once("open", async () => {
+      this.collections = (await connection.listCollections()).map(
+        (c) => c.name
+      );
+
       let pipeline = [];
       if (watch.length !== 0 && ignore.length === 0) {
         pipeline = [{ $match: { "ns.coll": { $in: watch } } }];
@@ -98,7 +144,65 @@ class MongoRealtime {
         fullDocumentBeforeChange: "whenAvailable",
       });
 
-      changeStream.on("change", (change) => {
+      /** Setup main streams */
+      let collectionsToStream = [];
+      if (autoListStream == null) collectionsToStream = this.collections;
+      else
+        collectionsToStream = this.collections.filter((c) =>
+          autoListStream.includes(c)
+        );
+      for (let col of collectionsToStream) this.addListStream(col, col);
+
+      /** Emit streams on change */
+      changeStream.on("change", async (change) => {
+        const coll = change.ns.coll;
+
+        if (!this.#cache[coll]) {
+          this.#cache[coll] = await connection.db
+            .collection(coll)
+            .find({})
+            .toArray();
+        } else
+          switch (change.operationType) {
+            case "insert":
+              this.#cache[coll].push(change.fullDocument);
+              break;
+
+            case "update":
+            case "replace":
+              this.#cache[coll] = this.#cache[coll].map((doc) =>
+                doc._id.toString() === change.documentKey._id.toString()
+                  ? change.fullDocument
+                  : doc
+              );
+              break;
+
+            case "delete":
+              this.#cache[coll] = this.#cache[coll].filter(
+                (doc) =>
+                  doc._id.toString() !== change.documentKey._id.toString()
+              );
+              break;
+          }
+
+        Object.entries(this.#streams).forEach(async (e) => {
+          const key = e[0];
+          const value = e[1];
+          if (value.collection != coll) return;
+          const filterResults = await Promise.allSettled(
+            this.#cache[coll].map((doc) => value.filter(doc))
+          );
+
+          const filtered = this.#cache[coll].filter(
+            (_, i) => filterResults[i] && filterResults[i].value
+          );
+
+          this.io.emit(`db:stream:${key}`, filtered);
+        });
+      });
+
+      /** Emit listen events on change */
+      changeStream.on("change", async (change) => {
         const colName = change.ns.coll.toLowerCase();
         change.col = colName;
 
@@ -156,6 +260,39 @@ class MongoRealtime {
     this.#listeners[key].push(cb);
   }
 
+  /**
+   *
+   * @param {String} streamId - StreamId of the list stream
+   * @param {String} collection - Name of the collection to stream
+   * @param { (doc:Object )=>Promise<boolean>} filter - Collection filter
+   *
+   * Register a new list stream to listen
+   */
+  static addListStream(streamId, collection, filter) {
+    if (!streamId) throw new Error("Stream id is required");
+    if (!collection) throw new Error("Collection is required");
+
+    filter ??= (_, __) => true;
+    if (this.#streams[streamId]) {
+      throw new Error(
+        `Stream '${streamId}' already registered or is reserved.`
+      );
+    }
+    this.#streams[streamId] = {
+      collection,
+      filter,
+    };
+  }
+
+  /**
+   * @param {String} streamId - StreamId of the stream
+   *
+   * Delete a registered stream
+   */
+  static removeListStream(streamId) {
+    delete this.#streams[streamId];
+  }
+
   /**
    * Remove one or all listeners of an event
    *

package/package.json

@@ -1,26 +1,26 @@
-{
-  "name": "mongo-realtime",
-  "version": "1.0.3",
-  "main": "index.js",
-  "scripts": {},
-  "keywords": [
-    "mongo",
-    "mongoose",
-    "replica",
-    "stream",
-    "realtime",
-    "mongodb",
-    "socket",
-    "db"
-  ],
-  "author": "D3R50N",
-  "license": "MIT",
-  "repository": {
-    "url": "https://github.com/D3R50N/mongo-realtime.git"
-  },
-  "description": "A Node.js package that combines Socket.IO and MongoDB Change Streams to deliver real-time database updates to your WebSocket clients.",
-  "dependencies": {
-    "mongoose": "^8.17.0",
-    "socket.io": "^4.8.1"
-  }
-}
+{
+  "name": "mongo-realtime",
+  "version": "1.1.0",
+  "main": "index.js",
+  "scripts": {},
+  "keywords": [
+    "mongo",
+    "mongoose",
+    "replica",
+    "stream",
+    "realtime",
+    "mongodb",
+    "socket",
+    "db"
+  ],
+  "author": "D3R50N",
+  "license": "MIT",
+  "repository": {
+    "url": "git+https://github.com/D3R50N/mongo-realtime.git"
+  },
+  "description": "A Node.js package that combines Socket.IO and MongoDB Change Streams to deliver real-time database updates to your WebSocket clients.",
+  "dependencies": {
+    "mongoose": "^8.17.0",
+    "socket.io": "^4.8.1"
+  }
+}