@rpcbase/server 0.204.0 → 0.206.0

package/mongoose.js

@@ -0,0 +1,8 @@
+/* @flow */
+const mongoose = require("mongoose")
+
+// WARNING: this shouldn't be necessary as strictQuery is back to false by default
+// https://mongoosejs.com/docs/migrating_to_7.html#strictquery
+mongoose.set("strictQuery", false)
+
+module.exports = mongoose

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rpcbase/server",
-  "version": "0.204.0",
+  "version": "0.206.0",
   "license": "SSPL-1.0",
   "main": "./index.js",
   "bin": {
@@ -12,6 +12,7 @@
   "dependencies": {
     "@rpcbase/agent": "0.30.0",
     "@rpcbase/std": "0.6.0",
+    "bluebird": "3.7.2",
     "body-parser": "1.20.2",
     "bull": "4.10.4",
     "connect-redis": "6.1.3",
@@ -28,6 +29,8 @@
     "postmark": "3.0.18",
     "redis": "4.6.6",
     "request-ip": "3.3.0",
+    "sift": "17.0.1",
+    "socket.io": "4.7.1",
     "validator": "13.9.0",
     "yargs": "17.7.2"
   }

package/redis.js

@@ -0,0 +1,2 @@
+/* @flow */
+module.exports = require("redis")

package/rts/index.js

@@ -0,0 +1,425 @@
+/* @flow */
+const Promise = require("bluebird")
+const _remove = require("lodash/remove")
+const _get = require("lodash/get")
+const _set = require("lodash/set")
+const {Server} = require("socket.io")
+const sift = require("sift")
+const debug = require("debug")
+const colors = require("picocolors")
+
+const log = debug("rb:rts:server")
+
+const is_production = process.env.IS_PRODUCTION === "yes"
+
+
+const QUERY_MAX_LIMIT = 4096
+
+// TODO: there are edge cases in dev where the client reconnects to the server
+// but the server already has the query results and sends it multiple times
+// the server should ensure it is only registering queries once
+
+// TODO: does this even support multiple clients ?
+// TODO: investigate multiple change streams event triggers per client
+// TODO: add tracing
+
+const _sockets = {}
+const _change_streams = {}
+// we index both queries and socket queries
+// to avoid going through all queries when a socket disconnects
+const _queries = {}
+const _socket_queries = {}
+
+
+const get_query_options = (ctx, options) => {
+  // https://mongoosejs.com/docs/api/query.html#Query.prototype.setOptions()
+  const query_options = {
+    ctx,
+    is_client: true
+    //
+  }
+
+  if (options.sort) {
+    query_options.sort = options.sort
+  }
+
+  const limit = typeof options.limit === "number" ? Math.min(QUERY_MAX_LIMIT, Math.abs(options.limit)) : QUERY_MAX_LIMIT
+  // TODO: warn if limit greater than default ?
+  query_options.limit = options.limit
+
+  return query_options
+}
+
+
+// helper to send and update / refresh event to all subscribed queries
+const dispatch_doc_change = (model, doc) => {
+  const model_name = model.modelName
+  const queries = _queries[model_name] || {}
+
+  // WARNING: INVESTIGATE: there are edge cases where you might want to delete an object
+  // that was matched, but now it's not anymore because it's been udpated or removed
+  // rts should still update the query and send a payload to the client
+  // we could use the new mongodb fullDocumentBeforeChange param in watch,
+  // but it doesn't seem to be guaranteed to be there
+
+  // filter queries that match the document
+  const target_queries = Object.keys(queries).filter((query_key) => {
+    const {query} = queries[query_key]
+    // console.log("query", query)
+    const test_fn = sift(query)
+    return test_fn(doc)
+  })
+
+  if (!doc.__txn_id) {
+    console.log(colors.yellow("Warning!"), "change on model:", model.modelName, "matching queries", target_queries, "did not include a valid", colors.yellow("__txn"), "field")
+  }
+
+  // TODO: we should not re-run each query here!!!!
+  // dispatch change to target query clients
+  target_queries.forEach(async(query_key) => {
+    const {query, options, sockets} = queries[query_key]
+
+    // TODO: remove this
+    await Promise.map(sockets, async(socket_id) => {
+      const s = _sockets[socket_id]
+      // socket was destroyed
+      // TODO: remove socket id from _queries when destroyed
+      if (!s) {
+        log("NO SOCKET RETURNING")
+        return
+      }
+      // console.log("dipatch change", query_key)
+      const ctx = {req: s.request}
+
+      if (!ctx.req.session.user_id) {
+        console.error("dispatch_doc_change::error", "no user id for client query")
+        return
+      }
+
+
+      const query_options = get_query_options(ctx, options)
+      const query_promise = model.find(query, options.projection, query_options)
+      if (query_options.sort) {
+        query_promise.sort(query_options.sort)
+      }
+      if (typeof query_options.limit === "number" && query_options.limit >= 0) {
+        query_promise.limit(query_options.limit)
+      }
+      // WARNING: DANGER
+      // TODO: we should not be doing a read here, use doc from event
+      const data = await query_promise
+
+      const data_buf = JSON.stringify(data)
+
+      log("emit payload")
+
+      s.emit("query_payload", {
+        model_name,
+        query_key,
+        data_buf,
+        txn_id: doc.__txn_id,
+      })
+    }, {concurrency: 1024}) // TODO: tweak concurrency it should be infinite
+
+  })
+}
+
+
+// mongoose middleware used to broadcast delete events to appropriate sockets
+const mongoose_delete_plugin = (schema) => {
+  schema.pre("deleteOne", function(next) {
+    throw new Error("rts::deleteOne is not supported, use findOneAndDelete")
+  })
+
+  // TODO: add other delete operations
+  // https://mongoosejs.com/docs/queries.html
+  schema.post("findOneAndDelete", function(doc) {
+    dispatch_doc_change(this.model, doc)
+  })
+}
+
+
+// Mongoose transaction plugin to save txn on each doc
+// WARNING: query middlewares are not supported yet
+// https://mongoosejs.com/docs/middleware.html#types-of-middleware
+const mongoose_txn_plugin = (schema, options) => {
+  // console.log("register TXN plugin on schema", schema)
+  // WARNING: possible middlewares here:
+  // https://mongoosejs.com/docs/middleware.html
+  // this function isn't async but could be?
+  const apply_txn = /* async */function(next, save_options) {
+    // TODO: keep an eye on this, this seems to happen for nested schemas that aren't models
+    if (!save_options) {
+      next()
+      return
+    }
+
+    const {ctx} = save_options
+    if (!ctx) {
+      console.log(colors.yellow("Warning!"), "{ctx} not attached to save")
+    }
+
+    const txn_id = _get(ctx, ["req", "headers", "rts-txn-id"])
+
+    if (!txn_id) {
+      console.log(colors.yellow("Warning!"), "mongoose save request is missing txn_id this will cause issues with rts-client")
+    } else {
+      this.set("__txn_id", txn_id)
+    }
+
+    next()
+  }
+
+
+  schema.pre("save", apply_txn)
+}
+
+
+// responds to a change stream "change" event
+const get_dispatch_change_handler = (mongoose, model_name) => async(change) => {
+  const queries = _queries[model_name]
+  // console.log("onchange::queries", queries)
+
+  const model = mongoose.model(model_name)
+
+  let doc
+  if (["insert", "update"].includes(change.operationType)) {
+    // console.log("rts onChange", change)
+    doc = change.fullDocument
+  } else if (change.operationType === "delete") {
+    console.log("op is delete")
+  } else if (!["delete", "drop", "invalidate"].includes(change.operationType)) {
+    console.log("SKIPPING: unknown operation type", change.operationType, change)
+    return
+  }
+
+  if (!doc) {
+    return
+  }
+
+  dispatch_doc_change(model, doc)
+}
+
+
+const add_change_stream = (mongoose, socket_id, {model_name, query, query_key, options}) => {
+  if (!model_name) throw new Error("empty model name")
+
+  if (!_change_streams[model_name]) {
+    log("rts: initializing change stream for model:", model_name)
+    let model
+    try {
+      model = mongoose.model(model_name)
+    } catch (err) {
+      console.error("ERROR registering model:", model_name)
+      console.error(err)
+      return
+    }
+
+    // TODO: fix this, this has been released
+    // TODO: implement delete operation fullDocument retrieve,
+    // when this is released https://jira.mongodb.org/browse/SERVER-36941
+    const emitter = model.watch({
+      fullDocument: "updateLookup",
+      fullDocumentBeforeChange: "whenAvailable",
+    })
+
+    emitter.on("change", get_dispatch_change_handler(mongoose, model_name))
+
+    emitter.on("error", (err) => {
+      console.log("change listener emitter got error", err)
+    })
+
+    emitter.on("close", () => {
+      delete _change_streams[model_name]
+    })
+
+    _change_streams[model_name] = emitter
+  }
+
+  // save query
+  if (!_queries[model_name]) {
+    _queries[model_name] = {}
+  }
+
+  if (!_queries[model_name][query_key]) {
+    _queries[model_name][query_key] = {query, options, sockets: []}
+  }
+  if (!_queries[model_name][query_key].sockets.includes(socket_id)) {
+    _queries[model_name][query_key].sockets.push(socket_id)
+  }
+
+  // save socket_query
+  if (!_socket_queries[socket_id]) {
+    _socket_queries[socket_id] = {}
+  }
+  if (!_socket_queries[socket_id][model_name]) {
+    _socket_queries[socket_id][model_name] = {}
+  }
+  _socket_queries[socket_id][model_name][query_key] = true
+}
+
+
+const run_query = async(mongoose, socket_id, {model_name, query, query_key, options}) => {
+  const model = mongoose.model(model_name)
+
+  if (options) {
+    log("options", options)
+  }
+
+  if (!query) throw new Error("run_query empty query")
+
+  const s = _sockets[socket_id]
+  const ctx = {req: s.request}
+
+  log("initial run_query AUTH", ctx.req.session.user_id)
+  if (!ctx.req.session.user_id) {
+    // TODO: retry in node16! why is the throw not caught in try catch block
+    // throw new Error("no user ID for client query")
+    console.log("no user ID for client query")
+    // TODO: this happens right after sign in, investigate
+    s.emit("query_payload", {model_name, query_key, error: "error no user id"})
+    return
+  }
+
+  const query_options = get_query_options(ctx, options)
+
+  const query_promise = model.find(query, options.projection, query_options)
+
+  if (query_options.sort) {
+    query_promise.sort(query_options.sort)
+  }
+
+  if (typeof query_options.limit === "number" && query_options.limit >= 0) {
+    query_promise.limit(query_options.limit)
+  }
+
+  const data = await query_promise
+
+  // console.log("TMP Artificial delay")
+  // await Promise.delay(500)
+
+  const data_buf = JSON.stringify(data)
+
+  s.emit("query_payload", {model_name, query_key, data_buf})
+}
+
+
+const remove_query = (socket_id, {model_name, query, query_key}) => {
+  // const query_key = JSON.stringify(query)
+  const sockets = _queries[model_name]?.[query_key]?.sockets || []
+
+  let new_sockets
+  if (sockets.includes(socket_id)) {
+    new_sockets = _remove(sockets, (s) => s === socket_id)
+  } else new_sockets = sockets
+
+  _set(_queries, `${model_name}.${query_key}.sockets`, new_sockets)
+}
+
+const disconnect_handler = (socket_id, reason) => {
+  if (reason !== "transport close") {
+    console.log("client disconnected, reason:", reason, "socket:", socket_id)
+  }
+
+
+  const local_queries = _socket_queries[socket_id] || {}
+
+  // remove all queries matching that socket id
+  Object.keys(local_queries)
+    .forEach((model_name) => {
+
+      Object.keys(local_queries[model_name]).forEach((query_key) => {
+        const active_sockets = _queries[model_name]?.[query_key]?.sockets || []
+
+        const socket_index = active_sockets.indexOf(socket_id)
+
+        // remove the socket id from the array
+        if (socket_index > -1) {
+          // _queries[model_name][query_key].sockets.splice(socket_index, 1)
+          active_sockets.splice(socket_index, 1)
+        }
+
+        // remove query if there are no more listeners
+        if (active_sockets.length === 0) {
+          delete _queries[model_name]?.[query_key]
+        }
+
+        // TODO: why the || {} here ?
+        // remove change stream if there are no more queries on this collection
+        if (Object.keys(_queries[model_name] || {}).length === 0) {
+          delete _queries[model_name]
+          _change_streams[model_name]?.close()
+        }
+
+      })
+    })
+
+  // remove socket
+  delete _sockets[socket_id]
+  delete _socket_queries[socket_id]
+}
+
+
+const rts_server = (mongoose, session_middleware) => {
+
+  // register the delete plugin
+  mongoose.plugin(mongoose_delete_plugin)
+  // txn plugin
+  mongoose.plugin(mongoose_txn_plugin)
+
+
+  return async(server) => {
+    //
+    const io = new Server(server, {
+      transports: is_production ? ["websocket", "polling"] : ["websocket"],
+      allowUpgrades: true,
+      cors: {
+        // TODO: dynamic client ports
+        // TODO: check if is production
+        origin: `http://localhost:${process.env.CLIENT_PORT}`,
+        methods: ["GET", "POST"],
+        credentials: true,
+      },
+    })
+
+    // socket io session middleware
+    io.use((socket, next) => {
+      session_middleware(socket.request, {}, next)
+    })
+
+    // WARNING: TODO: socket session tmp disabled
+    // io.on('connection', (socket) => {
+    //   const session = socket.request.session;
+    //   session.connections++;
+    //   session.save();
+    // });
+
+    io.on("connection", (socket) => {
+      _sockets[socket.id] = socket
+      console.log("socket connected", socket.id)
+      //
+      socket.on("register_query", (payload) => {
+        add_change_stream(mongoose, socket.id, payload)
+      })
+
+      socket.on("remove_query", (payload) => {
+        remove_query(socket.id, payload)
+      })
+
+      socket.on("run_query", (payload) => {
+        try {
+          run_query(mongoose, socket.id, payload)
+        } catch (err) {
+          console.log("run_query caught error", err)
+        }
+      })
+
+      socket.on("disconnect", (reason) => {
+        disconnect_handler(socket.id, reason)
+      })
+    })
+  }
+}
+
+
+module.exports = rts_server

package/mongoose/index.js

@@ -1,6 +0,0 @@
-/* @flow */
-const mongoose = require("mongoose")
-
-mongoose.set("strictQuery", false)
-
-module.exports = mongoose