@rpcbase/server 0.328.0 → 0.329.0

package/boot/worker.js

@@ -3,6 +3,10 @@ require("./shared")
 
 require("./register_worker_tasks")
 
+const register_queue_listener = require("../queue/register_queue_listener")
+
 const queue = require("../queue")
 
 queue.start()
+
+register_queue_listener()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rpcbase/server",
-  "version": "0.328.0",
+  "version": "0.329.0",
   "license": "SSPL-1.0",
   "main": "./index.js",
   "bin": {

package/queue/register_queue_listener.js

@@ -1,4 +1,5 @@
 /* @flow */
+const assert = require("assert")
 const debug = require("debug")
 const _get = require("lodash/get")
 const _set = require("lodash/set")
@@ -11,10 +12,14 @@ const dispatch_queue = require("./dispatch_queue")
 
 const log = debug("rb:queue:listener")
 
+const {RB_APP_NAME} = process.env
+
+const RETRY_MAXIMUM_DELAY = 2000
 const RETRY_MINIMUM_DELAY = 10
-const MAX_RETRIES = 16
+const MAX_RETRIES = 64
+
 // TODO: reset retry counters when connection becomes healthy again?
-const retry_counters = {}
+let retry_counter = 0
 
 
 // Listens for mongodb change events,
@@ -31,58 +36,75 @@ const mongoose_delete_plugin = (schema) => {
   schema.post("findOneAndDelete", function(doc) {
     log("queue:findOneAndDelete", "dispatch_doc_change NYI")
     log("DEL PLUGIN", doc)
-    const model_name = this.model.modelName
+    const coll_name = change.ns.coll
+
+    const model_name = Object.keys(mongoose.models).find((k) => {
+      return mongoose.models[k].collection.collectionName === coll_name
+    })
+
+    assert(model_name, `unable to resolve model name for coll_name:${coll_name}`)
+
     dispatch_queue(queue, model_name, "delete", doc)
   })
 }
 
 
-const get_dispatch_change_handler = (model_name) => (change) => {
+const dispatch_change_handler = (change) => {
+  // skip if change is on another db
+  if (change?.ns?.db !== RB_APP_NAME) {
+    return
+  }
+
   const op = change.operationType
   if (["insert", "update"].includes(op)) {
+    const coll_name = change.ns.coll
+
+    const model_name = Object.keys(mongoose.models).find((k) => {
+      return mongoose.models[k].collection.collectionName === coll_name
+    })
+
+    assert(model_name, `unable to resolve model name for coll_name:${coll_name}`)
+
     dispatch_queue(queue, model_name, op, change.fullDocument, change.updateDescription)
   }
 }
 
 
-const register_model_emitter = (model_name) => {
-  log("registering emitter for", model_name)
+const register_db_emitter = () => {
+  log("registering db emitter")
 
-  const model = mongoose.model(model_name)
   // TODO: implement delete operation fullDocument retrieve,
   // when this is released https://jira.mongodb.org/browse/SERVER-36941
   // this is done via a plugin right now
   // https://mongodb.github.io/node-mongodb-native/4.9/interfaces/ChangeStreamOptions.html
-  const emitter = model.watch({fullDocument: "updateLookup"})
+  const emitter = mongoose.connection.watch({fullDocument: "updateLookup"})
 
-  emitter.on("change", get_dispatch_change_handler(model_name))
+  emitter.on("change", dispatch_change_handler)
 
   emitter.on("error", (err) => {
-    log("server:queue:register_queue_listener: change listener emitter got error", err)
+    console.log("server:queue:register_queue_listener: change listener emitter got error", err)
   })
 
   emitter.on("close", (arg, arg2) => {
-    // console.log("close", arg, arg2)
     // TODO: investigate why we disconnect here and can't reconnect easily
-    const current_retry = _get(retry_counters, model_name, 0)
-    _set(retry_counters, model_name, current_retry + 1)
+    retry_counter++
 
-    if (current_retry > MAX_RETRIES) {
-      console.log("queue listener reached max retries for:", model_name, "exiting with failure")
+    if (retry_counter > MAX_RETRIES) {
+      console.log("queue listener reached max retries, exiting with failure")
       process.exit(1)
       return
     }
 
-    const timeout_amount = RETRY_MINIMUM_DELAY + 10 * Math.pow(2, current_retry)
-    console.log("queue_listener: emitter closed, model:", model_name, "retrying in", timeout_amount)
+    const timeout_amount = Math.min(RETRY_MAXIMUM_DELAY, RETRY_MINIMUM_DELAY + 10 * Math.pow(2, retry_counter))
+    console.log("queue_listener: emitter closed, retrying in", timeout_amount)
 
     setTimeout(() => {
-      register_model_emitter(model_name)
+      register_db_emitter()
     }, timeout_amount)
   })
 
   emitter.on("end", (arg) => {
-    console.log("emitter ENDED", arg)
+    console.log("EMITTER ENDED", arg)
   })
 }
 
@@ -93,10 +115,7 @@ const register_queue_listener = () => {
   mongoose.plugin(mongoose_delete_plugin)
 
   mongoose.connection.once("open", () => {
-    process.nextTick(() => {
-      const models = mongoose.modelNames()
-      models.forEach((model_name) => register_model_emitter(model_name))
-    })
+    register_db_emitter()
   })
 }