@rpcbase/server 0.172.0 → 0.174.0

package/express/session_middleware.js

@@ -25,7 +25,7 @@ console.log("session-store hostname", hostname, "port", SESSION_STORE_PORT)
 
 let session_middleware = null
 
-// extreme warning: docker issue
+// EXTREME WARNING: docker issue
 // when there is synchronous io redis client fails to connect
 setTimeout(async() => {
   if (!SESSION_STORE_PORT) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rpcbase/server",
-  "version": "0.172.0",
+  "version": "0.174.0",
   "license": "SSPL-1.0",
   "main": "./index.js",
   "bin": {
@@ -10,9 +10,9 @@
     "test": "echo \"Error: no test specified\" && exit 0"
   },
   "dependencies": {
-    "@rpcbase/agent": "0.29.0",
-    "@rpcbase/std": "0.5.0",
-    "body-parser": "1.20.1",
+    "@rpcbase/agent": "0.30.0",
+    "@rpcbase/std": "0.6.0",
+    "body-parser": "1.20.2",
     "bull": "4.10.4",
     "connect-redis": "6.1.3",
     "cors": "2.8.5",
@@ -22,13 +22,13 @@
     "express-session": "1.17.3",
     "glob": "8.1.0",
     "lodash": "4.17.21",
-    "mkdirp": "1.0.4",
-    "mongoose": "6.9.2",
+    "mkdirp": "2.1.3",
+    "mongoose": "6.10.0",
     "picocolors": "1.0.0",
     "postmark": "3.0.15",
     "redis": "4.6.4",
     "request-ip": "3.3.0",
     "validator": "13.9.0",
-    "yargs": "17.7.0"
+    "yargs": "17.7.1"
   }
 }

package/queue/register_queue_listener.js

@@ -1,5 +1,7 @@
 /* @flow */
 const debug = require("debug")
+const _get = require("lodash/get")
+const _set = require("lodash/set")
 
 const mongoose = require("../mongoose")
 const queue = require("./index")
@@ -9,8 +11,11 @@ const dispatch_queue = require("./dispatch_queue")
 
 const log = debug("rb:queue:listener")
 
-const MAX_RETRIES = 100
-let current_retry = 0
+const RETRY_MINIMUM_DELAY = 10
+const MAX_RETRIES = 16
+// TODO: reset retry counters when connection becomes healthy again?
+const retry_counters = {}
+
 
 // Listens for mongodb change events,
 // when a document is created, updated or deleted, dispatch job message
@@ -32,18 +37,7 @@ const mongoose_delete_plugin = (schema) => {
 }
 
 
-const get_dispatch_change_handler = (model_name, filter_fn) => (change) => {
-  if (typeof filter_fn === "function") {
-    let ret
-    try {
-      ret = filter_fn(model_name, change)
-    } catch (err) {
-      console.log("get_dispatch_change_handler::filter_fn::error:", err)
-    }
-    if (!ret) {
-      return
-    }
-  }
+const get_dispatch_change_handler = (model_name) => (change) => {
   const op = change.operationType
   if (["insert", "update"].includes(op)) {
     dispatch_queue(queue, model_name, op, change.fullDocument)
@@ -51,7 +45,7 @@ const get_dispatch_change_handler = (model_name, filter_fn) => (change) => {
 }
 
 
-const register_model_emitter = (model_name, filter_fn) => {
+const register_model_emitter = (model_name) => {
   log("registering emitter for", model_name)
 
   const model = mongoose.model(model_name)
@@ -61,7 +55,7 @@ const register_model_emitter = (model_name, filter_fn) => {
   // https://mongodb.github.io/node-mongodb-native/4.9/interfaces/ChangeStreamOptions.html
   const emitter = model.watch({fullDocument: "updateLookup"})
 
-  emitter.on("change", get_dispatch_change_handler(model_name, filter_fn))
+  emitter.on("change", get_dispatch_change_handler(model_name))
 
   emitter.on("error", (err) => {
     log("server:queue:register_queue_listener: change listener emitter got error", err)
@@ -70,20 +64,21 @@ const register_model_emitter = (model_name, filter_fn) => {
   emitter.on("close", (arg, arg2) => {
     // console.log("close", arg, arg2)
     // TODO: investigate why we disconnect here and can't reconnect easily
-    current_retry++
-    const timeout = 1000 * current_retry
-    console.log("queue_listener: emitter closed, model:", model_name, "retrying in", timeout)
+    const current_retry = _get(retry_counters, model_name, 0)
+    _set(retry_counters, model_name, current_retry + 1)
 
     if (current_retry > MAX_RETRIES) {
-      console.log("queue listener reached max retries, exiting with failure")
+      console.log("queue listener reached max retries for:", model_name, "exiting with failure")
       process.exit(1)
       return
     }
 
+    const timeout = RETRY_MINIMUM_DELAY + 10 * Math.pow(2, current_retry)
+    console.log("queue_listener: emitter closed, model:", model_name, "retrying in", timeout)
+
     setTimeout(() => {
-      register_model_emitter(model_name, filter_fn)
+      register_model_emitter(model_name)
     }, timeout)
-
   })
 
   emitter.on("end", (arg) => {
@@ -92,14 +87,16 @@ const register_model_emitter = (model_name, filter_fn) => {
 }
 
 
-const register_queue_listener = (filter_fn) => {
+const register_queue_listener = () => {
   // register the mongoose delete plugin
   log("registering mongoose_delete_plugin")
   mongoose.plugin(mongoose_delete_plugin)
 
   mongoose.connection.once("open", () => {
-    const models = mongoose.modelNames()
-    models.forEach((model_name) => register_model_emitter(model_name, filter_fn))
+    process.nextTick(() => {
+      const models = mongoose.modelNames()
+      models.forEach((model_name) => register_model_emitter(model_name))
+    })
   })
 }