keuss 2.0.7 → 2.2.0

package/lib/mubsub/connection.js

@@ -0,0 +1,79 @@
+const EventEmitter = require('events')
+const MongoClient = require('mongodb').MongoClient
+const Channel = require('./channel')
+const noop = function () {}
+
+class Connection extends EventEmitter {
+  constructor (uri, options) {
+    super()
+    const self = this
+
+    options || (options = {})
+
+    // It's a Db instance.
+    if (uri.collection) {
+      this.db = uri
+    } else {
+      const p = MongoClient.connect(uri, options)
+      p.then((client) => {
+        self.client = client
+        self.db = client.db()
+        self.emit('connect', self.db)
+        self.client.on('error', function (err) {
+          self.emit('error', err)
+        })
+      }).catch((err) => {
+        self.emit('error', err)
+      })
+    }
+
+    this.destroyed = false
+    this.channels = {}
+  }
+
+  get state () {
+    let state
+
+    // Using 'destroyed' to be compatible with the driver.
+    if (this.destroyed) {
+      state = 'destroyed'
+    } else if (
+      (this.db && !this.client) ||
+      this.client?.topology?.isConnected()
+    ) {
+      // https://github.com/mongodb/node-mongodb-native/blob/d266158c9e968c92e8041211ef99f1783025be40/src/operations/connect.ts#L18
+      state = 'connected'
+    } else {
+      state = 'connecting'
+    }
+
+    return state
+  }
+
+  channel (name, options) {
+    if (typeof name === 'object') {
+      options = name
+      name = 'mubsub'
+    }
+
+    if (!this.channels[name] || this.channels[name].closed) {
+      this.channels[name] = new Channel(this, name, options)
+    }
+
+    return this.channels[name]
+  }
+
+  close (callback) {
+    this.destroyed = true
+    if (this.client) {
+      callback || (callback = noop)
+      this.client
+        .close()
+        .then(() => callback())
+        .catch((e) => callback(e))
+    }
+    return this
+  }
+}
+
+module.exports = Connection

package/lib/mubsub/index.js

@@ -0,0 +1,10 @@
+const Connection = require('./connection')
+const Channel = require('./channel')
+const mongodb = require('mongodb')
+
+module.exports = exports = function (uri, options) {
+  return new Connection(uri, options)
+}
+
+exports.Connection = Connection
+exports.Channel = Channel

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "keuss",
-  "version": "2.0.7",
+  "version": "2.2.0",
   "keywords": [
     "queue",
     "persistent",
@@ -28,23 +28,21 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@nodebb/mubsub": "~1.8.0",
     "async": "~3.2.6",
     "async-lock": "~1.4.1",
     "debug": "~4.3.6",
     "ioredis": "~5.4.1",
     "lodash": "~4.17.21",
     "mitt": "~3.0.1",
-    "mongodb": "~4.17.0",
+    "mongodb": "^6.0.0",
     "uuid": "~8.3.2",
     "pg": "~8.12.0"
   },
   "devDependencies": {
-    "chance": "~1.1.12",
-    "mocha": "~10.7.3",
+    "chance": "~1.1.13",
+    "mocha": "~11.7.5",
     "should": "~13.2.3",
-    "nyc": "~17.0.0",
-    "why-is-node-running": "~3.2.0"
+    "nyc": "~17.1.0"
   },
   "scripts": {
     "test": "docker compose up -d; sleep 5; mocha --reporter spec --check-leaks --no-timeouts --exit test/ ; docker compose down",

package/signal/mongo-capped.js

@@ -1,4 +1,4 @@
-var mubsub = require('@nodebb/mubsub');
+var mubsub = require('../lib/mubsub');
 var _ =      require('lodash');
 
 var Signal = require ('../Signal');

package/stats/mongo.js

@@ -1,11 +1,13 @@
-var _ =           require ('lodash');
-var async =       require ('async');
-var MongoClient = require ('mongodb').MongoClient;
+const  _ =          require ('lodash');
+const async =       require ('async');
+const MongoClient = require ('mongodb').MongoClient;
 
-var Stats = require ('../Stats');
+const Stats = require ('../Stats');
 
-var debug = require('debug')('keuss:Stats:Mongo');
+const debug = require('debug')('keuss:Stats:Mongo');
 
+
+//////////////////////////////////////////////////////////////////////////////////
 /*
  * plain into a single mongo coll
 */
@@ -16,52 +18,62 @@ class MongoStats  extends Stats {
     this._opts = opts || {};
     this._cache = {};
 
-    var upd = {
+    const upd = {
       $set: {
         ns:   this._ns,
         name: this._name,
       }
     };
 
-    this._coll().updateOne ({_id: this._id}, upd, {upsert: true}, (err, ret) => {
+    this._coll().updateOne ({_id: this._id}, upd, {upsert: true})
+    .finally (() => {
       debug ('mongo stats created, ns %s, name %s, opts %j', ns, name, opts);
     });
   }
 
-  values(cb) {
-    this._coll().findOne ({_id: this._id}, {projection: {counters: 1}}, (err, res) => {
-      if (err) return cb (err);
 
+  //////////////////////////////////////////////////////////////////////////////////
+  values(cb) {
+    this._coll().findOne ({_id: this._id}, {projection: {counters: 1}})
+    .then (res => {
       debug ('mongo stats: get %s -> %j', this._name, res);
       cb (null, (res && res.counters) || {});
-    });
+    })
+    .catch (cb);
   }
 
+
+  //////////////////////////////////////////////////////////////////////////////////
   paused (val, cb) {
     if (!cb) {
       // get, val is cb
       cb = val;
       val = undefined;
 
-      this._coll().findOne ({_id: this._id}, {projection: {paused: 1}}, (err, res) => {
-        if (err) return cb (err);
+      this._coll().findOne ({_id: this._id}, {projection: {paused: 1}})
+      .then (res => {
         debug ('mongo stats - paused: get %s -> %j', this._name, res);
         cb (null, (res && res.paused) || false);
-      });
+      })
+      .catch (cb)
     }
     else {
       // set
-      var upd = {$set: {paused : val}};
-      this._coll().updateOne ({_id: this._id}, upd, {upsert: true}, (err) => {
+      const upd = {$set: {paused : val}};
+      this._coll().updateOne ({_id: this._id}, upd, {upsert: true})
+      .then (res => {
         debug  ('mongo stats: updated %s -> %j', this._name, upd);
-        cb (err);
-      });
+        cb ();
+      })
+      .catch (cb);
     }
   }
 
+
+  //////////////////////////////////////////////////////////////////////////////////
   _flush (cb) {
-    var upd = {$inc: {}};
-    var some_added = false;
+    const upd = {$inc: {}};
+    let some_added = false;
 
     _.forEach(this._cache, (value, key) => {
       if (value) {
@@ -72,9 +84,13 @@ class MongoStats  extends Stats {
     });
 
     if (some_added) {
-      this._coll().updateOne ({_id: this._id}, upd, {upsert: true}, (err) => {
+      this._coll().updateOne ({_id: this._id}, upd, {upsert: true})
+      .then (res => {
         debug ('mongo stats: updated %s -> %j', this._name, upd);
-        if (cb) cb (err);
+        if (cb) cb ();
+      })
+      .catch (err => {
+        if (cb) return cb(err);
       });
     }
     else {
@@ -83,6 +99,7 @@ class MongoStats  extends Stats {
   }
 
 
+  //////////////////////////////////////////////////////////////////////////////////
   _ensureFlush() {
     if (this._flusher) return;
 
@@ -93,6 +110,7 @@ class MongoStats  extends Stats {
   }
 
 
+  //////////////////////////////////////////////////////////////////////////////////
   _cancelFlush() {
     if (this._flusher) {
       clearTimeout(this._flusher);
@@ -101,16 +119,19 @@ class MongoStats  extends Stats {
   }
 
 
+  //////////////////////////////////////////////////////////////////////////////////
   _mongocl () {
     return this._factory._mongocl;
   }
 
 
+  //////////////////////////////////////////////////////////////////////////////////
   _coll () {
     return this._factory._coll;
   }
 
 
+  //////////////////////////////////////////////////////////////////////////////////
   incr(v, delta, cb) {
     if ((delta === null) || (delta === undefined)) delta = 1;
 
@@ -125,48 +146,57 @@ class MongoStats  extends Stats {
   }
 
 
+  //////////////////////////////////////////////////////////////////////////////////
   decr(v, delta, cb) {
     if ((delta === null) || (delta === undefined)) delta = 1;
     this.incr(v, -delta, cb);
   }
 
+
+  //////////////////////////////////////////////////////////////////////////////////
   opts (opts, cb) {
     if (!cb) {
       // get
       cb = opts;
-      this._coll().findOne ({_id: this._id}, {projection: {opts: 1}}, (err, res) => {
-        if (err) return cb (err);
+      this._coll().findOne ({_id: this._id}, {projection: {opts: 1}})
+      .then (res => {
         debug ('mongo stats - opts: get %s -> %j', this._name, res);
         cb (null, (res && res.opts) || {});
-      });
+      })
+      .catch (cb);
     }
     else {
       // set
-      var upd = {$set: {opts : opts}};
-      this._coll().updateOne ({_id: this._id}, upd, {upsert: true}, (err) => {
+      const upd = {$set: {opts : opts}};
+      this._coll().updateOne ({_id: this._id}, upd, {upsert: true})
+      .then (res => {
         debug ('mongo stats: updated %s -> %j', this._name, upd);
-        cb (err);
-      });
+        cb ();
+      })
+      .catch (cb);
     }
   }
 
+
+  //////////////////////////////////////////////////////////////////////////////////
   clear(cb) {
     this._cancelFlush();
     this._cache = {};
 
-    var upd = {
+    const upd = {
       $unset: {
         counters: 1,
         opts: 1
       }
     };
 
-    this._coll().updateOne ({_id: this._id}, upd, (err) => {
-      cb (err);
-    });
+    this._coll().updateOne ({_id: this._id}, upd)
+    .then (() => cb ())
+    .catch(cb);
   }
 
 
+  //////////////////////////////////////////////////////////////////////////////////
   close (cb) {
     this._cancelFlush();
     this._flush (cb);
@@ -175,6 +205,7 @@ class MongoStats  extends Stats {
 
 
 
+//////////////////////////////////////////////////////////////////////////////////
 class MongoStatsFactory {
   constructor(cl, coll, opts) {
     this._opts = opts || {};
@@ -190,6 +221,7 @@ class MongoStatsFactory {
   type() { return MongoStatsFactory.Type() }
 
 
+  //////////////////////////////////////////////////////////////////////////////////
   stats(ns, name, opts) {
     var k = name + '@' + ns;
     if (!this._instances [k]) {
@@ -200,6 +232,7 @@ class MongoStatsFactory {
     return this._instances [k];
   }
 
+  //////////////////////////////////////////////////////////////////////////////////
   queues (ns, opts, cb) {
     if (!cb) {
       cb = opts;
@@ -207,11 +240,11 @@ class MongoStatsFactory {
     }
 
     if (opts.full) {
-      this._coll.find({_id: {$regex: '^keuss:stats:' + ns}}).toArray (function (err, arr) {
-        if (err) return cb (err);
-
-        var res = {};
-        arr.forEach (function (elem){
+      this._coll.find({_id: {$regex: '^keuss:stats:' + ns}}).
+      toArray ()
+      .then (arr => {
+        const res = {};
+        arr.forEach (elem => {
           res [elem.name] = {
             ns: elem.ns,
             name: elem.name,
@@ -222,24 +255,25 @@ class MongoStatsFactory {
         });
 
         cb (null, res);
-      });
+      })
+      .catch (cb);
     }
     else {
-      this._coll.find({_id: {$regex: '^keuss:stats:' + ns}}).project ({_id: 1, name: 1}).toArray (function (err, arr) {
-        if (err) return cb (err);
-
-        var res = [];
-        arr.forEach (function (elem){
-          res.push (elem.name);
-        });
-
+      this._coll.find({_id: {$regex: '^keuss:stats:' + ns}})
+      .project ({_id: 1, name: 1})
+      .toArray ()
+      .then (arr => {
+        const res = [];
+        arr.forEach (elem => res.push (elem.name));
         cb (null, res);
-      });
+      })
+      .catch (cb);
     }
   }
 
+  //////////////////////////////////////////////////////////////////////////////////
   close (cb) {
-    var tasks = [];
+    const tasks = [];
 
     // flush pending stats
     _.each (this._instances, (v, k) => {
@@ -253,12 +287,15 @@ class MongoStatsFactory {
       (cb) => async.parallel (tasks, cb),
       (cb) => {
         debug (`closing MongoStatsFactory mongodb conn`);
-        this._mongocl.close (cb);
+        this._mongocl.close ()
+        .then (res => cb (null, res))
+        .catch (cb);
       }
     ], cb);
   }
 }
 
+//////////////////////////////////////////////////////////////////////////////////
 function creator (opts, cb) {
   if (!cb) {
     cb = opts;
@@ -274,13 +311,13 @@ function creator (opts, cb) {
 
   debug ('initializing creator of MongoStatsFactory, connecting to %s', m_url);
 
-  MongoClient.connect (m_url, { useNewUrlParser: true }, function (err, cl) {
-    if (err) return cb (err);
-
+  MongoClient.connect (m_url)
+  .then (cl => {
     debug ('initializing creator of MongoStatsFactory, connected to %s', m_url);
     var coll = cl.db().collection (m_coll);
     cb (null, new MongoStatsFactory (cl, coll, opts));
-  });
+  })
+  .catch (cb);
 }
 
 module.exports = creator;

package/utils/RedisOrderedQueue.js

@@ -7,6 +7,13 @@ const _s_lua_code_push = `
   -- mature-t in ARGV[2]
   -- val in ARGV[3]
 
+  -- check id exists
+  local exists = redis.call ('HGET', 'keuss:q:ordered_queue:hash:' .. KEYS[1], ARGV[1])
+
+  if (exists) then
+    return redis.error_reply('EDUP duplicated _id ' .. ARGV[1])
+  end
+
   -- insert obj in hash by id
   redis.call ('HSET', 'keuss:q:ordered_queue:hash:' .. KEYS[1], ARGV[1], ARGV[3])
 
@@ -174,20 +181,33 @@ class RedisOrderedQueue {
   push (entry, done) {
   //////////////////////////////////
     var pl = {
-      _id:     entry.id || uuid.v4(),
+      _id:     entry._id || uuid.v4(),
       payload: entry.payload,
       tries:   entry.tries,
       hdrs:    entry.hdrs || {},
       mature:  (entry.mature || new Date ()).getTime ()
     };
 
-
     if (Buffer.isBuffer (pl.payload)) {
       pl.payload = pl.payload.toString ('base64');
       pl.type = 'buffer';
     }
 
-    this._rediscl.roq_push (this._name, pl._id, pl.mature, JSON.stringify (pl), done);
+    this._rediscl.roq_push (this._name, pl._id, pl.mature, JSON.stringify (pl), (err, res) => {
+      if (err) {
+        if (err.message.startsWith ('EDUP ')) {
+          const e = new Error (`duplicated entry with _id ${pl._id}`)
+          e.code = 'EDUP';
+          done(e);
+        }
+        else {
+          done (err)
+        }
+      }
+      else {
+        done (null, res);
+      }
+    });
   }