keuss 1.6.4 → 1.6.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "keuss",
-  "version": "1.6.4",
+  "version": "1.6.8",
   "keywords": [
     "queue",
     "persistent",
@@ -26,19 +26,19 @@
   },
   "license": "GPL-3.0",
   "dependencies": {
-    "@nodebb/mubsub": "~1.6.2",
-    "async": "~3.2.0",
-    "async-lock": "~1.2.4",
-    "debug": "~4.1.1",
-    "ioredis": "~4.17.3",
+    "@nodebb/mubsub": "~1.7.1",
+    "async": "~3.2.1",
+    "async-lock": "~1.3.0",
+    "debug": "~4.3.2",
+    "ioredis": "~4.27.10",
     "lodash": "~4.17.21",
-    "mitt": "~2.1.0",
-    "mongodb": "~3.5.9",
-    "uuid": "~8.3.0"
+    "mitt": "~3.0.0",
+    "mongodb": "~3.6.12",
+    "uuid": "~8.3.2"
   },
   "devDependencies": {
-    "chance": "~1.1.7",
-    "mocha": "~8.4.0",
+    "chance": "~1.1.8",
+    "mocha": "~9.1.2",
     "should": "~13.2.3"
   },
   "scripts": {

package/stats/mem.js

@@ -39,8 +39,10 @@ class MemStats extends Stats {
     if (!this._s.counters[v]) {
       this._s.counters[v] = 0;
     }
-
+    
     if ((delta == null) || (delta == undefined)) delta = 1;
+    var old_v = this._s.counters[v]
+    debug ('incr %s by %d: %d --> %d', v, delta, old_v, this._s.counters[v]);
     this._s.counters[v] = this._s.counters[v] + delta;
     if (cb) cb(null, this._s.counters[v]);
   }

package/test/backends_bucket-at-least-once.js

@@ -52,7 +52,7 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
       function (cb) {q.totalSize (cb)},
       function (cb) {q.next_t (cb)},
     ], function(err, results) {
-      results.should.eql ([{get: 0, put: 0}, 0, 0, null])
+      results.should.eql ([{get: 0, put: 0, reserve: 0, commit: 0, rollback: 0}, 0, 0, null])
       done();
     });
   });
@@ -70,7 +70,7 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
         cb();
       })},
       function (cb) {q.stats (function (err, res) {
-        res.should.eql ({get: 0, put: 2});
+        res.should.eql ({get: 0, put: 2, reserve: 0, commit: 0, rollback: 0});
         cb();
       })},
       function (cb) {q.next_t (function (err, res) {
@@ -86,7 +86,7 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
         cb();
       })},
       function (cb) {q.stats (function (err, res) {
-        res.should.eql ({get: 1, put: 2});
+        res.should.eql ({get: 1, put: 2, reserve: 0, commit: 0, rollback: 0});
         cb();
       })},
       function (cb) {q.pop ('c2', function (err, ret) {
@@ -98,7 +98,7 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
         cb();
       })},
       function (cb) {q.stats (function (err, res) {
-        res.should.eql ({get: 2, put: 2});
+        res.should.eql ({get: 2, put: 2, reserve: 0, commit: 0, rollback: 0});
         cb();
       })},
     ], function(err, results) {
@@ -139,7 +139,7 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
         cb();
       })},
       function (cb) {q.stats (function (err, res) {
-        res.should.eql ({get: 2, put: 2});
+        res.should.eql ({get: 2, put: 2, reserve: 0, commit: 0, rollback: 0});
         cb();
       })},
     ], function(err, results) {
@@ -160,7 +160,7 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
       (cb) => setTimeout (cb, 666),
 
       (cb) => q.size ((err, size) => {size.should.equal (4); cb(); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 4}); cb(); }),
+      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 4, reserve: 0, commit: 0, rollback: 0}); cb(); }),
 
       (cb) => q.push ({elem:5, pl:'twetrwte'}, cb),
       (cb) => q.push ({elem:6, pl:'twetrwte'}, cb),
@@ -168,7 +168,7 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
       (cb) => setTimeout (cb, 666),
 
       (cb) => q.size ((err, size) => {size.should.equal (7); cb(); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 7}); cb(); }),
+      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 7, reserve: 0, commit: 0, rollback: 0}); cb(); }),
 
       (cb) => q.next_t ((err, res) => {res.getTime().should.be.approximately(new Date().getTime(), 2000); cb(); }),
 
@@ -181,7 +181,7 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
       (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:7, pl:'twetrwte'}); cb (err); })},
 
       (cb) => q.size ((err, size) => {size.should.equal (0); cb(); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 7, put: 7}); cb(); }),
+      (cb) => q.stats ((err, res) => {res.should.eql ({get: 7, put: 7, reserve: 0, commit: 0, rollback: 0}); cb(); }),
     ], function(err, results) {
       q.drain();
       done();
@@ -211,7 +211,7 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
       (cb) => q.size ((err, size) => {size.should.equal (0); cb(); }),
       (cb) => q.totalSize ((err, size) => {size.should.equal (7); cb(); }),
       (cb) => q.schedSize ((err, size) => {size.should.equal (7); cb(); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 7}); cb(); }),
+      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 7, reserve: 0, commit: 0, rollback: 0}); cb(); }),
 
       (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:5, pl:'twetrwte'}); cb (err); })},
       (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:6, pl:'twetrwte'}); cb (err); })},
@@ -222,7 +222,7 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
       (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:4, pl:'twetrwte'}); cb (err); })},
 
       (cb) => q.size ((err, size) => {size.should.equal (0); cb(); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 7, put: 7}); cb(); }),
+      (cb) => q.stats ((err, res) => {res.should.eql ({get: 7, put: 7, reserve: 0, commit: 0, rollback: 0}); cb(); }),
     ], function(err, results) {
       q.drain();
       done();
@@ -239,13 +239,13 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
       (cb) => setTimeout (cb, 700),
 
       (cb) => q.size ((err, size) => {size.should.equal (2); cb(); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 2}); cb(); }),
+      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 2, reserve: 0, commit: 0, rollback: 0}); cb(); }),
 
       (cb) => q.push ({elem:3, pl:'twetrwte'}, cb),
       (cb) => setTimeout (cb, 700),
 
       (cb) => q.size ((err, size) => {size.should.equal (3); cb(); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 3}); cb(); }),
+      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 3, reserve: 0, commit: 0, rollback: 0}); cb(); }),
 
       (cb) => q.next_t ((err, res) => {res.getTime().should.be.approximately(new Date().getTime() - 1400, 100); cb(); }),
 
@@ -283,7 +283,7 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
       (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:1, pl:'twetrwte'}); cb (err); })},
 
       (cb) => q.size ((err, size) => {size.should.equal (0); cb(); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 5, put: 3}); cb(); }),
+      (cb) => q.stats ((err, res) => {res.should.eql ({get: 3, put: 3, reserve: 2, commit: 0, rollback: 2}); cb(); }),
     ], function(err, results) {
       q.drain();
       done();

package/test/backends_remove.js

@@ -0,0 +1,299 @@
+var async =  require ('async');
+var should = require ('should');
+var _ =      require ('lodash');
+
+var LocalSignal = require ('../signal/local');
+var MemStats =    require ('../stats/mem');
+
+const MongoClient = require ('mongodb').MongoClient;
+const Redis =       require("ioredis");
+
+function stats (q, cb) {
+  async.series ({
+    stats: cb => q.stats(cb),
+    tsize: cb => q.totalSize(cb),
+    rsize: cb => q.resvSize(cb),
+  }, (err, res) => {
+//    console.log ('stats:', res);
+    cb (err, res);
+  });
+}
+
+function pop (q, stage, cb) {
+  q.pop('c1', { reserve: true }, (err, res) => {
+    stage.obj = res;
+//    console.log('reserved element %o', res);
+    cb(err);
+  });
+}
+
+function reject (q, stage, cb) {
+  var next_t = new Date().getTime() + 2000;
+
+  q.ko (stage.obj, next_t, (err, res) => {
+    if (err) {
+//      console.error ('error in rollback of %s: %j', stage.obj._id, err);
+      return cb (err);
+    }
+
+//    console.log('rolled back element %s: %o', stage.obj._id, res);
+    cb();
+  });
+}
+
+function accept (q, stage, cb) {
+  q.ok (stage.obj, (err, res) => {
+    if (err) {
+//      console.error ('error in rollback of %s: %j', stage.obj._id, err);
+      return cb (err);
+    }
+
+//    console.log('commited element %s: %j', stage.obj._id, res);
+    cb();
+  });
+}
+
+function get_mq_factory (MQ, opts, cb) {
+  const common_opts = {
+    url: 'mongodb://localhost/keuss_test_backends_remove',
+    signaller: { provider: LocalSignal},
+    stats: {provider: MemStats},
+    deadletter: {
+    }
+  };
+
+  // initialize factory
+  MQ (_.merge ({}, common_opts, opts), cb);
+}
+
+function release_mq_factory (q, factory, cb) {
+//  console.log ('releasing mq factory');
+
+  setTimeout (() => {
+    q.cancel ();
+    factory.close (cb);
+  }, 1000);
+}
+
+
+/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+[
+  {label: 'Simple MongoDB',     mq: require ('../backends/mongo')},
+  {label: 'Pipelined MongoDB',  mq: require ('../backends/pl-mongo')},
+  {label: 'Tape MongoDB',       mq: require ('../backends/ps-mongo')},
+  {label: 'Redis OrderedQueue', mq: require ('../backends/redis-oq')},
+  {label: 'MongoDB SafeBucket', mq: require ('../backends/bucket-mongo-safe')}
+].forEach(function (MQ_item) {
+  var is_redis = (MQ_item.label == 'Redis OrderedQueue');
+
+  describe('remove operations on ' + MQ_item.label + ' queue backend', () => {
+    const MQ = MQ_item.mq;
+
+    beforeEach (done => {
+      done();
+    });
+
+    afterEach (done => async.series ([
+      cb => setTimeout (cb, 1000),
+      cb => MongoClient.connect ('mongodb://localhost/keuss_test_backends_remove', (err, cl) => {
+        if (err) return done (err);
+        cl.db().dropDatabase (() => cl.close (cb))
+      }),
+      cb => {
+        const redis = new Redis();
+        redis.keys ('*', (err, keys) => {
+          const tasks = [];
+          _.each (keys, k => tasks.push (cb => redis.del (k, cb))); 
+          async.series (tasks, (err, res) => {
+            redis.disconnect();
+            cb (err, res);
+          });
+        });
+      }
+    ], done));
+
+
+    it('fails on invalid id', done => {
+      // skip if redis
+      if (is_redis) return done();
+
+      async.waterfall ([
+        cb => get_mq_factory (MQ, {}, cb),
+        (factory, cb) => {
+          const q = factory.queue('test_queue_remove', {});
+          q.remove ('invalid-id', err => cb (null, err, q, factory))
+        },
+        (err, q, factory, cb) => {
+          err.should.match (/id \[invalid-id\] can not be used as remove id/);
+          release_mq_factory (q, factory, cb);
+        }
+      ], done);
+    });
+
+
+    it('fails on nonexistent id', done => {
+      async.waterfall ([
+        cb => get_mq_factory (MQ, {}, cb),
+        (factory, cb) => {
+          const q = factory.queue('test_queue_remove', {});
+          q.remove ('00112233445566778899aabb', (err, res) => cb (null, err, res, q, factory))
+        },
+        (err, res, q, factory, cb) => {
+          should (err).be.null();
+          res.should.be.false();
+          release_mq_factory (q, factory, cb);
+        }
+      ], done);
+    });
+
+
+    it('deletes regular element ok', done => {
+      const ctx = {};
+      async.waterfall ([
+        cb => get_mq_factory (MQ, {}, cb),
+        (factory, cb) => {
+          const q = factory.queue('test_queue_remove', {});
+          cb (null, q, factory);
+        },
+        (q, factory, cb) => async.series ([
+          cb => q.push ({a: 1, b: 'oo'}, cb),
+          cb => setTimeout (cb, 1000),
+          cb => stats (q, cb)
+        ], (err, res) => {
+          ctx.id = res[0];
+          ctx.id.should.not.be.null();
+          res[2].should.eql ({
+            stats: { get: 0, put: 1, reserve: 0, commit: 0, rollback: 0 },
+            tsize: 1,
+            rsize: is_redis ? null : 0
+          })
+          cb (err, q, factory);
+        }),
+        (q, factory, cb) => {
+          q.remove (ctx.id, (err, res) => {
+            should (err).be.null();
+            res.should.be.true();
+            cb (err, q, factory);
+          });
+        },
+        (q, factory, cb) => async.series ([
+          cb => stats (q, cb),
+          cb => setTimeout (cb, 1000),
+        ], (err, res) => {
+          res[0].should.eql ({
+            stats: { get: 0, put: 1, reserve: 0, commit: 0, rollback: 0 },
+            tsize: 0,
+            rsize: is_redis ? null : 0
+          });
+          cb (err, q, factory);
+        }),
+        (q, factory, cb) => {
+          release_mq_factory (q, factory, cb);
+        }
+      ], done);
+    });
+
+
+    it('does not delete reserved element', done => {
+      const ctx = {};
+      async.waterfall ([
+        cb => get_mq_factory (MQ, {}, cb),
+        (factory, cb) => {
+          const q = factory.queue('test_queue_remove', {});
+          cb (null, q, factory);
+        },
+        (q, factory, cb) => async.series ([
+          cb => q.push ({a: 1, b: 'oo'}, cb),
+          cb => setTimeout (cb, 1000),
+        ], (err, res) => {
+          ctx.id = res[0];
+          ctx.id.should.not.be.null();
+          cb (err, q, factory);
+        }),
+        (q, factory, cb) => { 
+          q.pop('c1', { reserve: true }, (err, res) => {
+//            res._id.should.eql (ctx.id);
+            cb (err, q, factory);
+          });
+        },
+        (q, factory, cb) => {
+          q.remove (ctx.id, (err, res) => {
+            should (err).be.null();
+            res.should.be.false();
+            cb (err, q, factory);
+          });
+        },
+        (q, factory, cb) => async.series ([
+          cb => stats (q, cb),
+          cb => setTimeout (cb, 1000),
+        ], (err, res) => {
+          res[0].should.eql ({
+            stats: { get: 0, put: 1, reserve: 1, commit: 0, rollback: 0 },
+            tsize: 1,
+            rsize: is_redis ? null : 1
+          });
+          cb (err, q, factory);
+        }),
+        (q, factory, cb) => {
+          release_mq_factory (q, factory, cb);
+        }
+      ], done);
+    });
+
+
+    it('does delete reserved+rolledback element', done => {
+      const ctx = {};
+      async.waterfall ([
+        cb => get_mq_factory (MQ, {}, cb),
+        (factory, cb) => {
+          const q = factory.queue('test_queue_remove', {});
+          cb (null, q, factory);
+        },
+        (q, factory, cb) => async.series ([
+          cb => q.push ({a: 1, b: 'oo'}, cb),
+          cb => setTimeout (cb, 1000),
+        ], (err, res) => {
+          ctx.id = res[0];
+          ctx.id.should.not.be.null();
+          cb (err, q, factory);
+        }),
+        (q, factory, cb) => { 
+          q.pop('c1', { reserve: true }, (err, res) => {
+            res._id.should.eql (ctx.id);
+            cb (err, q, factory);
+          });
+        },
+        (q, factory, cb) =>  async.series ([ 
+          cb => q.ko(ctx.id, new Date().getTime() + 2000, cb),
+          cb => setTimeout (cb, 1000),
+        ], (err, res) => {
+          res[0].should.be.true();
+          cb (err, q, factory);
+        }),
+        (q, factory, cb) => {
+          q.remove (ctx.id, (err, res) => {
+            should (err).be.null();
+            res.should.be.true();
+            cb (err, q, factory);
+          });
+        },
+        (q, factory, cb) => async.series ([
+          cb => stats (q, cb),
+          cb => setTimeout (cb, 1000),
+        ], (err, res) => {
+          res[0].should.eql ({
+            stats: { get: 0, put: 1, reserve: 1, commit: 0, rollback: 1 },
+            tsize: 0,
+            rsize: is_redis ? null : 0
+          });
+          cb (err, q, factory);
+        }),
+        (q, factory, cb) => {
+          release_mq_factory (q, factory, cb);
+        }
+      ], done);
+    });
+
+
+  });
+});