keuss 1.6.7 → 1.6.8

package/test/backends_remove.js

@@ -0,0 +1,299 @@
+var async =  require ('async');
+var should = require ('should');
+var _ =      require ('lodash');
+
+var LocalSignal = require ('../signal/local');
+var MemStats =    require ('../stats/mem');
+
+const MongoClient = require ('mongodb').MongoClient;
+const Redis =       require("ioredis");
+
+function stats (q, cb) {
+  async.series ({
+    stats: cb => q.stats(cb),
+    tsize: cb => q.totalSize(cb),
+    rsize: cb => q.resvSize(cb),
+  }, (err, res) => {
+//    console.log ('stats:', res);
+    cb (err, res);
+  });
+}
+
+function pop (q, stage, cb) {
+  q.pop('c1', { reserve: true }, (err, res) => {
+    stage.obj = res;
+//    console.log('reserved element %o', res);
+    cb(err);
+  });
+}
+
+function reject (q, stage, cb) {
+  var next_t = new Date().getTime() + 2000;
+
+  q.ko (stage.obj, next_t, (err, res) => {
+    if (err) {
+//      console.error ('error in rollback of %s: %j', stage.obj._id, err);
+      return cb (err);
+    }
+
+//    console.log('rolled back element %s: %o', stage.obj._id, res);
+    cb();
+  });
+}
+
+function accept (q, stage, cb) {
+  q.ok (stage.obj, (err, res) => {
+    if (err) {
+//      console.error ('error in rollback of %s: %j', stage.obj._id, err);
+      return cb (err);
+    }
+
+//    console.log('commited element %s: %j', stage.obj._id, res);
+    cb();
+  });
+}
+
+function get_mq_factory (MQ, opts, cb) {
+  const common_opts = {
+    url: 'mongodb://localhost/keuss_test_backends_remove',
+    signaller: { provider: LocalSignal},
+    stats: {provider: MemStats},
+    deadletter: {
+    }
+  };
+
+  // initialize factory
+  MQ (_.merge ({}, common_opts, opts), cb);
+}
+
+function release_mq_factory (q, factory, cb) {
+//  console.log ('releasing mq factory');
+
+  setTimeout (() => {
+    q.cancel ();
+    factory.close (cb);
+  }, 1000);
+}
+
+
+/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+[
+  {label: 'Simple MongoDB',     mq: require ('../backends/mongo')},
+  {label: 'Pipelined MongoDB',  mq: require ('../backends/pl-mongo')},
+  {label: 'Tape MongoDB',       mq: require ('../backends/ps-mongo')},
+  {label: 'Redis OrderedQueue', mq: require ('../backends/redis-oq')},
+  {label: 'MongoDB SafeBucket', mq: require ('../backends/bucket-mongo-safe')}
+].forEach(function (MQ_item) {
+  var is_redis = (MQ_item.label == 'Redis OrderedQueue');
+
+  describe('remove operations on ' + MQ_item.label + ' queue backend', () => {
+    const MQ = MQ_item.mq;
+
+    beforeEach (done => {
+      done();
+    });
+
+    afterEach (done => async.series ([
+      cb => setTimeout (cb, 1000),
+      cb => MongoClient.connect ('mongodb://localhost/keuss_test_backends_remove', (err, cl) => {
+        if (err) return done (err);
+        cl.db().dropDatabase (() => cl.close (cb))
+      }),
+      cb => {
+        const redis = new Redis();
+        redis.keys ('*', (err, keys) => {
+          const tasks = [];
+          _.each (keys, k => tasks.push (cb => redis.del (k, cb))); 
+          async.series (tasks, (err, res) => {
+            redis.disconnect();
+            cb (err, res);
+          });
+        });
+      }
+    ], done));
+
+
+    it('fails on invalid id', done => {
+      // skip if redis
+      if (is_redis) return done();
+
+      async.waterfall ([
+        cb => get_mq_factory (MQ, {}, cb),
+        (factory, cb) => {
+          const q = factory.queue('test_queue_remove', {});
+          q.remove ('invalid-id', err => cb (null, err, q, factory))
+        },
+        (err, q, factory, cb) => {
+          err.should.match (/id \[invalid-id\] can not be used as remove id/);
+          release_mq_factory (q, factory, cb);
+        }
+      ], done);
+    });
+
+
+    it('fails on nonexistent id', done => {
+      async.waterfall ([
+        cb => get_mq_factory (MQ, {}, cb),
+        (factory, cb) => {
+          const q = factory.queue('test_queue_remove', {});
+          q.remove ('00112233445566778899aabb', (err, res) => cb (null, err, res, q, factory))
+        },
+        (err, res, q, factory, cb) => {
+          should (err).be.null();
+          res.should.be.false();
+          release_mq_factory (q, factory, cb);
+        }
+      ], done);
+    });
+
+
+    it('deletes regular element ok', done => {
+      const ctx = {};
+      async.waterfall ([
+        cb => get_mq_factory (MQ, {}, cb),
+        (factory, cb) => {
+          const q = factory.queue('test_queue_remove', {});
+          cb (null, q, factory);
+        },
+        (q, factory, cb) => async.series ([
+          cb => q.push ({a: 1, b: 'oo'}, cb),
+          cb => setTimeout (cb, 1000),
+          cb => stats (q, cb)
+        ], (err, res) => {
+          ctx.id = res[0];
+          ctx.id.should.not.be.null();
+          res[2].should.eql ({
+            stats: { get: 0, put: 1, reserve: 0, commit: 0, rollback: 0 },
+            tsize: 1,
+            rsize: is_redis ? null : 0
+          })
+          cb (err, q, factory);
+        }),
+        (q, factory, cb) => {
+          q.remove (ctx.id, (err, res) => {
+            should (err).be.null();
+            res.should.be.true();
+            cb (err, q, factory);
+          });
+        },
+        (q, factory, cb) => async.series ([
+          cb => stats (q, cb),
+          cb => setTimeout (cb, 1000),
+        ], (err, res) => {
+          res[0].should.eql ({
+            stats: { get: 0, put: 1, reserve: 0, commit: 0, rollback: 0 },
+            tsize: 0,
+            rsize: is_redis ? null : 0
+          });
+          cb (err, q, factory);
+        }),
+        (q, factory, cb) => {
+          release_mq_factory (q, factory, cb);
+        }
+      ], done);
+    });
+
+
+    it('does not delete reserved element', done => {
+      const ctx = {};
+      async.waterfall ([
+        cb => get_mq_factory (MQ, {}, cb),
+        (factory, cb) => {
+          const q = factory.queue('test_queue_remove', {});
+          cb (null, q, factory);
+        },
+        (q, factory, cb) => async.series ([
+          cb => q.push ({a: 1, b: 'oo'}, cb),
+          cb => setTimeout (cb, 1000),
+        ], (err, res) => {
+          ctx.id = res[0];
+          ctx.id.should.not.be.null();
+          cb (err, q, factory);
+        }),
+        (q, factory, cb) => { 
+          q.pop('c1', { reserve: true }, (err, res) => {
+//            res._id.should.eql (ctx.id);
+            cb (err, q, factory);
+          });
+        },
+        (q, factory, cb) => {
+          q.remove (ctx.id, (err, res) => {
+            should (err).be.null();
+            res.should.be.false();
+            cb (err, q, factory);
+          });
+        },
+        (q, factory, cb) => async.series ([
+          cb => stats (q, cb),
+          cb => setTimeout (cb, 1000),
+        ], (err, res) => {
+          res[0].should.eql ({
+            stats: { get: 0, put: 1, reserve: 1, commit: 0, rollback: 0 },
+            tsize: 1,
+            rsize: is_redis ? null : 1
+          });
+          cb (err, q, factory);
+        }),
+        (q, factory, cb) => {
+          release_mq_factory (q, factory, cb);
+        }
+      ], done);
+    });
+
+
+    it('does delete reserved+rolledback element', done => {
+      const ctx = {};
+      async.waterfall ([
+        cb => get_mq_factory (MQ, {}, cb),
+        (factory, cb) => {
+          const q = factory.queue('test_queue_remove', {});
+          cb (null, q, factory);
+        },
+        (q, factory, cb) => async.series ([
+          cb => q.push ({a: 1, b: 'oo'}, cb),
+          cb => setTimeout (cb, 1000),
+        ], (err, res) => {
+          ctx.id = res[0];
+          ctx.id.should.not.be.null();
+          cb (err, q, factory);
+        }),
+        (q, factory, cb) => { 
+          q.pop('c1', { reserve: true }, (err, res) => {
+            res._id.should.eql (ctx.id);
+            cb (err, q, factory);
+          });
+        },
+        (q, factory, cb) =>  async.series ([ 
+          cb => q.ko(ctx.id, new Date().getTime() + 2000, cb),
+          cb => setTimeout (cb, 1000),
+        ], (err, res) => {
+          res[0].should.be.true();
+          cb (err, q, factory);
+        }),
+        (q, factory, cb) => {
+          q.remove (ctx.id, (err, res) => {
+            should (err).be.null();
+            res.should.be.true();
+            cb (err, q, factory);
+          });
+        },
+        (q, factory, cb) => async.series ([
+          cb => stats (q, cb),
+          cb => setTimeout (cb, 1000),
+        ], (err, res) => {
+          res[0].should.eql ({
+            stats: { get: 0, put: 1, reserve: 1, commit: 0, rollback: 1 },
+            tsize: 0,
+            rsize: is_redis ? null : 0
+          });
+          cb (err, q, factory);
+        }),
+        (q, factory, cb) => {
+          release_mq_factory (q, factory, cb);
+        }
+      ], done);
+    });
+
+
+  });
+});

package/utils/RedisOrderedQueue.js

@@ -11,7 +11,8 @@ const _s_lua_code_push = `
   redis.call ('HSET', 'keuss:q:ordered_queue:hash:' .. KEYS[1], ARGV[1], ARGV[3])
 
   -- insert id+mature in index
-  return redis.call ('ZADD', 'keuss:q:ordered_queue:index:' .. KEYS[1], ARGV[2], ARGV[1])
+  redis.call ('ZADD', 'keuss:q:ordered_queue:index:' .. KEYS[1], ARGV[2], ARGV[1])
+  return ARGV[1]
 `;
 
 const _s_lua_code_pop = `
@@ -86,7 +87,7 @@ const _s_lua_code_commit = `
   local id = ARGV[1]
   local val = redis.call ('HGET', 'keuss:q:ordered_queue:hash:' .. KEYS[1], id)
 
-  if (val == nil) then
+  if (not(val)) then
     return nil
   end
 
@@ -113,7 +114,7 @@ const _s_lua_code_rollback = `
   local id = ARGV[1]
   local val = redis.call ('HGET', 'keuss:q:ordered_queue:hash:' .. KEYS[1], id)
 
-  if (val==false) then
+  if (not(val)) then
     return nil
   end
 
@@ -135,6 +136,31 @@ const _s_lua_code_rollback = `
   return id
 `;
 
+const _s_lua_code_remove = `
+  -- qname in KEYS[1]
+  -- id in ARGV[1]
+
+  local id = ARGV[1]
+  local val = redis.call ('HGET', 'keuss:q:ordered_queue:hash:' .. KEYS[1], id)
+
+  if (not (val)) then
+    return nil
+  end
+
+  -- check if it was reserved
+  local obj_val = cjson.decode (val)
+
+  if (obj_val.reserved) then
+    -- it is reserved
+    return nil
+  end
+
+  -- delete from index, hash
+  redis.call ('ZREM', 'keuss:q:ordered_queue:index:' .. KEYS[1], id)
+  redis.call ('HDEL', 'keuss:q:ordered_queue:hash:' ..  KEYS[1], id)
+
+  return id
+`;
 
 
 class RedisOrderedQueue {
@@ -164,6 +190,7 @@ class RedisOrderedQueue {
     this._rediscl.roq_push (this._name, pl._id, pl.mature, JSON.stringify (pl), done);
   }
 
+
   //////////////////////////////////
   pop (done) {
   //////////////////////////////////
@@ -184,6 +211,7 @@ class RedisOrderedQueue {
     });
   }
 
+
   //////////////////////////////////
   reserve (incr, done) {
   //////////////////////////////////
@@ -204,18 +232,21 @@ class RedisOrderedQueue {
     });
   }
 
+
   //////////////////////////////////
   commit (id, done) {
   //////////////////////////////////
     this._rediscl.roq_commit (this._name, id, done);
   }
 
+
   //////////////////////////////////
   rollback (id, next_t, done) {
   //////////////////////////////////
     this._rediscl.roq_rollback (this._name, id, next_t || (new Date().getTime ()), done);
   }
 
+
   //////////////////////////////////
   // queue size including non-mature elements
   totalSize (callback) {
@@ -223,6 +254,7 @@ class RedisOrderedQueue {
     this._rediscl.zcard ('keuss:q:ordered_queue:index:' + this._name,  callback);
   }
 
+
   //////////////////////////////////
   // queue size NOT including non-mature elements
   size (callback) {
@@ -230,6 +262,7 @@ class RedisOrderedQueue {
     this._rediscl.zcount ('keuss:q:ordered_queue:index:' + this._name, '-inf', new Date().getTime(), callback);
   }
 
+
   //////////////////////////////////
   // queue size of non-mature elements only
   schedSize (callback) {
@@ -237,12 +270,20 @@ class RedisOrderedQueue {
     this._rediscl.zcount ('keuss:q:ordered_queue:index:' + this._name, new Date().getTime(), '+inf', callback);
   }
 
+
   //////////////////////////////////
   // get first
   peek (callback) {
   //////////////////////////////////
     this._rediscl.zrange ('keuss:q:ordered_queue:index:' + this._name, 0, 0, 'WITHSCORES', callback);
   }
+
+
+  //////////////////////////////////
+  remove (id, done) {
+  //////////////////////////////////
+    this._rediscl.roq_remove (this._name, id, done);
+  }
 }
 
 
@@ -255,8 +296,10 @@ class Factory {
     this._rediscl.defineCommand('roq_reserve',  {numberOfKeys: 1, lua: _s_lua_code_reserve});
     this._rediscl.defineCommand('roq_commit',   {numberOfKeys: 1, lua: _s_lua_code_commit});
     this._rediscl.defineCommand('roq_rollback', {numberOfKeys: 1, lua: _s_lua_code_rollback});
+    this._rediscl.defineCommand('roq_remove',   {numberOfKeys: 1, lua: _s_lua_code_remove});
   }
 
+
   roq (name) {
     return new RedisOrderedQueue (name, this);
   }