keuss 1.7.3 → 2.0.0

package/test/backends_bucket-at-least-once.js

@@ -1,302 +0,0 @@
-
-var async =   require ('async');
-var should =  require ('should');
-
-var LocalSignal = require ('../signal/local');
-var MemStats =    require ('../stats/mem');
-
-const MongoClient = require ('mongodb').MongoClient;
-
-var factory = null;
-
-[
-  {label: 'MongoDB SafeBucket', mq: require ('../backends/bucket-mongo-safe')}
-].forEach (function (MQ_item) {
-
-describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', function () {
-  var MQ = MQ_item.mq;
-
-  before (function (done) {
-    var opts = {
-      url: 'mongodb://localhost/keuss_test_bucket_at_least_once',
-      signaller: { provider: LocalSignal},
-      stats: {provider: MemStats}
-    };
-
-    MQ (opts, function (err, fct) {
-      if (err) return done (err);
-      factory = fct;
-      done();
-    });
-  });
-
-
-  after (done => async.series ([
-    cb => setTimeout (cb, 1000),
-    cb => factory.close (cb),
-    cb => MongoClient.connect ('mongodb://localhost/keuss_test_bucket_at_least_once', (err, cl) => {
-      if (err) return done (err);
-      cl.db().dropDatabase (() => cl.close (cb))
-    })
-  ], done));
-
-
-  it ('queue is created empty and ok', function (done){
-    var q = factory.queue('test_queue_1');
-    should.equal (q.nextMatureDate (), null);
-    q.name ().should.equal ('test_queue_1');
-
-    async.series([
-      function (cb) {q.stats(cb)},
-      function (cb) {q.size (cb)},
-      function (cb) {q.totalSize (cb)},
-      function (cb) {q.next_t (cb)},
-    ], function(err, results) {
-      if (err) return done (err);
-      results.should.eql ([{get: 0, put: 0, reserve: 0, commit: 0, rollback: 0, deadletter: 0}, 0, 0, null])
-      done();
-    });
-  });
-
-
-  it ('sequential push & pops, go as expected', function (done){
-    var q = factory.queue('test_queue_2');
-
-    async.series([
-      function (cb) {q.push ({elem:1, pl:'twetrwte'}, cb)},
-      function (cb) {q.push ({elem:2, pl:'twetrwte'}, cb)},
-      function (cb) {setTimeout (cb, 1111)},
-      function (cb) {q.size (function (err, size) {
-        size.should.equal (2);
-        cb(err);
-      })},
-      function (cb) {q.stats (function (err, res) {
-        res.should.eql ({get: 0, put: 2, reserve: 0, commit: 0, rollback: 0, deadletter: 0});
-        cb(err);
-      })},
-      function (cb) {q.next_t (function (err, res) {
-        res.getTime().should.be.approximately(new Date().getTime(), 2000);
-        cb(err);
-      })},
-      function (cb) {q.pop ('c1', function (err, ret) {
-        ret.payload.should.eql ({elem:1, pl:'twetrwte'});
-        cb (err);
-      })},
-      function (cb) {q.size (function (err, size) {
-        size.should.equal (0);
-        cb(err);
-      })},
-      function (cb) {q.stats (function (err, res) {
-        res.should.eql ({get: 1, put: 2, reserve: 0, commit: 0, rollback: 0, deadletter: 0});
-        cb(err);
-      })},
-      function (cb) {q.pop ('c2', function (err, ret) {
-        ret.payload.should.eql ({elem:2, pl:'twetrwte'});
-        cb (err);
-      })},
-      function (cb) {q.size (function (err, size) {
-        size.should.equal (0);
-        cb(err);
-      })},
-      function (cb) {q.stats (function (err, res) {
-        res.should.eql ({get: 2, put: 2, reserve: 0, commit: 0, rollback: 0, deadletter: 0});
-        cb(err);
-      })},
-    ], function(err, results) {
-      if (err) return done (err);
-      q.drain();
-      done();
-    });
-  });
-
-
-  it ('pop cancellation works as expected', function (done){
-    var q = factory.queue('test_queue_3');
-
-    async.series([
-      function (cb) {
-        var tid1 = q.pop ('c1', {timeout: 2000}, function (err, ret) {err.should.equal('cancel')});
-        q.consumers().length.should.equal (1);
-        var tid2 = q.pop ('c2', {timeout: 2000}, function (err, ret) {err.should.equal('cancel')});
-        q.nConsumers().should.equal (2);
-        q.cancel (tid1);
-        q.nConsumers().should.equal (1);
-        q.cancel (tid2);
-        q.nConsumers().should.equal (0);
-        cb();
-      },
-      function (cb) {q.push ({elem:2, pl:'cancel-twetrwte'}, cb)},
-      function (cb) {q.push ({elem:1, pl:'cancel-twetrwte'}, cb)},
-      function (cb) {setTimeout (cb, 1111)},
-      function (cb) {q.pop ('c3', {timeout: 15000}, function (err, ret) {
-        ret.payload.should.eql ({elem:2, pl:'cancel-twetrwte'});
-        cb (err, ret);
-      })},
-      function (cb) {q.pop ('c4', {timeout: 15000}, function (err, ret) {
-        ret.payload.should.eql ({elem:1, pl:'cancel-twetrwte'});
-        cb (err, ret);
-      })},
-      function (cb) {q.size (function (err, size) {
-        size.should.equal (0);
-        cb(err);
-      })},
-      function (cb) {q.stats (function (err, res) {
-        res.should.eql ({get: 2, put: 2, reserve: 0, commit: 0, rollback: 0, deadletter: 0});
-        cb(err);
-      })},
-    ], function(err, results) {
-      if (err) return done (err);
-      q.drain();
-      done();
-    });
-  });
-
-
-  it ('push & pop on 2 buckets ok', function (done) {
-    var q = factory.queue('test_queue_4');
-
-    async.series([
-      (cb) => q.push ({elem:1, pl:'twetrwte'}, cb),
-      (cb) => q.push ({elem:2, pl:'twetrwte'}, cb),
-      (cb) => q.push ({elem:3, pl:'twetrwte'}, cb),
-      (cb) => q.push ({elem:4, pl:'twetrwte'}, cb),
-      (cb) => setTimeout (cb, 666),
-
-      (cb) => q.size ((err, size) => {size.should.equal (4); cb(err); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 4, reserve: 0, commit: 0, rollback: 0, deadletter: 0}); cb(err); }),
-
-      (cb) => q.push ({elem:5, pl:'twetrwte'}, cb),
-      (cb) => q.push ({elem:6, pl:'twetrwte'}, cb),
-      (cb) => q.push ({elem:7, pl:'twetrwte'}, cb),
-      (cb) => setTimeout (cb, 666),
-
-      (cb) => q.size ((err, size) => {size.should.equal (7); cb(err); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 7, reserve: 0, commit: 0, rollback: 0, deadletter: 0}); cb(err); }),
-
-      (cb) => q.next_t ((err, res) => {res.getTime().should.be.approximately(new Date().getTime(), 2000); cb(err); }),
-
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:1, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:2, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:3, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:4, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:5, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:6, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:7, pl:'twetrwte'}); cb (err); })},
-
-      (cb) => q.size ((err, size) => {size.should.equal (0); cb(err); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 7, put: 7, reserve: 0, commit: 0, rollback: 0, deadletter: 0}); cb(err); }),
-    ], function(err, results) {
-      if (err) return done (err);
-      q.drain();
-      done();
-    });
-  });
-
-
-  it ('push & pop with delay ok', function (done) {
-    var q = factory.queue('test_queue_5');
-
-    async.series([
-      (cb) => q.push ({elem:1, pl:'twetrwte'}, {delay: 4}, cb),
-      (cb) => q.push ({elem:2, pl:'twetrwte'}, {delay: 2}, cb),
-      (cb) => q.push ({elem:3, pl:'twetrwte'}, {delay: 3}, cb),
-      (cb) => q.push ({elem:4, pl:'twetrwte'}, {delay: 4}, cb),
-      (cb) => setTimeout (cb, 700),
-
-      (cb) => q.next_t ((err, res) => {res.getTime().should.be.approximately(new Date().getTime() + 3300, 100); cb(err); }),
-
-      (cb) => q.push ({elem:5, pl:'twetrwte'}, {delay: 2}, cb),
-      (cb) => q.push ({elem:6, pl:'twetrwte'}, {delay: 1}, cb),
-      (cb) => q.push ({elem:7, pl:'twetrwte'}, {delay: 2}, cb),
-      (cb) => setTimeout (cb, 700),
-
-      (cb) => q.next_t ((err, res) => {res.getTime().should.be.approximately(new Date().getTime() + 1300, 100); cb(err); }),
-
-      (cb) => q.size ((err, size) => {size.should.equal (0); cb(err); }),
-      (cb) => q.totalSize ((err, size) => {size.should.equal (7); cb(err); }),
-      (cb) => q.schedSize ((err, size) => {size.should.equal (7); cb(err); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 7, reserve: 0, commit: 0, rollback: 0, deadletter: 0}); cb(err); }),
-
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:5, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:6, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:7, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:1, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:2, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:3, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:4, pl:'twetrwte'}); cb (err); })},
-
-      (cb) => q.size ((err, size) => {size.should.equal (0); cb(err); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 7, put: 7, reserve: 0, commit: 0, rollback: 0, deadletter: 0}); cb(err); }),
-    ], function(err, results) {
-      if (err) return done (err);
-      q.drain();
-      done();
-    });
-  });
-
-
-  it ('insert & rejections over various buckets go as expected', function (done) {
-    var q = factory.queue('test_queue_6');
-
-    async.series([
-      (cb) => q.push ({elem:1, pl:'twetrwte'}, cb),
-      (cb) => q.push ({elem:2, pl:'twetrwte'}, cb),
-      (cb) => setTimeout (cb, 700),
-
-      (cb) => q.size ((err, size) => {size.should.equal (2); cb(err); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 2, reserve: 0, commit: 0, rollback: 0, deadletter: 0}); cb(err); }),
-
-      (cb) => q.push ({elem:3, pl:'twetrwte'}, cb),
-      (cb) => setTimeout (cb, 700),
-
-      (cb) => q.size ((err, size) => {size.should.equal (3); cb(err); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 3, reserve: 0, commit: 0, rollback: 0, deadletter: 0}); cb(err); }),
-
-      (cb) => q.next_t ((err, res) => {res.getTime().should.be.approximately(new Date().getTime() - 1400, 100); cb(err); }),
-
-      (cb) => {
-        q.pop ('c1', {reserve: true}, (err, ret) => {
-          if (err) return cb (err);
-          ret.payload.should.eql ({elem:1, pl:'twetrwte'});
-          q.ko (ret._id, (new Date().getTime() + 5000 ), (err, res) => {
-            should (res).equal (true);
-            cb (err);
-          });
-        });
-      },
-
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:2, pl:'twetrwte'}); cb (err); })},
-
-      (cb) => {
-        q.pop ('c1', {reserve: true}, (err, ret) => {
-          if (err) return cb (err);
-          ret.payload.should.eql ({elem:3, pl:'twetrwte'});
-          q.ko (ret._id, (new Date().getTime() + 3000 ), (err, res) => {
-            should (res).equal (true);
-            cb (err);
-          });
-        });
-      },
-
-      (cb) => setTimeout (cb, 700),
-
-      (cb) => q.next_t ((err, res) => {res.getTime().should.be.approximately(new Date().getTime() + 2500, 500); cb(err); }),
-
-      (cb) => q.size ((err, size) => {size.should.equal (0); cb(err); }),
-      (cb) => q.totalSize ((err, size) => {size.should.equal (2); cb(err); }),
-      (cb) => q.schedSize ((err, size) => {size.should.equal (2); cb(err); }),
-
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:3, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:1, pl:'twetrwte'}); cb (err); })},
-
-      (cb) => q.size ((err, size) => {size.should.equal (0); cb(err); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 3, put: 3, reserve: 2, commit: 0, rollback: 2, deadletter: 0}); cb(err); }),
-    ], function(err, results) {
-      if (err) return done (err);
-      q.drain();
-      done();
-    });
-  });
-});
-
-});

package/test/backends_deadletter.js

@@ -1,227 +0,0 @@
-var async =  require ('async');
-var should = require ('should');
-var _ =      require ('lodash');
-
-var LocalSignal = require ('../signal/local');
-var MemStats =    require ('../stats/mem');
-
-const MongoClient = require ('mongodb').MongoClient;
-
-
-function stats (q, cb) {
-  async.series ({
-    stats: cb => q.stats(cb),
-    tsize: cb => q.totalSize(cb)
-  }, (err, res) => {
-//    console.log ('stats:', res);
-    cb (err, res);
-  });
-}
-
-function pop (q, stage, cb) {
-  q.pop('c1', { reserve: true }, (err, res) => {
-    stage.obj = res;
-//    console.log('reserved element %o', res);
-    cb(err);
-  });
-}
-
-function reject (q, stage, cb) {
-  var next_t = new Date().getTime() + 2000;
-
-  q.ko (stage.obj, next_t, (err, res) => {
-    if (err) {
-//      console.error ('error in rollback of %s: %j', stage.obj._id, err);
-      return cb (err);
-    }
-
-//    console.log('rolled back element %s: %o', stage.obj._id, res);
-    cb();
-  });
-}
-
-function accept (q, stage, cb) {
-  q.ok (stage.obj, (err, res) => {
-    if (err) {
-//      console.error ('error in rollback of %s: %j', stage.obj._id, err);
-      return cb (err);
-    }
-
-//    console.log('commited element %s: %j', stage.obj._id, res);
-    cb();
-  });
-}
-
-function get_mq_factory (MQ, opts, cb) {
-  const common_opts = {
-    url: 'mongodb://localhost/keuss_test_backends_deadletter',
-    signaller: { provider: LocalSignal},
-    stats: {provider: MemStats},
-    deadletter: {
-    }
-  };
-
-  // initialize factory
-  MQ (_.merge ({}, common_opts, opts), cb);
-}
-
-function release_mq_factory (q, factory, cb) {
-//  console.log ('releasing mq factory');
-
-  setTimeout (() => {
-    q.cancel ();
-    factory.close (cb);
-  }, 1000);
-}
-
-
-/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-[
-//  {label: 'Simple MongoDB',     mq: require ('../backends/mongo')},
-//  {label: 'Pipelined MongoDB',  mq: require ('../backends/pl-mongo')},
-//  {label: 'Tape MongoDB',       mq: require ('../backends/ps-mongo')},
-//  {label: 'Stream MongoDB',     mq: require ('../backends/stream-mongo')},
-//  {label: 'Redis OrderedQueue', mq: require ('../backends/redis-oq')},
-//  {label: 'MongoDB SafeBucket', mq: require ('../backends/bucket-mongo-safe')},
-  {label: 'Mongo IntraOrder',   mq: require ('../backends/intraorder')},
-].forEach(function (MQ_item) {
-  describe('rollback and deadletters with ' + MQ_item.label + ' queue backend', function () {
-    const MQ = MQ_item.mq;
-
-    beforeEach (done => {
-      done();
-    });
-
-    afterEach (done => async.series ([
-      cb => setTimeout (cb, 1000),
-      cb => MongoClient.connect ('mongodb://localhost/keuss_test_backends_deadletter', (err, cl) => {
-        if (err) return done (err);
-        cl.db().dropDatabase (() => cl.close (cb))
-      })
-    ], done));
-
-
-    it('repeated ko causes item to be moved to deadletter if deadletter is set', done => {
-      var factory_opts = {
-        deadletter: {
-          max_ko: 3
-        }
-      };
-
-      const pl = {
-        elem: 1,
-        pl: 'twetrwte'
-      };
-
-      const hdrs = {aaa: 'qw', bbb: '666'};
-
-      async.waterfall ([
-        cb => get_mq_factory (MQ, factory_opts, cb),
-        (factory, cb) => {
-          const q = factory.queue('test_queue_deadletter', {});
-          const stage = {};
-          let tries= 0;
-
-          async.race ([
-            cb => async.series([
-              cb => q.push (pl, {hdrs}, cb),
-              cb => pop (q, stage, cb),
-              cb => reject (q, stage, (err) => {tries++;cb()}),
-              cb => pop (q, stage, cb),
-              cb => reject (q, stage,  (err) => {tries++;cb()}),
-              cb => pop (q, stage, cb),
-              cb => reject (q, stage,  (err) => {tries++;cb()}),
-              cb => pop (q, stage, cb),
-              cb => reject (q, stage,  (err) => {tries++;cb()}),
-              cb => pop (q, stage, cb),
-              cb => reject (q, stage,  (err) => {tries++;cb()}),
-              cb => pop (q, stage, cb),
-              cb => reject (q, stage,  (err) => {tries++;cb()}),
-            ], err => {
-              err.should.equal ('cancel');
-              cb ();
-            }),
-            cb => factory.deadletter_queue().pop('c2', (err, res) => {
-              res.payload.should.eql (pl);
-              res.hdrs.should.match ({
-                aaa: "qw",
-                bbb: "666",
-                'x-dl-from-queue': "test_queue_deadletter",
-                'x-dl-t': /.+/,
-                'x-dl-tries': 4
-              });
-              tries.should.equal (5);
-              cb (err);
-            })
-          ], err => cb (err, q, factory));
-        },
-        (q, factory, cb) => {
-          async.series ([
-            cb => setTimeout (cb, 1000),
-            cb => stats (q, cb),
-            cb => stats (factory.deadletter_queue(), cb),
-          ], (err, res) => {
-              res[1].tsize.should.equal (0);
-              res[2].tsize.should.equal (0);
-
-            cb (err, q, factory);
-          });
-        },
-        (q, factory, cb) => release_mq_factory (q, factory, cb)
-      ], done);
-    });
-
-
-    it('repeated ko causes item NOT to be moved to deadletter if deadletter is not set', done => {
-      var factory_opts = {
-      };
-
-      const pl = {
-        elem: 1,
-        pl: 'twetrwte'
-      };
-
-      async.waterfall ([
-        cb => get_mq_factory (MQ, factory_opts, cb),
-        (factory, cb) => {
-          const q = factory.queue('test_queue_deadletter', {});
-          const stage = {};
-
-          async.series([
-            cb => q.push (pl, cb),
-            cb => pop (q, stage, cb),
-            cb => reject (q, stage, cb),
-            cb => pop (q, stage, cb),
-            cb => reject (q, stage, cb),
-            cb => pop (q, stage, cb),
-            cb => reject (q, stage, cb),
-            cb => pop (q, stage, cb),
-            cb => reject (q, stage, cb),
-            cb => pop (q, stage, cb),
-            cb => reject (q, stage, cb),
-            cb => pop (q, stage, cb),
-            cb => accept (q, stage, cb)
-          ], err => {
-            cb (err, q, factory)
-          });
-        },
-        (q, factory, cb) => {
-          async.series ([
-            cb => setTimeout (cb, 1000),
-            cb => stats (q, cb),
-            cb => stats (factory.deadletter_queue(), cb),
-          ], (err, res) => {
-              res[1].tsize.should.equal (0);
-              res[2].tsize.should.equal (0);
-
-            cb (err, q, factory);
-          });
-        },
-        (q, factory, cb) => release_mq_factory (q, factory, cb)
-      ], done);
-    });
-
-
-
-  });
-});