keuss 1.6.15 → 1.7.0

package/test/backends_deadletter.js

@@ -80,6 +80,7 @@ function release_mq_factory (q, factory, cb) {
   {label: 'Simple MongoDB',     mq: require ('../backends/mongo')},
   {label: 'Pipelined MongoDB',  mq: require ('../backends/pl-mongo')},
   {label: 'Tape MongoDB',       mq: require ('../backends/ps-mongo')},
+  {label: 'Stream MongoDB',     mq: require ('../backends/stream-mongo')},
   {label: 'Redis OrderedQueue', mq: require ('../backends/redis-oq')},
   {label: 'MongoDB SafeBucket', mq: require ('../backends/bucket-mongo-safe')}
 ].forEach(function (MQ_item) {
@@ -159,8 +160,8 @@ function release_mq_factory (q, factory, cb) {
             cb => stats (q, cb),
             cb => stats (factory.deadletter_queue(), cb),
           ], (err, res) => {
-            res[1].tsize.should.equal (0);
-            res[2].tsize.should.equal (0);
+              res[1].tsize.should.equal (0);
+              res[2].tsize.should.equal (0);
 
             cb (err, q, factory);
           });
@@ -209,8 +210,8 @@ function release_mq_factory (q, factory, cb) {
             cb => stats (q, cb),
             cb => stats (factory.deadletter_queue(), cb),
           ], (err, res) => {
-            res[1].tsize.should.equal (0);
-            res[2].tsize.should.equal (0);
+              res[1].tsize.should.equal (0);
+              res[2].tsize.should.equal (0);
 
             cb (err, q, factory);
           });

package/test/backends_payload.js

@@ -13,7 +13,7 @@ var factory = null;
   {label: 'Simple MongoDB',         mq: require ('../backends/mongo')},
   {label: 'Pipelined MongoDB',      mq: require ('../backends/pl-mongo')},
   {label: 'Tape MongoDB',           mq: require ('../backends/ps-mongo')},
-  {label: 'Simple MongoDB Buckets', mq: require ('../backends/bucket-mongo')},
+  {label: 'Stream MongoDB',         mq: require ('../backends/stream-mongo')},
   {label: 'Safe MongoDB Buckets',   mq: require ('../backends/bucket-mongo-safe')},
   {label: 'Redis List',             mq: require ('../backends/redis-list')},
   {label: 'Redis OrderedQueue',     mq: require ('../backends/redis-oq')},
@@ -264,6 +264,7 @@ var factory = null;
   {label: 'Simple MongoDB',       mq: require ('../backends/mongo')},
   {label: 'Pipelined MongoDB',    mq: require ('../backends/pl-mongo')},
   {label: 'Tape MongoDB',         mq: require ('../backends/ps-mongo')},
+  {label: 'Stream MongoDB',       mq: require ('../backends/stream-mongo')},
   {label: 'Safe MongoDB Buckets', mq: require ('../backends/bucket-mongo-safe')},
   {label: 'Redis OrderedQueue',   mq: require ('../backends/redis-oq')},
 ].forEach(function (MQ_item) {

package/test/backends_reserve-commit-rollback.js

@@ -13,6 +13,7 @@ var factory = null;
   {label: 'Simple MongoDB',       mq: require ('../backends/mongo')},
   {label: 'Pipelined MongoDB',    mq: require ('../backends/pl-mongo')},
   {label: 'Tape MongoDB',         mq: require ('../backends/ps-mongo')},
+  {label: 'Stream MongoDB',       mq: require ('../backends/stream-mongo')},
  // {label: 'Safe MongoDB Buckets', mq: require ('../backends/bucket-mongo-safe')},
   {label: 'Redis OrderedQueue',   mq: require ('../backends/redis-oq')}
 ].forEach(function (MQ_item) {
@@ -79,7 +80,7 @@ var factory = null;
           })
         },
         cb => q.stats((err, res) => {
-          res.should.eql({
+          res.should.match({
             get: 0,
             put: 2,
             reserve: 0,
@@ -99,7 +100,7 @@ var factory = null;
           cb();
         }),
         cb => q.stats((err, res) => {
-          res.should.eql({
+          res.should.match({
             get: 1,
             put: 2,
             reserve: 0,
@@ -118,7 +119,7 @@ var factory = null;
         },
         cb => setTimeout (cb, 1000),
         cb => q.stats((err, res) => {
-          res.should.eql({
+          res.should.match({
             get: 2,
             put: 2,
             reserve: 0,
@@ -170,7 +171,7 @@ var factory = null;
         },
         function (cb) {
           q.stats(function (err, res) {
-            res.should.eql({
+            res.should.match({
               get: 0,
               put: 2,
               reserve: 0,
@@ -204,7 +205,7 @@ var factory = null;
         },
         function (cb) {
           q.stats(function (err, res) {
-            res.should.eql({
+            res.should.match({
               get: 1,
               put: 2,
               reserve: 0,
@@ -238,7 +239,7 @@ var factory = null;
         },
         function (cb) {
           q.stats(function (err, res) {
-            res.should.eql({
+            res.should.match({
               get: 2,
               put: 2,
               reserve: 0,
@@ -293,7 +294,7 @@ var factory = null;
         },
         function (cb) {
           q.stats(function (err, res) {
-            res.should.eql({
+            res.should.match({
               get: 0,
               put: 2,
               reserve: 0,
@@ -320,7 +321,7 @@ var factory = null;
         },
         function (cb) {
           q.stats(function (err, res) {
-            res.should.eql({
+            res.should.match({
               get: 0,
               put: 2,
               reserve: 0,
@@ -347,7 +348,7 @@ var factory = null;
         },
         function (cb) {
           q.stats(function (err, res) {
-            res.should.eql({
+            res.should.match({
               get: 0,
               put: 2,
               reserve: 0,
@@ -388,7 +389,7 @@ var factory = null;
         },
         function (cb) {
           q.stats(function (err, res) {
-            res.should.eql({
+            res.should.match({
               get: 2,
               put: 2,
               reserve: 0,
@@ -469,7 +470,7 @@ var factory = null;
         },
         function (cb) {
           q.stats(function (err, res) {
-            res.should.eql({
+            res.should.match({
               get: 2,
               put: 2,
               reserve: 0,
@@ -526,7 +527,7 @@ var factory = null;
         },
         function (cb) {
           q.stats(function (err, res) {
-            res.should.eql({
+            res.should.match({
               get: 0,
               put: 3,
               reserve: 0,
@@ -584,7 +585,7 @@ var factory = null;
         },
         function (cb) {
           q.stats(function (err, res) {
-            res.should.eql({
+            res.should.match({
               get: 3,
               put: 3,
               reserve: 0,
@@ -858,7 +859,7 @@ var factory = null;
         },
         cb => setTimeout (cb, 1000),
         cb => q.stats((err, res) => {
-          res.should.eql({
+          res.should.match({
             get: 0,
             put: 1,
             reserve: 1,
@@ -996,7 +997,7 @@ var factory = null;
         cb => q.ok (state.reserved_obj, cb),
         cb => setTimeout (cb, 1000),
         cb => q.stats((err, res) => {
-          res.should.eql({
+          res.should.match({
             get: 0,
             put: 1,
             reserve: 3,

package/test/pause.js

@@ -11,8 +11,8 @@ var factory = null;
   {label: 'redis OrderedQueue',  backend: require ('../backends/redis-oq')},
   {label: 'pipeline MongoDB',    backend: require ('../backends/pl-mongo')},
   {label: 'persistent MongoDB',  backend: require ('../backends/ps-mongo')},
+  {label: 'Stream MongoDB',      backend: require ('../backends/stream-mongo')},
   {label: 'plain MongoDB',       backend: require ('../backends/mongo')},
-  {label: 'MongoDB Bucket',      backend: require ('../backends/bucket-mongo')},
   {label: 'Safe MongoDB Bucket', backend: require ('../backends/bucket-mongo-safe')},
 ].forEach (backend_item => {
   [
@@ -102,7 +102,7 @@ var factory = null;
 
           ], (err, res) => {
             if (err) return done (err);
-            res[4].should.eql ({ put: 6, get: 6 });
+            res[4].should.match ({ put: 6, get: 6 });
             res[5].should.equal (0);
             q.nConsumers().should.equal (0);
 
@@ -131,7 +131,7 @@ var factory = null;
             cb => q._stats.clear (cb),
           ], (err, res) => {
             if (err) return done (err);
-            res[4].should.eql ({ put: 1, get: 1 });
+            res[4].should.match ({ put: 1, get: 1 });
             res[5].should.equal (0);
             q.nConsumers().should.equal (0);
 
@@ -165,7 +165,7 @@ var factory = null;
             cb => q._stats.clear (cb)
           ], (err, res) => {
             if (err) return done (err);
-            res[4].should.eql ({ put: 1 });
+            res[4].should.match ({ put: 1 });
             res[5].should.equal (1);
             res[2][0].timeout.should.eql (true);
             res[7].payload.should.eql ({q:0, a: 'ryetyeryre 0'});
@@ -201,7 +201,7 @@ var factory = null;
             cb => q._stats.clear (cb)
           ], (err, res) => {
             if (err) return done (err);
-            res[6].should.eql ({ put: 1 });
+            res[6].should.match ({ put: 1 });
             res[7].should.equal (1);
             res[4].timeout.should.eql (true);
             res[9].payload.should.eql ({q:0, a: 'ryetyeryre 0'});

package/test/signal.js

@@ -40,6 +40,41 @@ _.forEach ({
         setTimeout (() => factory.close(done), 500);
       });
     });
+    
+
+    it ('signals insertion ok', done => {
+      CL ({}, (err, factory) => {
+        if (err) return done(err);
+        const q = {
+          ns() {return 'the-ns'}, 
+          name () {return 'the-queue'},
+          signalInsertion (d) {
+            d.getTime().should.equal (1234567890);
+            factory.close(done);
+          }
+        };
+
+        const signal = factory.signal (q, {});
+        setTimeout (() => signal.emitInsertion (new Date(1234567890)), 500);
+      });
+    });
+
+    it ('signals pause ok', done => {
+      CL ({}, (err, factory) => {
+        if (err) return done(err);
+        const q = {
+          ns() {return 'the-ns'}, 
+          name () {return 'the-queue'},
+          signalPaused(d) {
+            d.should.equal (true);
+            factory.close(done);
+          }
+        };
+
+        const signal = factory.signal (q, {});
+        setTimeout (() => signal.emitPaused (true), 500);
+      });
+    });
 
 
     describe (`extra/generic pubsub`, () => {

package/test/stats.js

@@ -39,9 +39,14 @@ _.forEach ({
       CL ((err, ftry) => {
         if (err) return done(err);
         var mem = ftry.stats (ns, name);
-        mem.values (function (err, vals) {
-          vals.should.eql ({});
-          ftry.close(done);
+
+        async.series([
+          cb => mem.clear (cb),
+        ], () => {
+          mem.values (function (err, vals) {
+            vals.should.eql ({});
+            ftry.close(done);
+          });
         });
       });
     });
@@ -52,6 +57,7 @@ _.forEach ({
         var mem = ftry.stats (ns, name);
 
         async.series([
+          cb => mem.clear (cb),
           cb => mem.incr ('v1', 1, cb),
           cb => mem.incr ('v2', 1, cb),
           cb => mem.incr ('v3', 1, cb)
@@ -251,5 +257,40 @@ _.forEach ({
         });
       });
     });
+
+
+    it ('managed hierarchy (dotted keys) ok', done => {
+      CL ((err, ftry) => {
+        if (err) return done(err);
+        var mem = ftry.stats (ns, name);
+
+        async.series([
+          cb => mem.clear (cb),
+          cb => mem.incr ('l1.l2.c', -1, cb),
+          cb => mem.incr ('l1.l2.b', 1, cb),
+          cb => mem.incr ('l1.l3.a', 7, cb),
+          cb => mem.incr ('v1', 3, cb),
+        ], (err, results) => {
+          setTimeout ( () => mem.values ((err, vals) => {
+            vals.should.eql ({
+              l1: {
+                l2: {
+                  b: 1,
+                  c: -1
+                },
+                l3: {
+                  a: 7
+                }
+              },
+              v1: 3
+            });
+            
+            ftry.close(done);
+          }), 200);
+        });
+      });
+    });
+
+
   });
 });

package/test/stream-mongo.js

@@ -0,0 +1,166 @@
+var async =  require ('async');
+var should = require ('should');
+var _ =      require ('lodash');
+
+var LocalSignal = require ('../signal/local');
+var MemStats =    require ('../stats/mem');
+
+const MongoClient = require ('mongodb').MongoClient;
+
+
+function stats (q, cb) {
+  async.series ({
+    stats: cb => q.stats(cb),
+    tsize: cb => q.totalSize(cb)
+  }, (err, res) => {
+//    console.log ('stats:', res);
+    cb (err, res);
+  });
+}
+
+function get_mq_factory (MQ, opts, cb) {
+  const common_opts = {
+    url: 'mongodb://localhost/keuss_test_backends_stream_mongo',
+    signaller: { provider: LocalSignal},
+    stats: {provider: MemStats},
+  };
+
+  // initialize factory
+  MQ (_.merge ({}, common_opts, opts), cb);
+}
+
+function release_mq_factory (factory, cb) {
+//  console.log ('releasing mq factory');
+
+  setTimeout (() => {
+    factory.close (cb);
+  }, 1000);
+}
+
+
+/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+[
+  {label: 'Stream MongoDB',  mq: require ('../backends/stream-mongo')},
+].forEach (MQ_item => {
+  describe('stream operations with ' + MQ_item.label + ' queue backend', () => {
+    const MQ = MQ_item.mq;
+
+    beforeEach (done => {
+      done();
+    });
+
+    afterEach (done => async.series ([
+      cb => setTimeout (cb, 1000),
+      cb => MongoClient.connect ('mongodb://localhost/keuss_test_backends_stream_mongo', (err, cl) => {
+        if (err) return done (err);
+        cl.db().dropDatabase (() => cl.close (cb))
+      })
+    ], done));
+
+
+    it('pushes one message, reads it on 3 clients', done => {
+      const pl = {elem: 1, pl: 'twetrwte'};
+      const hdrs = {aaa: 'qw', bbb: '666'};
+
+      async.waterfall ([
+        cb => get_mq_factory (MQ, {}, cb),
+        (factory, cb) => {
+          const q0 = factory.queue('test_queue_1', {groups: '000, 001, 002, 003'});
+          const q1 = factory.queue('test_queue_1', {group: '000'});
+          const q2 = factory.queue('test_queue_1', {group: '001'});
+          const q3 = factory.queue('test_queue_1', {group: '002'});
+
+          async.parallel ([
+            cb => setTimeout (() => q0.push (pl, {hdrs}, cb), 1111),
+            cb => q1.pop ('c1', cb),
+            cb => q2.pop ('c2', cb),
+            cb => q3.pop ('c3', cb),
+          ], (err, res) => {
+            if (err) return done (err);
+            res[1].should.eql (res[2]);
+            res[2].should.eql (res[3]);
+            res[1].payload.should.eql (pl);
+            res[1].hdrs.should.eql (hdrs);
+            cb (err, factory);
+          });
+        },
+        (factory, cb) => release_mq_factory (factory, cb)
+      ], done);
+    });
+
+
+    it('pushes one message, unaware cients do not get it', done => {
+      const pl = {elem: 1, pl: 'twetrwte'};
+      const hdrs = {aaa: 'qw', bbb: '666'};
+
+      async.waterfall ([
+        cb => get_mq_factory (MQ, {}, cb),
+        (factory, cb) => {
+          const q0 = factory.queue('test_queue_1', {groups: '000, 001, 002, 003'});
+          const q1 = factory.queue('test_queue_1', {group: '000'});
+          const q2 = factory.queue('test_queue_1', {group: '001'});
+          const q3 = factory.queue('test_queue_1', {group: '004'});
+          const q4 = factory.queue('test_queue_1', {group: '005'});
+
+          async.parallel ([
+            cb => setTimeout (() => q0.push (pl, {hdrs}, cb), 1111),
+            cb => q1.pop ('c1', cb),
+            cb => q2.pop ('c2', cb),
+            cb => q3.pop ('c3', {timeout: 2000}, err => cb (null, err)),
+            cb => q4.pop ('c4', {timeout: 2000}, err => cb (null, err)),
+          ], (err, res) => {
+            if (err) return done (err);
+            res[1].should.eql (res[2]);
+            res[1].payload.should.eql (pl);
+            res[1].hdrs.should.eql (hdrs);
+            res[3].timeout.should.be.true;
+            res[4].timeout.should.be.true;
+            cb (err, factory);
+          });
+        },
+        (factory, cb) => release_mq_factory (factory, cb)
+      ], done);
+    });
+
+    it('pushes one message, only on of several clients on same group get it', done => {
+      const pl = {elem: 1, pl: 'twetrwte'};
+      const hdrs = {aaa: 'qw', bbb: '666'};
+
+      async.waterfall ([
+        cb => get_mq_factory (MQ, {}, cb),
+        (factory, cb) => {
+          const q0 = factory.queue('test_queue_1', {groups: '000, 001, 002, 003'});
+          const q1 = factory.queue('test_queue_1', {group: '000'});
+          const q2 = factory.queue('test_queue_1', {group: '000'});
+          const q3 = factory.queue('test_queue_1', {group: '001'});
+          const q4 = factory.queue('test_queue_1', {group: '001'});
+
+          async.parallel ([
+            cb => setTimeout (() => q0.push (pl, {hdrs}, cb), 1111),
+            cb => q1.pop ('c1', {timeout: 2000}, (err, res) => cb (null, res || err)),
+            cb => q2.pop ('c2', {timeout: 2000}, (err, res) => cb (null, res || err)),
+            cb => q3.pop ('c3', {timeout: 2000}, (err, res) => cb (null, res || err)),
+            cb => q4.pop ('c4', {timeout: 2000}, (err, res) => cb (null, res || err)),
+          ], (err, res) => {
+            if (err) return done (err);
+
+            function __calc__ (ra) {
+              const res = {ok:0, ko:0};
+              ra.forEach (r => {
+                if (r.timeout === true) res.ko++;
+                else if (r.payload.elem == 1) res.ok++;
+              });
+              return res;
+            }
+
+            __calc__ ([res[1], res[2]]).should.eql ({ok:1, ko:1});
+            __calc__ ([res[3], res[4]]).should.eql ({ok:1, ko:1});
+            cb (err, factory);
+          });
+        },
+        (factory, cb) => release_mq_factory (factory, cb)
+      ], done);
+    });
+
+  });
+});