keuss 1.7.4 → 2.0.1

package/playground/pl1.js

@@ -1,74 +0,0 @@
-// mongodb: create a consumer and a producer
-var MQ = require ('../backends/pl-mongo');
-var PLL = require ('../Pipeline/DirectLink');
-var async = require ('async');
-
-
-var factory_opts = {
-  url: 'mongodb://localhost/qeus'
-};
-
-// initialize factory
-MQ (factory_opts, function (err, factory) {
-  if (err) {
-    return console.error (err);
-  }
-
-  // factory ready, create 3 queues on default pipeline
-  var q_opts = {};
-  var q1 = factory.queue ('test_pl_1', q_opts);
-  var q2 = factory.queue ('test_pl_2', q_opts);
-  var q3 = factory.queue ('test_pl_3', q_opts);
-
-  // tie them up, q1 -> q2 -> q3 -> q1
-  var pll1 = new PLL (q1, q2, {delay: 1});
-  var pll2 = new PLL (q2, q3);
-  var pll3 = new PLL (q3, q1);
-
-  pll1.start (function (elem, done) {
-    var pl = elem.payload;
-
-    if (!pl.processed_1) {
-      pl.processed_1=1
-    }
-    else {
-      pl.processed_1++
-    }
-
-    console.log ('%s: tick', pll1.name ());
-    done();
-  });
-
-  pll2.start (function (elem, done) {
-    var pl = elem.payload;
-
-    if (!pl.processed_2) {
-      pl.processed_2=1
-    }
-    else {
-      pl.processed_2++
-    }
-
-    console.log ('%s: tick', pll2.name ());
-    done();
-  });
-
-  pll3.start (function (elem, done) {
-    var pl = elem.payload;
-
-    if (!pl.processed_3) {
-      pl.processed_3=1
-    }
-    else {
-      pl.processed_3++
-    }
-
-    console.log ('%s: tick', pll3.name ());
-    done ();
-  });
-
-  // insert elements
-  async.timesLimit (111, 3, function (n, next) {
-    q1.push ({a:n, b:'see it spin...'}, {}, next);
-  });
-});

package/playground/q1.js

@@ -1,34 +0,0 @@
-// mongodb: create a consumer and a producer
-var MQ = require ('../backends/pl-mongo');
-
-var factory_opts = {
-  url: 'mongodb://localhost/qeus'
-};
-    
-// initialize factory 
-MQ (factory_opts, function (err, factory) {
-  if (err) {
-    return console.error (err);
-  }
-
-  // factory ready, create one queue
-  var q_opts = {};
-  var q = factory.queue ('test_queue', q_opts);
-
-  // insert element
-  q.push ({a:1, b:'666'}, {delay: 3}, function (err, res) {
-    if (err) {
-      return console.error (err);
-    }
-
-    // element inserted at this point. pop it again
-    var pop_opts = {};
-    q.pop ('consumer-one', pop_opts, function (err, res) {
-      if (err) {
-        return console.error (err);
-      }
-
-      console.log ('got this: ', res);
-    });
-  });
-});

package/playground/simple-pl.js

@@ -1,42 +0,0 @@
-var MQ = require ('../backends/pl-mongo');
-var PLL = require ('../Pipeline/DirectLink');
-var async = require ('async');
-
-
-var factory_opts = {
-  url: 'mongodb://localhost/qeus'
-};
-
-// initialize factory
-MQ (factory_opts, function (err, factory) {
-  if (err) {
-    return console.error (err);
-  }
-
-  // factory ready, create 2 queues on default pipeline
-  var q_opts = {};
-  var q1 = factory.queue ('test_pl_1', q_opts);
-  var q2 = factory.queue ('test_pl_2', q_opts);
-
-  // tie them up, q1 -> q2
-  var pll = new PLL (q1, q2);
-
-  pll.start (function (elem, done) {
-    var pl = elem.payload;
-    pl.pll_processed = true;
-    done();
-  });
-
-  // insert elements in the entry queue
-  async.timesLimit (111, 3, function (n, next) {
-    q1.push ({a:n, b:'see it spin...'}, {}, next);
-  });
-
-  async.timesLimit (111, 3, function (n, next) {
-    q2.pop ('exit', {}, function (err, res) {
-      console.log ('end point get', res);
-      next ();
-    });
-  });
-});
-

package/playground/stream-loops.js

@@ -1,114 +0,0 @@
-/*
- *
- * very simple example of stream-mongo: one element pushed, consumed three times
- *
- */
-
-const async = require ('async');
-const _ =     require ('lodash');
-const MQ =    require ('../backends/stream-mongo');
-
-
-const group_cardinality = 3;
-const mesgs = 1000000;
-
-const stats = {
-  consumer: {
-    last: 0,
-    total: 0
-  },
-  producer: {
-    last: 0,
-    total: 0
-  }
-}
-
-function do_times (cnt, fn, done) {
-  if (!cnt) {
-    console.log ('loop done')
-    return done(null, 0);
-  }
-
-  fn (cnt, err => {
-    if (err) return done (err, cnt);
-    do_times (cnt-1, fn, done);
-  })
-}
-
-
-function payload (n) {
-  return {elem: n, headline: 'something something', tags: {a: n, b: 2*n}}
-}
-
-
-function headers (n) {
-  return {cnt: n, h1: 'something something', h2: false}
-}
-
-
-function producer_looper (q, id) {
-  console.log (`creating producer looper ${id}`);
-  return (n, cb) => {
-    q.push (payload (n), {hdrs: headers(n)}, err => {
-//    console.log ('[%s] push #%d', id, n);
-      stats.producer.total++;
-      cb (err);
-    });
-  }
-}
-
-
-function consumer_looper (q, id) {
-  console.log (`creating consumer looper ${id}`);
-  return (n, cb) => {
-    q.pop (id, err => {
-//    console.log ('[%s] pop #%d', id, n);
-      stats.consumer.total++
-      cb (err);
-    });
-  }
-}
-
-
-// initialize factory
-MQ ({
-  url: 'mongodb://localhost/keuss_test_stream'
-}, (err, factory) => {
-  if (err) return console.error(err);
-
-  const groups = _.range (1, group_cardinality + 1).map (i => `G${i}`).join (',');
-  console.log('groups: ', groups);
-
-  // create queues and clients
-  const queues = {};
-  queues['qp'] =  factory.queue ('test_stream', {groups});
-
-  for (let i = 1; i <= group_cardinality; i++) {
-    queues[`qc${i}`] = factory.queue ('test_stream', {group: `G${i}`});
-  }
-
-  console.log (`created queues (one producer, ${group_cardinality} consumers)`);
-
-  // create tasks
-  const tasks = [];
-  tasks.push (cb => do_times (mesgs, producer_looper (queues['qp'], 'p0'),  cb));
-
-  for (let t = 1; t <= group_cardinality; t++) {
-    tasks.push (cb => do_times (mesgs, consumer_looper (queues[`qc${t}`], `c${t}`), cb),)
-  }
-
-  console.log (`created tasks (one producer, ${group_cardinality} consumers)`);
-
-  async.parallel (tasks, err => {
-    factory.close ();
-    if (err) console.error (err);
-    console.log ('done');
-  });
-
-  setInterval (() => {
-    console.log (`produced ${stats.producer.total - stats.producer.last} msg/s, consumed ${stats.consumer.total - stats.consumer.last} msg/s`);
-    stats.producer.last = stats.producer.total;
-    stats.consumer.last = stats.consumer.total
-  }, 1000);
-});
-

package/test/backends_bucket-at-least-once.js

@@ -1,302 +0,0 @@
-
-var async =   require ('async');
-var should =  require ('should');
-
-var LocalSignal = require ('../signal/local');
-var MemStats =    require ('../stats/mem');
-
-const MongoClient = require ('mongodb').MongoClient;
-
-var factory = null;
-
-[
-  {label: 'MongoDB SafeBucket', mq: require ('../backends/bucket-mongo-safe')}
-].forEach (function (MQ_item) {
-
-describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', function () {
-  var MQ = MQ_item.mq;
-
-  before (function (done) {
-    var opts = {
-      url: 'mongodb://localhost/keuss_test_bucket_at_least_once',
-      signaller: { provider: LocalSignal},
-      stats: {provider: MemStats}
-    };
-
-    MQ (opts, function (err, fct) {
-      if (err) return done (err);
-      factory = fct;
-      done();
-    });
-  });
-
-
-  after (done => async.series ([
-    cb => setTimeout (cb, 1000),
-    cb => factory.close (cb),
-    cb => MongoClient.connect ('mongodb://localhost/keuss_test_bucket_at_least_once', (err, cl) => {
-      if (err) return done (err);
-      cl.db().dropDatabase (() => cl.close (cb))
-    })
-  ], done));
-
-
-  it ('queue is created empty and ok', function (done){
-    var q = factory.queue('test_queue_1');
-    should.equal (q.nextMatureDate (), null);
-    q.name ().should.equal ('test_queue_1');
-
-    async.series([
-      function (cb) {q.stats(cb)},
-      function (cb) {q.size (cb)},
-      function (cb) {q.totalSize (cb)},
-      function (cb) {q.next_t (cb)},
-    ], function(err, results) {
-      if (err) return done (err);
-      results.should.eql ([{get: 0, put: 0, reserve: 0, commit: 0, rollback: 0, deadletter: 0}, 0, 0, null])
-      done();
-    });
-  });
-
-
-  it ('sequential push & pops, go as expected', function (done){
-    var q = factory.queue('test_queue_2');
-
-    async.series([
-      function (cb) {q.push ({elem:1, pl:'twetrwte'}, cb)},
-      function (cb) {q.push ({elem:2, pl:'twetrwte'}, cb)},
-      function (cb) {setTimeout (cb, 1111)},
-      function (cb) {q.size (function (err, size) {
-        size.should.equal (2);
-        cb(err);
-      })},
-      function (cb) {q.stats (function (err, res) {
-        res.should.eql ({get: 0, put: 2, reserve: 0, commit: 0, rollback: 0, deadletter: 0});
-        cb(err);
-      })},
-      function (cb) {q.next_t (function (err, res) {
-        res.getTime().should.be.approximately(new Date().getTime(), 2000);
-        cb(err);
-      })},
-      function (cb) {q.pop ('c1', function (err, ret) {
-        ret.payload.should.eql ({elem:1, pl:'twetrwte'});
-        cb (err);
-      })},
-      function (cb) {q.size (function (err, size) {
-        size.should.equal (0);
-        cb(err);
-      })},
-      function (cb) {q.stats (function (err, res) {
-        res.should.eql ({get: 1, put: 2, reserve: 0, commit: 0, rollback: 0, deadletter: 0});
-        cb(err);
-      })},
-      function (cb) {q.pop ('c2', function (err, ret) {
-        ret.payload.should.eql ({elem:2, pl:'twetrwte'});
-        cb (err);
-      })},
-      function (cb) {q.size (function (err, size) {
-        size.should.equal (0);
-        cb(err);
-      })},
-      function (cb) {q.stats (function (err, res) {
-        res.should.eql ({get: 2, put: 2, reserve: 0, commit: 0, rollback: 0, deadletter: 0});
-        cb(err);
-      })},
-    ], function(err, results) {
-      if (err) return done (err);
-      q.drain();
-      done();
-    });
-  });
-
-
-  it ('pop cancellation works as expected', function (done){
-    var q = factory.queue('test_queue_3');
-
-    async.series([
-      function (cb) {
-        var tid1 = q.pop ('c1', {timeout: 2000}, function (err, ret) {err.should.equal('cancel')});
-        q.consumers().length.should.equal (1);
-        var tid2 = q.pop ('c2', {timeout: 2000}, function (err, ret) {err.should.equal('cancel')});
-        q.nConsumers().should.equal (2);
-        q.cancel (tid1);
-        q.nConsumers().should.equal (1);
-        q.cancel (tid2);
-        q.nConsumers().should.equal (0);
-        cb();
-      },
-      function (cb) {q.push ({elem:2, pl:'cancel-twetrwte'}, cb)},
-      function (cb) {q.push ({elem:1, pl:'cancel-twetrwte'}, cb)},
-      function (cb) {setTimeout (cb, 1111)},
-      function (cb) {q.pop ('c3', {timeout: 15000}, function (err, ret) {
-        ret.payload.should.eql ({elem:2, pl:'cancel-twetrwte'});
-        cb (err, ret);
-      })},
-      function (cb) {q.pop ('c4', {timeout: 15000}, function (err, ret) {
-        ret.payload.should.eql ({elem:1, pl:'cancel-twetrwte'});
-        cb (err, ret);
-      })},
-      function (cb) {q.size (function (err, size) {
-        size.should.equal (0);
-        cb(err);
-      })},
-      function (cb) {q.stats (function (err, res) {
-        res.should.eql ({get: 2, put: 2, reserve: 0, commit: 0, rollback: 0, deadletter: 0});
-        cb(err);
-      })},
-    ], function(err, results) {
-      if (err) return done (err);
-      q.drain();
-      done();
-    });
-  });
-
-
-  it ('push & pop on 2 buckets ok', function (done) {
-    var q = factory.queue('test_queue_4');
-
-    async.series([
-      (cb) => q.push ({elem:1, pl:'twetrwte'}, cb),
-      (cb) => q.push ({elem:2, pl:'twetrwte'}, cb),
-      (cb) => q.push ({elem:3, pl:'twetrwte'}, cb),
-      (cb) => q.push ({elem:4, pl:'twetrwte'}, cb),
-      (cb) => setTimeout (cb, 666),
-
-      (cb) => q.size ((err, size) => {size.should.equal (4); cb(err); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 4, reserve: 0, commit: 0, rollback: 0, deadletter: 0}); cb(err); }),
-
-      (cb) => q.push ({elem:5, pl:'twetrwte'}, cb),
-      (cb) => q.push ({elem:6, pl:'twetrwte'}, cb),
-      (cb) => q.push ({elem:7, pl:'twetrwte'}, cb),
-      (cb) => setTimeout (cb, 666),
-
-      (cb) => q.size ((err, size) => {size.should.equal (7); cb(err); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 7, reserve: 0, commit: 0, rollback: 0, deadletter: 0}); cb(err); }),
-
-      (cb) => q.next_t ((err, res) => {res.getTime().should.be.approximately(new Date().getTime(), 2000); cb(err); }),
-
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:1, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:2, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:3, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:4, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:5, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:6, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:7, pl:'twetrwte'}); cb (err); })},
-
-      (cb) => q.size ((err, size) => {size.should.equal (0); cb(err); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 7, put: 7, reserve: 0, commit: 0, rollback: 0, deadletter: 0}); cb(err); }),
-    ], function(err, results) {
-      if (err) return done (err);
-      q.drain();
-      done();
-    });
-  });
-
-
-  it ('push & pop with delay ok', function (done) {
-    var q = factory.queue('test_queue_5');
-
-    async.series([
-      (cb) => q.push ({elem:1, pl:'twetrwte'}, {delay: 4}, cb),
-      (cb) => q.push ({elem:2, pl:'twetrwte'}, {delay: 2}, cb),
-      (cb) => q.push ({elem:3, pl:'twetrwte'}, {delay: 3}, cb),
-      (cb) => q.push ({elem:4, pl:'twetrwte'}, {delay: 4}, cb),
-      (cb) => setTimeout (cb, 700),
-
-      (cb) => q.next_t ((err, res) => {res.getTime().should.be.approximately(new Date().getTime() + 3300, 100); cb(err); }),
-
-      (cb) => q.push ({elem:5, pl:'twetrwte'}, {delay: 2}, cb),
-      (cb) => q.push ({elem:6, pl:'twetrwte'}, {delay: 1}, cb),
-      (cb) => q.push ({elem:7, pl:'twetrwte'}, {delay: 2}, cb),
-      (cb) => setTimeout (cb, 700),
-
-      (cb) => q.next_t ((err, res) => {res.getTime().should.be.approximately(new Date().getTime() + 1300, 100); cb(err); }),
-
-      (cb) => q.size ((err, size) => {size.should.equal (0); cb(err); }),
-      (cb) => q.totalSize ((err, size) => {size.should.equal (7); cb(err); }),
-      (cb) => q.schedSize ((err, size) => {size.should.equal (7); cb(err); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 7, reserve: 0, commit: 0, rollback: 0, deadletter: 0}); cb(err); }),
-
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:5, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:6, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:7, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:1, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:2, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:3, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:4, pl:'twetrwte'}); cb (err); })},
-
-      (cb) => q.size ((err, size) => {size.should.equal (0); cb(err); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 7, put: 7, reserve: 0, commit: 0, rollback: 0, deadletter: 0}); cb(err); }),
-    ], function(err, results) {
-      if (err) return done (err);
-      q.drain();
-      done();
-    });
-  });
-
-
-  it ('insert & rejections over various buckets go as expected', function (done) {
-    var q = factory.queue('test_queue_6');
-
-    async.series([
-      (cb) => q.push ({elem:1, pl:'twetrwte'}, cb),
-      (cb) => q.push ({elem:2, pl:'twetrwte'}, cb),
-      (cb) => setTimeout (cb, 700),
-
-      (cb) => q.size ((err, size) => {size.should.equal (2); cb(err); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 2, reserve: 0, commit: 0, rollback: 0, deadletter: 0}); cb(err); }),
-
-      (cb) => q.push ({elem:3, pl:'twetrwte'}, cb),
-      (cb) => setTimeout (cb, 700),
-
-      (cb) => q.size ((err, size) => {size.should.equal (3); cb(err); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 3, reserve: 0, commit: 0, rollback: 0, deadletter: 0}); cb(err); }),
-
-      (cb) => q.next_t ((err, res) => {res.getTime().should.be.approximately(new Date().getTime() - 1400, 100); cb(err); }),
-
-      (cb) => {
-        q.pop ('c1', {reserve: true}, (err, ret) => {
-          if (err) return cb (err);
-          ret.payload.should.eql ({elem:1, pl:'twetrwte'});
-          q.ko (ret._id, (new Date().getTime() + 5000 ), (err, res) => {
-            should (res).equal (true);
-            cb (err);
-          });
-        });
-      },
-
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:2, pl:'twetrwte'}); cb (err); })},
-
-      (cb) => {
-        q.pop ('c1', {reserve: true}, (err, ret) => {
-          if (err) return cb (err);
-          ret.payload.should.eql ({elem:3, pl:'twetrwte'});
-          q.ko (ret._id, (new Date().getTime() + 3000 ), (err, res) => {
-            should (res).equal (true);
-            cb (err);
-          });
-        });
-      },
-
-      (cb) => setTimeout (cb, 700),
-
-      (cb) => q.next_t ((err, res) => {res.getTime().should.be.approximately(new Date().getTime() + 2500, 500); cb(err); }),
-
-      (cb) => q.size ((err, size) => {size.should.equal (0); cb(err); }),
-      (cb) => q.totalSize ((err, size) => {size.should.equal (2); cb(err); }),
-      (cb) => q.schedSize ((err, size) => {size.should.equal (2); cb(err); }),
-
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:3, pl:'twetrwte'}); cb (err); })},
-      (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:1, pl:'twetrwte'}); cb (err); })},
-
-      (cb) => q.size ((err, size) => {size.should.equal (0); cb(err); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 3, put: 3, reserve: 2, commit: 0, rollback: 2, deadletter: 0}); cb(err); }),
-    ], function(err, results) {
-      if (err) return done (err);
-      q.drain();
-      done();
-    });
-  });
-});
-
-});