keuss 1.7.1 → 1.7.2

package/QFactory-MongoDB-defaults.js

@@ -11,7 +11,7 @@ class QFactory_MongoDB_defaults extends QFactory {
     super (opts);
 
     if (!this._opts.url) {
-      this._opts.url = 'http://localhost:27017/keuss';
+      this._opts.url = 'mongodb://localhost:27017/keuss';
       debug ('added url default to %s: %o', this._name, this._opts.url);
     }
 

package/Queue.js

@@ -443,7 +443,7 @@ class Queue {
       }
 
       cb (err, res);
-    });
+    }, obj);
   }
 
 
@@ -765,7 +765,7 @@ class Queue {
           return cb (null, 'deadletter');
         }
       });
-    });
+    }, obj);
   }
 }
 

package/TODO

@@ -8,6 +8,14 @@ prio 1
 * pipelines
   + add some commonplace transforms (etl-like)
   + create a server for that, allow processor code defined externally
+* documentation
+  + blog entries:
+    - motivations on using mongodb+redis. comparatives, pros&cons
+    - buckets
+    - pipelines
+    - tape
+    - redis-oq
+    - exotic experiments: intraorder
 
 prio 0
 -----------------------------------------------------------------

package/backends/intraorder.js

@@ -0,0 +1,374 @@
+const _ =     require ('lodash');
+const async = require ('async');
+const uuid =  require ('uuid');
+
+const MongoClient = require ('mongodb').MongoClient;
+const mongo =       require ('mongodb');
+
+const Queue =                     require ('../Queue');
+const QFactory_MongoDB_defaults = require ('../QFactory-MongoDB-defaults');
+
+const debug = require('debug')('keuss:Queue:intraqueue');
+
+
+class IntraOrderedQueue extends Queue {
+
+  //////////////////////////////////////////////
+  constructor (name, factory, opts, orig_opts) {
+    super (name, factory, opts, orig_opts);
+
+    this._factory = factory;
+    this._col = factory._db.collection (name);
+    this.ensureIndexes (err => {});
+  }
+
+
+  /////////////////////////////////////////
+  static Type () {
+    return 'mongo:intraorder';
+  }
+  
+
+  /////////////////////////////////////////
+  type () {
+    return 'mongo:intraorder';
+  }
+
+
+  /////////////////////////////////////////
+  // add element to queue
+  insert (entry, callback) {
+    const q = { _id: entry.payload.iid || uuid.v4 () };
+    const upd = { 
+      $push: { q: entry }, 
+      $inc: { qcnt: 1 },
+      $set: { mature: entry.mature, tries: entry.tries },
+    };
+    const opts = { upsert: true };
+
+    this._col.updateOne (q, upd, opts, (err, result) => {
+      debug ('insert: updateOne (%j, %j, %j) => (%j, %j)', q, upd, opts, err, result);
+      if (err) return callback (err);
+      // TODO result.insertedCount must be 1
+      callback (null, result.upsertedId || q._id);
+    });
+  }
+
+
+  /////////////////////////////////////////
+  // get element from queue
+  get (callback) {
+    // actually, a reserve followed by a commit
+    this.reserve ((err, elem) => {
+      if (err)   return callback (err);
+      if (!elem) return callback ();
+
+      this.commit (elem._id, (err, res) => {
+        if (err) return callback (err);
+
+        // clear _env: not needed here
+        delete elem._env;
+        callback (null, elem);
+      }, elem);
+    })
+  }
+
+
+  //////////////////////////////////
+  // reserve element: call cb (err, pl) where pl has an id
+  reserve (callback) {
+    const delay = this._opts.reserve_delay || 120;
+
+    const q = {
+      mature: {$lte: Queue.nowPlusSecs (0)},
+      qcnt: { $gt: 0}
+    };
+
+    const upd = {
+      $set: {
+        mature: Queue.nowPlusSecs (delay), 
+        reserved: new Date ()
+      },
+      $inc: {tries: 1}
+    };
+
+    const opts = {
+      sort: {mature : 1},
+      returnDocument: 'before'
+    };
+
+    this._col.findOneAndUpdate (q, upd, opts, (err, result) => {
+      debug ('reserve: findOneAndUpdate (%j, %j, %j) => (%j, %j)', q, upd, opts, err, result);
+      if (err) return callback (err);
+      const v = result && result.value;
+      if (!v) return callback ();
+
+      // construct the real object to return
+      const vq = v.q[0];
+      vq._id = v._id;  // use the whole obj's _id
+      vq.tries = v.tries
+      vq._env = v; // pass along the whole obj too
+      if (vq.payload._bsontype == 'Binary') vq.payload = vq.payload.buffer;
+      callback (null, vq);
+    });
+  }
+
+
+  //////////////////////////////////
+  // commit previous reserve, by p.id
+  commit (id, callback, obj) {
+    if (!(obj || (obj && obj._id))) {
+      // full obj must be passed to commit
+      return callback ('full obj must be passed to commit');
+    }
+
+    const q  =  {
+      _id: id,
+      reserved: {$exists: true}
+    };
+    
+    const upd = {
+// do not alter mature on commit: leave it be
+//      $set:   {
+//        mature: Queue.nowPlusSecs (100 * this._opts.ttl)
+//      },
+      $pop: {q: -1},  // pop entry from queue
+      $inc: { qcnt: -1 }, // one less element
+      $unset: {reserved: ''}
+    };
+
+    if ((obj._env && obj._env.qcnt) > 1) {
+      debug ('it is certain there are still entries in the intraqueue: set mature and tries');
+      upd.$set = {
+        mature: obj._env.q[1].mature,
+        tries: obj._env.q[1].tries,
+      }
+    }
+    else {
+      // last in queue: set mature to distant future to get it out of the way while it's GCed
+// not really, it'd impact if there were an insert in between
+//      upd.$set.mature = Queue.nowPlusSecs (60*60*24*1000);
+    }
+
+    const opts = {};
+
+    this._col.updateOne (q, upd, opts, (err, result) => {
+      debug ('commit: updateOne (%j, %j, %j) => (%j, %j)', q, upd, opts, err, result);
+      if (err) return callback (err);
+      callback (null, result && (result.modifiedCount == 1));
+    });
+  }
+
+
+  //////////////////////////////////
+  // rollback previous reserve, by p.id
+  rollback (id, next_t, callback) {
+    if (_.isFunction (next_t)) {
+      callback = next_t;
+      next_t = null;
+    }
+
+    const q =  {
+      _id: id,
+      reserved: {$exists: true}
+    };
+
+    const upd = {
+      $set:   {mature: (next_t ? new Date (next_t) : Queue.now ())},
+      $unset: {reserved: ''}
+    };
+
+    const opts = {};
+
+    this._col.updateOne (q, upd, opts, (err, result) => {
+      debug ('rollback: updateOne (%j, %j, %j) => (%j, %j)', q, upd, opts, err, result);
+      if (err) return callback (err);
+      callback (null, result && (result.modifiedCount == 1));
+    });
+  }
+
+
+  //////////////////////////////////
+  // queue size of non-mature elements only
+  totalSize (callback) {
+    const cursor = this._col.aggregate ([
+      {$match: {
+        qcnt: {$gt: 0}
+      }},
+      {$group:{_id:'t', v: {$sum: '$qcnt'}}}
+    ]);
+
+    cursor.toArray ((err, res) => {
+      debug ('calculating schedSize: aggregation pipeline returns %o', res);
+      if (err) return callback (err);
+      if (res.length == 0) return callback (null, 0);
+      callback (null, res[0].v);
+    });
+  }
+
+
+  //////////////////////////////////
+  // queue size NOT including non-mature elements
+  size (callback) {
+    const cursor = this._col.aggregate ([
+      {$match: {
+        mature: {$lte: Queue.now ()},
+        qcnt: {$gt: 0}
+      }},
+      {$group:{_id:'t', v: {$sum: '$qcnt'}}}
+    ]);
+
+    cursor.toArray ((err, res) => {
+      debug ('calculating schedSize: aggregation pipeline returns %o', res);
+      if (err) return callback (err);
+      if (res.length == 0) return callback (null, 0);
+      callback (null, res[0].v);
+    });
+  }
+
+
+  //////////////////////////////////
+  // queue size of non-mature elements only
+  schedSize (callback) {
+    const cursor = this._col.aggregate ([
+      {$match: {
+        mature: {$gt: Queue.now ()},
+        reserved: {$exists: false},
+        qcnt: {$gt: 0}
+      }},
+      {$group:{_id:'t', v: {$sum: '$qcnt'}}}
+    ]);
+
+    cursor.toArray ((err, res) => {
+      debug ('calculating schedSize: aggregation pipeline returns %o', res);
+      if (err) return callback (err);
+      if (res.length == 0) return callback (null, 0);
+      callback (null, res[0].v);
+    });
+  }
+
+
+  //////////////////////////////////
+  // queue size of reserved elements only
+  resvSize (callback) {
+    const cursor = this._col.aggregate ([
+      {$match: {
+        mature: {$gt: Queue.now ()},
+        reserved: {$exists: true},
+        qcnt: {$gt: 0}
+      }},
+      {$group:{_id:'t', v: {$sum: '$qcnt'}}}
+    ]);
+
+    cursor.toArray ((err, res) => {
+      debug ('calculating schedSize: aggregation pipeline returns %o', res);
+      if (err) return callback (err);
+      if (res.length == 0) return callback (null, 0);
+      callback (null, res[0].v);
+    });
+  }
+
+
+  //////////////////////////////////////////////
+  // remove by id
+  remove (id, callback) {
+    const q =  {
+      _id: id,
+      qcnt: { $eq: 1 },  // allow deletion ONLY if it has just one element in the intraqueue
+      reserved: {$exists: false}
+    };
+
+    const opts = {};
+
+    this._col.deleteOne (q, opts, (err, result) => {
+      debug ('remove: deleteOne (%j, %j) => (%j, %j)', q, opts, err, result);
+      if (err) return callback (err);
+      callback (null, result && (result.deletedCount == 1));
+    });
+  }
+
+
+  /////////////////////////////////////////
+  // get element from queue
+  next_t (callback) {
+    this._col
+    .find ()
+    .limit(1)
+    .sort ({mature:1})
+    .project ({mature:1})
+    .next ((err, result) => {
+      if (err) return callback (err);
+      callback (null, result && result.mature);
+    });
+  }
+
+
+  ///////////////////////////////////////////////////////////////////////////////
+  // private parts
+
+  //////////////////////////////////////////////////////////////////
+  // create needed indexes for O(1) functioning
+  ensureIndexes (cb) {
+    async.series ([
+      cb => this._col.createIndex ({mature : 1, qcnt: 1}, cb),
+    ], cb);
+  }
+}
+
+
+class Factory extends QFactory_MongoDB_defaults {
+  constructor (opts, mongo_conn) {
+    super (opts);
+    this._mongo_conn = mongo_conn;
+    this._db = mongo_conn.db();
+  }
+
+  queue (name, opts) {
+    var full_opts = {};
+    _.merge(full_opts, this._opts, opts);
+    return new IntraOrderedQueue (name, this, full_opts, opts);
+  }
+
+  close (cb) {
+    super.close (() => {
+      if (this._mongo_conn) {
+        this._mongo_conn.close ();
+        this._mongo_conn = null;
+      }
+
+      if (cb) return cb ();
+    });
+  }
+
+  type () {
+    return IntraOrderedQueue.Type ();
+  }
+
+  capabilities () {
+    return {
+      sched:    true,
+      reserve:  true,
+      pipeline: false,
+      tape:     false,
+      remove:   false
+    };
+  }
+}
+
+function creator (opts, cb) {
+  var _opts = opts || {};
+  var m_url = _opts.url || 'mongodb://localhost:27017/keuss';
+
+  MongoClient.connect (m_url, { useNewUrlParser: true }, (err, cl) => {
+    if (err) return cb (err);
+    var F = new Factory (_opts, cl);
+    F.async_init (err => cb (null, F));
+  });
+}
+
+module.exports = creator;
+
+
+
+
+

package/docker-compose.yaml

@@ -0,0 +1,18 @@
+version: '3'
+
+services:
+  mongo:
+    image: mongo:4.4.2-bionic
+    command: --logpath /dev/null
+    restart: on-failure
+    ports:
+      - 27017:27017
+
+  redis:
+    image: redis:alpine
+    ports:
+     - 6379:6379
+
+networks:
+  default:
+

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "keuss",
-  "version": "1.7.1",
+  "version": "1.7.2",
   "keywords": [
     "queue",
     "persistent",
@@ -27,23 +27,23 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@nodebb/mubsub": "~1.8.0",
-    "async": "~3.2.3",
+    "async": "~3.2.4",
     "async-lock": "~1.3.1",
     "debug": "~4.3.4",
     "ioredis": "~5.0.4",
     "lodash": "~4.17.21",
     "mitt": "~3.0.0",
     "mongodb": "~4.5.0",
-    "uuid": "~8.3.2",
-    "why-is-node-running": "^2.2.2"
+    "uuid": "~8.3.2"
   },
   "devDependencies": {
     "chance": "~1.1.8",
-    "mocha": "~9.2.2",
-    "should": "~13.2.3"
+    "mocha": "~10.2.0",
+    "should": "~13.2.3",
+    "why-is-node-running": "^2.2.2"
   },
   "scripts": {
-    "test": "mocha --reporter spec --check-leaks --no-timeouts --exit test/",
-    "test-with-coverage": "nyc --reporter=html -- mocha --reporter spec --check-leaks --no-timeouts --exit test/"
+    "test": "docker compose up -d; sleep 5; mocha --reporter spec --check-leaks --no-timeouts --exit test/ ; docker compose down",
+    "test-with-coverage": "docker compose up -d; sleep 5; nyc --reporter=html -- mocha --reporter spec --check-leaks --no-timeouts --exit ; test/docker compose down"
   }
 }

package/playground/irc.js

@@ -0,0 +1,53 @@
+const async = require ('async');
+const MQ =    require ('../backends/intraorder');
+
+// initialize factory
+MQ ({
+  url: 'mongodb://localhost/keuss'
+}, (err, factory) => {
+  if (err) return console.error(err);
+
+  const q = factory.queue ('test_queue', {});
+/*
+  async.waterfall ([
+    cb =>            q.push ({iid: 123, elem: 1, headline: 'something something', tags: {a: 1, b: 2}}, cb),  
+    (item_id, cb) => q.push ({iid: 123, elem: 2, headline: 'other other', tags: {a: 3, b: 4}}, cb),  
+    (item_id, cb) => q.pop ('consumer-1', {reserve: true}, cb), 
+    (item, cb) =>    q.ko (item, new Date().getTime () + 1500, cb), 
+    (item_id, cb) => q.pop ('consumer-1', {reserve: true}, cb), 
+    (item, cb) =>    {console.log ('%s: got %o', new Date().toISOString (), item.payload); q.ok (item, cb); },
+    (item_id, cb) => q.pop ('consumer-1', {reserve: true}, cb), 
+    (item, cb) =>    q.ko (item, new Date().getTime () + 1500, cb), 
+    (item_id, cb) => q.pop ('consumer-1', {reserve: true}, cb), 
+    (item, cb) =>    {console.log ('%s: got %o', new Date().toISOString (), item.payload); q.ok (item, cb); },
+    (i, cb) =>       setTimeout (cb, 100),
+    cb =>            q.status (cb),
+  ], (err, res) => {
+    if (err) console.error (err);
+    console.log (res)
+    factory.close ();
+  });
+
+*/
+
+
+async.series ([
+  cb => q.status (cb),
+  cb => q.push ({iid: 123, elem: 1, headline: 'something something', tags: {a: 1, b: 2}}, cb), 
+  cb => q.status (cb), 
+  cb => q.push ({iid: 123, elem: 2, headline: 'other other', tags: {a: 3, b: 4}}, cb),  
+  cb => q.status (cb),
+  cb => q.pop ('consumer-1', cb), 
+  cb => q.status (cb),
+  cb => q.pop ('consumer-1', cb), 
+  cb => q.status (cb),
+  cb => setTimeout (cb, 100),
+  cb => q.status (cb),
+], (err, res) => {
+  if (err) console.error (err);
+  factory.close ();
+  res.forEach ((v, i) => console.log ('%d:', i, v ));
+});
+
+});
+

package/test/backends_deadletter.js

@@ -77,12 +77,13 @@ function release_mq_factory (q, factory, cb) {
 
 /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 [
-  {label: 'Simple MongoDB',     mq: require ('../backends/mongo')},
-  {label: 'Pipelined MongoDB',  mq: require ('../backends/pl-mongo')},
-  {label: 'Tape MongoDB',       mq: require ('../backends/ps-mongo')},
-  {label: 'Stream MongoDB',     mq: require ('../backends/stream-mongo')},
-  {label: 'Redis OrderedQueue', mq: require ('../backends/redis-oq')},
-  {label: 'MongoDB SafeBucket', mq: require ('../backends/bucket-mongo-safe')}
+//  {label: 'Simple MongoDB',     mq: require ('../backends/mongo')},
+//  {label: 'Pipelined MongoDB',  mq: require ('../backends/pl-mongo')},
+//  {label: 'Tape MongoDB',       mq: require ('../backends/ps-mongo')},
+//  {label: 'Stream MongoDB',     mq: require ('../backends/stream-mongo')},
+//  {label: 'Redis OrderedQueue', mq: require ('../backends/redis-oq')},
+//  {label: 'MongoDB SafeBucket', mq: require ('../backends/bucket-mongo-safe')},
+  {label: 'Mongo IntraOrder',   mq: require ('../backends/intraorder')},
 ].forEach(function (MQ_item) {
   describe('rollback and deadletters with ' + MQ_item.label + ' queue backend', function () {
     const MQ = MQ_item.mq;

package/test/backends_payload.js

@@ -10,13 +10,14 @@ var MongoClient = require ('mongodb').MongoClient;
 var factory = null;
 
 [
-  {label: 'Simple MongoDB',         mq: require ('../backends/mongo')},
-  {label: 'Pipelined MongoDB',      mq: require ('../backends/pl-mongo')},
-  {label: 'Tape MongoDB',           mq: require ('../backends/ps-mongo')},
-  {label: 'Stream MongoDB',         mq: require ('../backends/stream-mongo')},
-  {label: 'Safe MongoDB Buckets',   mq: require ('../backends/bucket-mongo-safe')},
-  {label: 'Redis List',             mq: require ('../backends/redis-list')},
-  {label: 'Redis OrderedQueue',     mq: require ('../backends/redis-oq')},
+  {label: 'Simple MongoDB',       mq: require ('../backends/mongo')},
+  {label: 'Pipelined MongoDB',    mq: require ('../backends/pl-mongo')},
+  {label: 'Tape MongoDB',         mq: require ('../backends/ps-mongo')},
+  {label: 'Stream MongoDB',       mq: require ('../backends/stream-mongo')},
+  {label: 'Safe MongoDB Buckets', mq: require ('../backends/bucket-mongo-safe')},
+  {label: 'Redis List',           mq: require ('../backends/redis-list')},
+  {label: 'Redis OrderedQueue',   mq: require ('../backends/redis-oq')},
+  {label: 'Mongo IntraOrder',     mq: require ('../backends/intraorder')},
 ].forEach(function (MQ_item) {
   describe('payload aspects on ' + MQ_item.label + ' queue backend, round 1', function () {
     var MQ = MQ_item.mq;

package/test/backends_reserve-commit-rollback.js

@@ -15,7 +15,8 @@ var factory = null;
   {label: 'Tape MongoDB',         mq: require ('../backends/ps-mongo')},
   {label: 'Stream MongoDB',       mq: require ('../backends/stream-mongo')},
  // {label: 'Safe MongoDB Buckets', mq: require ('../backends/bucket-mongo-safe')},
-  {label: 'Redis OrderedQueue',   mq: require ('../backends/redis-oq')}
+  {label: 'Redis OrderedQueue',   mq: require ('../backends/redis-oq')},
+  {label: 'Mongo IntraOrder',     mq: require ('../backends/intraorder')},
 ].forEach(function (MQ_item) {
   describe('reserve-commit-rollback with ' + MQ_item.label + ' queue backend', function () {
     var MQ = MQ_item.mq;
@@ -130,7 +131,9 @@ var factory = null;
           cb();
         }),
         cb => q.next_t((err, res) => {
-          should.equal(res, null);
+          if (q.type () !=  'mongo:intraorder') {
+            should.equal(res, null);
+          }
           cb();
         })
       ], (err, results) => {
@@ -252,7 +255,9 @@ var factory = null;
         },
         function (cb) {
           q.next_t(function (err, res) {
-            should.equal(res, null);
+            if (q.type () !=  'mongo:intraorder') {
+              should.equal(res, null);
+            }
             cb();
           });
         }
@@ -604,6 +609,7 @@ var factory = null;
     it('should do raw reserve & commit as expected', function (done) {
       var q = factory.queue('test_queue_7');
       var id = null;
+      var obj = null;
 
       async.series([
         cb => q.push({elem: 1, pl: 'twetrwte'}, cb),
@@ -617,6 +623,7 @@ var factory = null;
         }),
         cb => q.reserve((err, res) => {
           id = res._id;
+          obj = res;
           res.payload.should.eql({
             elem: 1,
             pl: 'twetrwte'
@@ -639,7 +646,7 @@ var factory = null;
         cb => q.commit(id, (err, res) => {
           res.should.equal(true);
           cb();
-        }),
+        }, obj),
         cb => q.size((err, size) => {
           size.should.equal(0);
           cb();
@@ -649,17 +656,20 @@ var factory = null;
           cb();
         }),
         cb => q.next_t((err, res) => {
-          should.equal(res, null);
+          if (q.type () !=  'mongo:intraorder') {
+            should.equal(res, null);
+          }
           cb();
         }),
       ], (err, results) => {
-        done();
+        done(err);
       });
     });
 
     it('should do raw reserve & rollback as expected', function (done) {
       var q = factory.queue('test_queue_8');
       var id = null;
+      var obj = null;
 
       async.series([
         function (cb) {
@@ -683,6 +693,7 @@ var factory = null;
         function (cb) {
           q.reserve(function (err, res) {
             id = res._id;
+            obj = res;
             res.payload.should.eql({
               elem: 1,
               pl: 'twetrwte'
@@ -737,7 +748,7 @@ var factory = null;
           q.commit(id, function (err, res) {
             res.should.equal(false);
             cb();
-          })
+          }, obj)
         },
         function (cb) {
           q.size(function (err, size) {
@@ -782,7 +793,9 @@ var factory = null;
         },
         function (cb) {
           q.next_t(function (err, res) {
-            should.equal(res, null);
+            if (q.type () !=  'mongo:intraorder') {
+              should.equal(res, null);
+            }
             cb();
           })
         },
@@ -794,6 +807,7 @@ var factory = null;
     it('should do get.reserve & ok as expected', function (done) {
       var q = factory.queue('test_queue_9');
       var id = null;
+      var obj = null;
 
       async.series([
         function (cb) {
@@ -807,6 +821,7 @@ var factory = null;
             reserve: true
           }, function (err, res) {
             id = res._id;
+            obj = res;
             res.payload.should.eql({
               elem: 1,
               pl: 'twetrwte'
@@ -834,7 +849,7 @@ var factory = null;
           })
         },
         function (cb) {
-          q.ok(id, function (err, res) {
+          q.ok(obj, function (err, res) {
             res.should.equal(true);
             cb();
           })
@@ -853,7 +868,9 @@ var factory = null;
         },
         function (cb) {
           q.next_t(function (err, res) {
-            should.equal(res, null);
+            if (q.type () !=  'mongo:intraorder') {
+              should.equal(res, null);
+            }
             cb();
           })
         },
@@ -875,10 +892,11 @@ var factory = null;
     });
 
     it('should manage rollback on invalid id as expected', function (done) {
-      if (MQ_item.label == 'Redis OrderedQueue') return done ();
-
       var q = factory.queue('test_queue_10');
 
+      if (q.type () ==  'redis:oq') return done ();
+      if (q.type () ==  'mongo:intraorder') return done ();
+
       async.series([
         function (cb) {
           q.rollback('invalid-id', function (err, res) {
@@ -924,10 +942,11 @@ var factory = null;
         cb => q.pop ('me', {reserve: true}, (err, res) => {
           if (err) return db (err);
           state.reserved_id = res._id;
+          state.reserved_obj = res;
           cb (null, res._id);
         }),
         cb => _get_all_sizes (q, cb),
-        cb => q.ok (state.reserved_id, cb),
+        cb => q.ok (state.reserved_id, cb, state.reserved_obj),
         cb => _get_all_sizes (q, cb),
         cb => q.pop ('me', cb),
         cb => q.pop ('me', cb),

package/test/intraorder.js

@@ -0,0 +1,325 @@
+const async =  require ('async');
+const should = require ('should');
+const _ =      require ('lodash');
+
+const LocalSignal = require ('../signal/local');
+const MemStats =    require ('../stats/mem');
+
+const MongoClient = require ('mongodb').MongoClient;
+
+
+const MQ = require ('../backends/intraorder');
+
+var factory = null;
+
+
+  describe('IntraOrder backend: specific operations', () => {
+
+    /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+    before(done => {
+      var opts = {
+        url: 'mongodb://localhost/keuss_test_intraorder',
+        signaller: {provider: LocalSignal},
+        stats: {provider: MemStats}
+      };
+
+      MQ(opts, (err, fct) => {
+        if (err) return done(err);
+        factory = fct;
+        done();
+      });
+    });
+
+
+    /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+    after (done => async.series ([
+      cb => setTimeout (cb, 1000),
+      cb => factory.close (cb),
+      cb => MongoClient.connect ('mongodb://localhost/keuss_test_intraorder', (err, cl) => {
+        if (err) return done (err);
+        cl.db().dropDatabase (() => cl.close (cb))
+      })
+    ], done));
+
+
+    /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+    it('sequential push & pops with no retries preserves order', done => {
+      const q = factory.queue('test_queue_1');
+
+      async.series([
+        cb => q.push({elem: 1, iid: 'twetrwte', pl: {d: 't-', a: 56}}, cb),
+        cb => q.push({elem: 2, iid: 'twetrwte', pl: {d: 't--', a: 156}}, cb),
+        cb => q.push({elem: 3, iid: 'twetrwte', pl: {d: 't---', a: 256}}, cb),
+        cb => q.push({elem: 4, iid: 'twetrwte', pl: {d: 't----', a: 356}}, cb),
+        cb => q.push({elem: 5, iid: 'twetrwte', pl: {d: 't-----', a: 456}}, cb),
+        cb => {
+          q.size ((err, size) => {
+            if (err) return done (err);
+            size.should.equal(5);
+            cb();
+          })
+        },
+        cb => q.stats((err, res) => {
+          if (err) return done (err);
+          res.should.match({
+            get: 0,
+            put: 5,
+            reserve: 0,
+            commit: 0,
+            rollback: 0, 
+            deadletter: 0
+          });
+          cb();
+        }),
+        cb => q.pop('c1', cb),
+        cb => q.pop('c1', cb),
+        cb => q.pop('c1', cb),
+        cb => q.pop('c1', cb),
+        cb => q.pop('c1', cb),
+        cb => q.size((err, size) => {
+          if (err) return done (err);
+          size.should.equal(0);
+          cb();
+        }),
+        cb => q.stats((err, res) => {
+          if (err) return done (err);
+          res.should.match({
+            get: 5,
+            put: 5,
+            reserve: 0,
+            commit: 0,
+            rollback: 0,
+            deadletter: 0
+          });
+          cb();
+        }),
+      ], (err, results) => {
+        if (err) return done (err);
+        results[7].payload.should.eql ({ elem: 1, iid: 'twetrwte', pl: { d: 't-', a: 56 } });
+        results[8].payload.should.eql ({ elem: 2, iid: 'twetrwte', pl: { d: 't--', a: 156 } });
+        results[9].payload.should.eql ({ elem: 3, iid: 'twetrwte', pl: { d: 't---', a: 256 } });
+        results[10].payload.should.eql ({ elem: 4, iid: 'twetrwte', pl: { d: 't----', a: 356 } });
+        results[11].payload.should.eql ({ elem: 5, iid: 'twetrwte', pl: { d: 't-----', a: 456 } });
+        done();
+      });
+    });
+
+
+    /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+    it('sequential reserve-commit with retries preserves order', done => {
+      const q = factory.queue('test_queue_2');
+      const t0 = process.hrtime();
+      async.series([
+        cb => q.push({elem: 1, iid: 'twetrwte', pl: {d: 't-', a: 56}}, cb),
+        cb => q.push({elem: 2, iid: 'twetrwte', pl: {d: 't--', a: 156}}, cb),
+        cb => q.push({elem: 3, iid: 'twetrwte', pl: {d: 't---', a: 256}}, cb),
+        cb => q.push({elem: 4, iid: 'twetrwte', pl: {d: 't----', a: 356}}, cb),
+        cb => q.push({elem: 5, iid: 'twetrwte', pl: {d: 't-----', a: 456}}, cb),
+
+        cb => q.pop('c1', {reserve: true}, (err, res) => {
+          if (err) return done (err);
+          res.payload.should.eql ({ elem: 1, iid: 'twetrwte', pl: { d: 't-', a: 56 } });
+          q.ko (res, new Date().getTime () + 2000, cb)
+        }),
+
+        cb => q.pop('c1', {reserve: true}, (err, res) => {
+          if (err) return done (err);
+          res.payload.should.eql ({ elem: 1, iid: 'twetrwte', pl: { d: 't-', a: 56 } });
+          q.ko (res, new Date().getTime () + 3000, cb)
+        }),
+
+        cb => q.pop('c1', {reserve: true}, (err, res) => {
+          if (err) return done (err);
+          res.payload.should.eql ({ elem: 1, iid: 'twetrwte', pl: { d: 't-', a: 56 } });
+          q.ko (res, new Date().getTime () + 1000, cb)
+        }),
+
+//        cb => q.pop('c1', (err, res) => {console.log (new Date().toISOString (), res.payload); cb (err, res)}),
+
+        cb => q.pop('c1', cb),
+        cb => q.pop('c1', cb),
+        cb => q.pop('c1', cb),
+        cb => q.pop('c1', cb),
+        cb => q.pop('c1', cb),
+
+        cb => q.size((err, size) => {
+          if (err) return done (err);
+          size.should.equal(0);
+          cb();
+        }),
+      ], (err, results) => {
+        if (err) return done (err);
+
+        // duration must be about 6 secs
+        const delta = process.hrtime(t0);
+        delta[0].should.eql (6);
+        results[8].payload.should.eql ({ elem: 1, iid: 'twetrwte', pl: { d: 't-', a: 56 } });
+        results[9].payload.should.eql ({ elem: 2, iid: 'twetrwte', pl: { d: 't--', a: 156 } });
+        results[10].payload.should.eql ({ elem: 3, iid: 'twetrwte', pl: { d: 't---', a: 256 } });
+        results[11].payload.should.eql ({ elem: 4, iid: 'twetrwte', pl: { d: 't----', a: 356 } });
+        results[12].payload.should.eql ({ elem: 5, iid: 'twetrwte', pl: { d: 't-----', a: 456 } });
+        done();
+      });
+    });
+
+
+    /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+    it('parallel reserve-commit on same iid blocks until commit', done => {
+      const q = factory.queue('test_queue_3');
+      const t0 = process.hrtime();
+      const pops = [];
+
+      async.series([
+        cb => q.push({elem: 1, iid: 'twetrwte', pl: {d: 't-', a: 56}}, cb),
+        cb => q.push({elem: 2, iid: 'twetrwte', pl: {d: 't--', a: 156}}, cb),
+        cb => q.push({elem: 3, iid: 'twetrwte', pl: {d: 't---', a: 256}}, cb),
+
+        cb => async.parallel ([
+          cb => {
+            let elem;
+            async.series ([
+              cb => setTimeout (cb, 500),
+              cb => q.pop('c1', {reserve: true}, (err, res) => {elem = res; pops.push (res); cb (err); }),
+              cb => setTimeout (cb, 1500),
+              cb => q.ok (elem, cb),
+            ], cb);
+          },
+          cb => {
+            let elem;
+            async.series ([
+              cb => setTimeout (cb, 1000),
+              cb => q.pop('c1', {reserve: true}, (err, res) => {elem = res; pops.push (res);; cb (err); }),
+              cb => setTimeout (cb, 1500),
+              cb => q.ok (elem, cb),
+            ], cb);
+          },
+          cb => {
+            let elem;
+            async.series ([
+              cb => setTimeout (cb, 1500),
+              cb => q.pop('c1', {reserve: true}, (err, res) => {elem = res; pops.push (res);; cb (err); }),
+              cb => setTimeout (cb, 1500),
+              cb => q.ok (elem, cb),
+            ], cb);
+          },
+        ], cb)
+      ], (err, results) => {
+        if (err) return done (err);
+        
+        pops[0].payload.should.eql ({ elem: 1, iid: 'twetrwte', pl: { d: 't-', a: 56 } });
+        pops[1].payload.should.eql ({ elem: 2, iid: 'twetrwte', pl: { d: 't--', a: 156 } });
+        pops[2].payload.should.eql ({ elem: 3, iid: 'twetrwte', pl: { d: 't---', a: 256 } });
+
+        // duration must be about 6 secs
+        const delta = process.hrtime(t0);
+
+        // queue pop won't rearm after commit... so it will timeout after a min and only then rearm again for next pop/reserve
+//        delta[0].should.eql (7);
+        done();
+      });
+    });
+
+
+    /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+    it('parallel reserve-commit on same iid blocks until rollback', done => {
+      const q = factory.queue('test_queue_4');
+      const t0 = process.hrtime();
+      const pops = [];
+
+      async.series([
+        cb => q.push({elem: 1, iid: 'twetrwte', pl: {d: 't-', a: 56}}, cb),
+        cb => q.push({elem: 2, iid: 'twetrwte', pl: {d: 't--', a: 156}}, cb),
+        cb => q.push({elem: 3, iid: 'twetrwte', pl: {d: 't---', a: 256}}, cb),
+
+        cb => async.parallel ([
+          cb => {
+            let elem;
+            async.series ([
+              cb => setTimeout (cb, 500),
+              cb => q.pop('c1', {reserve: true}, (err, res) => {elem = res; pops.push (res); cb (err); }),
+              cb => q.ko (elem, new Date().getTime () + 1000, cb),
+            ], cb);
+          },
+          cb => {
+            let elem;
+            async.series ([
+              cb => setTimeout (cb, 700),
+              cb => q.pop('c1', {reserve: true}, (err, res) => {elem = res; pops.push (res);; cb (err); }),
+              cb => q.ko (elem, new Date().getTime () + 1000, cb),
+            ], cb);
+          },
+          cb => {
+            let elem;
+            async.series ([
+              cb => setTimeout (cb, 1500),
+              cb => q.pop('c1', {reserve: true}, (err, res) => {elem = res; pops.push (res);; cb (err); }),
+              cb => setTimeout (cb, 1500),
+              cb => q.ok (elem, cb),
+            ], cb);
+          },
+          cb => {
+            let elem;
+            async.series ([
+              cb => setTimeout (cb, 2000),
+              cb => q.pop('c1', {reserve: true}, (err, res) => {elem = res; pops.push (res);; cb (err); }),
+              cb => setTimeout (cb, 1500),
+              cb => q.ok (elem, cb),
+            ], cb);
+          },
+          cb => {
+            let elem;
+            async.series ([
+              cb => setTimeout (cb, 2500),
+              cb => q.pop('c1', {reserve: true}, (err, res) => {elem = res; pops.push (res);; cb (err); }),
+              cb => setTimeout (cb, 1500),
+              cb => q.ok (elem, cb),
+            ], cb);
+          },
+        ], cb)
+      ], (err, results) => {
+        if (err) return done (err);
+        
+        pops[0].payload.should.eql ({ elem: 1, iid: 'twetrwte', pl: { d: 't-', a: 56 } });
+        pops[1].payload.should.eql ({ elem: 1, iid: 'twetrwte', pl: { d: 't-', a: 56 } });
+        pops[2].payload.should.eql ({ elem: 1, iid: 'twetrwte', pl: { d: 't-', a: 56 } });
+        pops[3].payload.should.eql ({ elem: 2, iid: 'twetrwte', pl: { d: 't--', a: 156 } });
+        pops[4].payload.should.eql ({ elem: 3, iid: 'twetrwte', pl: { d: 't---', a: 256 } });
+
+        // duration must be about 6 secs
+        const delta = process.hrtime(t0);
+
+        // queue pop won't rearm after commit... so it will timeout after a min and only then rearm again for next pop/reserve
+//        delta[0].should.eql (7);
+        done();
+      });
+    });
+
+    
+    /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+    it('delayed push preserves order', done => {
+      const q = factory.queue('test_queue_1');
+
+      async.series([
+        cb => q.push({elem: 1, iid: 'twetrwte', pl: {d: 't-', a: 56}}, {delay: 5}, cb),
+        cb => q.push({elem: 2, iid: 'twetrwte', pl: {d: 't--', a: 156}}, {delay: 4}, cb),
+        cb => q.push({elem: 3, iid: 'twetrwte', pl: {d: 't---', a: 256}}, {delay: 3}, cb),
+        cb => q.push({elem: 4, iid: 'twetrwte', pl: {d: 't----', a: 356}}, {delay: 2}, cb),
+        cb => q.push({elem: 5, iid: 'twetrwte', pl: {d: 't-----', a: 456}}, {delay: 1}, cb),
+        cb => q.pop('c1', cb),
+        cb => q.pop('c1', cb),
+        cb => q.pop('c1', cb),
+        cb => q.pop('c1', cb),
+        cb => q.pop('c1', cb),
+      ], (err, results) => {
+        if (err) return done (err);
+        results[5].payload.should.eql ({ elem: 1, iid: 'twetrwte', pl: { d: 't-', a: 56 } });
+        results[6].payload.should.eql ({ elem: 2, iid: 'twetrwte', pl: { d: 't--', a: 156 } });
+        results[7].payload.should.eql ({ elem: 3, iid: 'twetrwte', pl: { d: 't---', a: 256 } });
+        results[8].payload.should.eql ({ elem: 4, iid: 'twetrwte', pl: { d: 't----', a: 356 } });
+        results[9].payload.should.eql ({ elem: 5, iid: 'twetrwte', pl: { d: 't-----', a: 456 } });
+        done();
+      });
+    });
+
+
+  });

package/test/pause.js

@@ -14,6 +14,7 @@ var factory = null;
   {label: 'Stream MongoDB',      backend: require ('../backends/stream-mongo')},
   {label: 'plain MongoDB',       backend: require ('../backends/mongo')},
   {label: 'Safe MongoDB Bucket', backend: require ('../backends/bucket-mongo-safe')},
+  {label: 'Mongo IntraOrder',    backend: require ('../backends/intraorder')},
 ].forEach (backend_item => {
   [
     {label: 'mem',   stats: require('../stats/mem')},