keuss 1.7.1 → 1.7.3

package/QFactory-MongoDB-defaults.js

@@ -11,7 +11,7 @@ class QFactory_MongoDB_defaults extends QFactory {
     super (opts);
 
     if (!this._opts.url) {
-      this._opts.url = 'http://localhost:27017/keuss';
+      this._opts.url = 'mongodb://localhost:27017/keuss';
       debug ('added url default to %s: %o', this._name, this._opts.url);
     }
 

package/Queue.js

@@ -443,7 +443,7 @@ class Queue {
       }
 
       cb (err, res);
-    });
+    }, obj);
   }
 
 
@@ -765,7 +765,7 @@ class Queue {
           return cb (null, 'deadletter');
         }
       });
-    });
+    }, obj);
   }
 }
 

package/TODO

@@ -8,6 +8,14 @@ prio 1
 * pipelines
   + add some commonplace transforms (etl-like)
   + create a server for that, allow processor code defined externally
+* documentation
+  + blog entries:
+    - motivations on using mongodb+redis. comparatives, pros&cons
+    - buckets
+    - pipelines
+    - tape
+    - redis-oq
+    - exotic experiments: intraorder
 
 prio 0
 -----------------------------------------------------------------

package/backends/intraorder.js

@@ -0,0 +1,374 @@
+const _ =     require ('lodash');
+const async = require ('async');
+const uuid =  require ('uuid');
+
+const MongoClient = require ('mongodb').MongoClient;
+const mongo =       require ('mongodb');
+
+const Queue =                     require ('../Queue');
+const QFactory_MongoDB_defaults = require ('../QFactory-MongoDB-defaults');
+
+const debug = require('debug')('keuss:Queue:intraqueue');
+
+
+class IntraOrderedQueue extends Queue {
+
+  //////////////////////////////////////////////
+  constructor (name, factory, opts, orig_opts) {
+    super (name, factory, opts, orig_opts);
+
+    this._factory = factory;
+    this._col = factory._db.collection (name);
+    this.ensureIndexes (err => {});
+  }
+
+
+  /////////////////////////////////////////
+  static Type () {
+    return 'mongo:intraorder';
+  }
+  
+
+  /////////////////////////////////////////
+  type () {
+    return 'mongo:intraorder';
+  }
+
+
+  /////////////////////////////////////////
+  // add element to queue
+  insert (entry, callback) {
+    const q = { _id: entry.payload.iid || uuid.v4 () };
+    const upd = { 
+      $push: { q: entry }, 
+      $inc: { qcnt: 1 },
+      $set: { mature: entry.mature, tries: entry.tries },
+    };
+    const opts = { upsert: true };
+
+    this._col.updateOne (q, upd, opts, (err, result) => {
+      debug ('insert: updateOne (%j, %j, %j) => (%j, %j)', q, upd, opts, err, result);
+      if (err) return callback (err);
+      // TODO result.insertedCount must be 1
+      callback (null, result.upsertedId || q._id);
+    });
+  }
+
+
+  /////////////////////////////////////////
+  // get element from queue
+  get (callback) {
+    // actually, a reserve followed by a commit
+    this.reserve ((err, elem) => {
+      if (err)   return callback (err);
+      if (!elem) return callback ();
+
+      this.commit (elem._id, (err, res) => {
+        if (err) return callback (err);
+
+        // clear _env: not needed here
+        delete elem._env;
+        callback (null, elem);
+      }, elem);
+    })
+  }
+
+
+  //////////////////////////////////
+  // reserve element: call cb (err, pl) where pl has an id
+  reserve (callback) {
+    const delay = this._opts.reserve_delay || 120;
+
+    const q = {
+      mature: {$lte: Queue.nowPlusSecs (0)},
+      qcnt: { $gt: 0}
+    };
+
+    const upd = {
+      $set: {
+        mature: Queue.nowPlusSecs (delay), 
+        reserved: new Date ()
+      },
+      $inc: {tries: 1}
+    };
+
+    const opts = {
+      sort: {mature : 1},
+      returnDocument: 'before'
+    };
+
+    this._col.findOneAndUpdate (q, upd, opts, (err, result) => {
+      debug ('reserve: findOneAndUpdate (%j, %j, %j) => (%j, %j)', q, upd, opts, err, result);
+      if (err) return callback (err);
+      const v = result && result.value;
+      if (!v) return callback ();
+
+      // construct the real object to return
+      const vq = v.q[0];
+      vq._id = v._id;  // use the whole obj's _id
+      vq.tries = v.tries
+      vq._env = v; // pass along the whole obj too
+      if (vq.payload._bsontype == 'Binary') vq.payload = vq.payload.buffer;
+      callback (null, vq);
+    });
+  }
+
+
+  //////////////////////////////////
+  // commit previous reserve, by p.id
+  commit (id, callback, obj) {
+    if (!(obj || (obj && obj._id))) {
+      // full obj must be passed to commit
+      return callback ('full obj must be passed to commit');
+    }
+
+    const q  =  {
+      _id: id,
+      reserved: {$exists: true}
+    };
+    
+    const upd = {
+// do not alter mature on commit: leave it be
+//      $set:   {
+//        mature: Queue.nowPlusSecs (100 * this._opts.ttl)
+//      },
+      $pop: {q: -1},  // pop entry from queue
+      $inc: { qcnt: -1 }, // one less element
+      $unset: {reserved: ''}
+    };
+
+    if ((obj._env && obj._env.qcnt) > 1) {
+      debug ('it is certain there are still entries in the intraqueue: set mature and tries');
+      upd.$set = {
+        mature: obj._env.q[1].mature,
+        tries: obj._env.q[1].tries,
+      }
+    }
+    else {
+      // last in queue: set mature to distant future to get it out of the way while it's GCed
+// not really, it'd impact if there were an insert in between
+//      upd.$set.mature = Queue.nowPlusSecs (60*60*24*1000);
+    }
+
+    const opts = {};
+
+    this._col.updateOne (q, upd, opts, (err, result) => {
+      debug ('commit: updateOne (%j, %j, %j) => (%j, %j)', q, upd, opts, err, result);
+      if (err) return callback (err);
+      callback (null, result && (result.modifiedCount == 1));
+    });
+  }
+
+
+  //////////////////////////////////
+  // rollback previous reserve, by p.id
+  rollback (id, next_t, callback) {
+    if (_.isFunction (next_t)) {
+      callback = next_t;
+      next_t = null;
+    }
+
+    const q =  {
+      _id: id,
+      reserved: {$exists: true}
+    };
+
+    const upd = {
+      $set:   {mature: (next_t ? new Date (next_t) : Queue.now ())},
+      $unset: {reserved: ''}
+    };
+
+    const opts = {};
+
+    this._col.updateOne (q, upd, opts, (err, result) => {
+      debug ('rollback: updateOne (%j, %j, %j) => (%j, %j)', q, upd, opts, err, result);
+      if (err) return callback (err);
+      callback (null, result && (result.modifiedCount == 1));
+    });
+  }
+
+
+  //////////////////////////////////
+  // queue size of non-mature elements only
+  totalSize (callback) {
+    const cursor = this._col.aggregate ([
+      {$match: {
+        qcnt: {$gt: 0}
+      }},
+      {$group:{_id:'t', v: {$sum: '$qcnt'}}}
+    ]);
+
+    cursor.toArray ((err, res) => {
+      debug ('calculating schedSize: aggregation pipeline returns %o', res);
+      if (err) return callback (err);
+      if (res.length == 0) return callback (null, 0);
+      callback (null, res[0].v);
+    });
+  }
+
+
+  //////////////////////////////////
+  // queue size NOT including non-mature elements
+  size (callback) {
+    const cursor = this._col.aggregate ([
+      {$match: {
+        mature: {$lte: Queue.now ()},
+        qcnt: {$gt: 0}
+      }},
+      {$group:{_id:'t', v: {$sum: '$qcnt'}}}
+    ]);
+
+    cursor.toArray ((err, res) => {
+      debug ('calculating schedSize: aggregation pipeline returns %o', res);
+      if (err) return callback (err);
+      if (res.length == 0) return callback (null, 0);
+      callback (null, res[0].v);
+    });
+  }
+
+
+  //////////////////////////////////
+  // queue size of non-mature elements only
+  schedSize (callback) {
+    const cursor = this._col.aggregate ([
+      {$match: {
+        mature: {$gt: Queue.now ()},
+        reserved: {$exists: false},
+        qcnt: {$gt: 0}
+      }},
+      {$group:{_id:'t', v: {$sum: '$qcnt'}}}
+    ]);
+
+    cursor.toArray ((err, res) => {
+      debug ('calculating schedSize: aggregation pipeline returns %o', res);
+      if (err) return callback (err);
+      if (res.length == 0) return callback (null, 0);
+      callback (null, res[0].v);
+    });
+  }
+
+
+  //////////////////////////////////
+  // queue size of reserved elements only
+  resvSize (callback) {
+    const cursor = this._col.aggregate ([
+      {$match: {
+        mature: {$gt: Queue.now ()},
+        reserved: {$exists: true},
+        qcnt: {$gt: 0}
+      }},
+      {$group:{_id:'t', v: {$sum: '$qcnt'}}}
+    ]);
+
+    cursor.toArray ((err, res) => {
+      debug ('calculating schedSize: aggregation pipeline returns %o', res);
+      if (err) return callback (err);
+      if (res.length == 0) return callback (null, 0);
+      callback (null, res[0].v);
+    });
+  }
+
+
+  //////////////////////////////////////////////
+  // remove by id
+  remove (id, callback) {
+    const q =  {
+      _id: id,
+      qcnt: { $eq: 1 },  // allow deletion ONLY if it has just one element in the intraqueue
+      reserved: {$exists: false}
+    };
+
+    const opts = {};
+
+    this._col.deleteOne (q, opts, (err, result) => {
+      debug ('remove: deleteOne (%j, %j) => (%j, %j)', q, opts, err, result);
+      if (err) return callback (err);
+      callback (null, result && (result.deletedCount == 1));
+    });
+  }
+
+
+  /////////////////////////////////////////
+  // get element from queue
+  next_t (callback) {
+    this._col
+    .find ()
+    .limit(1)
+    .sort ({mature:1})
+    .project ({mature:1})
+    .next ((err, result) => {
+      if (err) return callback (err);
+      callback (null, result && result.mature);
+    });
+  }
+
+
+  ///////////////////////////////////////////////////////////////////////////////
+  // private parts
+
+  //////////////////////////////////////////////////////////////////
+  // create needed indexes for O(1) functioning
+  ensureIndexes (cb) {
+    async.series ([
+      cb => this._col.createIndex ({mature : 1, qcnt: 1}, cb),
+    ], cb);
+  }
+}
+
+
+class Factory extends QFactory_MongoDB_defaults {
+  constructor (opts, mongo_conn) {
+    super (opts);
+    this._mongo_conn = mongo_conn;
+    this._db = mongo_conn.db();
+  }
+
+  queue (name, opts) {
+    var full_opts = {};
+    _.merge(full_opts, this._opts, opts);
+    return new IntraOrderedQueue (name, this, full_opts, opts);
+  }
+
+  close (cb) {
+    super.close (() => {
+      if (this._mongo_conn) {
+        this._mongo_conn.close ();
+        this._mongo_conn = null;
+      }
+
+      if (cb) return cb ();
+    });
+  }
+
+  type () {
+    return IntraOrderedQueue.Type ();
+  }
+
+  capabilities () {
+    return {
+      sched:    true,
+      reserve:  true,
+      pipeline: false,
+      tape:     false,
+      remove:   false
+    };
+  }
+}
+
+function creator (opts, cb) {
+  var _opts = opts || {};
+  var m_url = _opts.url || 'mongodb://localhost:27017/keuss';
+
+  MongoClient.connect (m_url, { useNewUrlParser: true }, (err, cl) => {
+    if (err) return cb (err);
+    var F = new Factory (_opts, cl);
+    F.async_init (err => cb (null, F));
+  });
+}
+
+module.exports = creator;
+
+
+
+
+

package/docker-compose.yaml

@@ -0,0 +1,18 @@
+version: '3'
+
+services:
+  mongo:
+    image: mongo:4.4.2-bionic
+    command: --logpath /dev/null
+    restart: on-failure
+    ports:
+      - 27017:27017
+
+  redis:
+    image: redis:alpine
+    ports:
+     - 6379:6379
+
+networks:
+  default:
+

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "keuss",
-  "version": "1.7.1",
+  "version": "1.7.3",
   "keywords": [
     "queue",
     "persistent",
@@ -24,26 +24,26 @@
     "name": "Jose Luis Martinez Juan",
     "email": "pep.martinez@gmail.com"
   },
-  "license": "GPL-3.0",
+  "license": "MIT",
   "dependencies": {
     "@nodebb/mubsub": "~1.8.0",
-    "async": "~3.2.3",
+    "async": "~3.2.4",
     "async-lock": "~1.3.1",
     "debug": "~4.3.4",
     "ioredis": "~5.0.4",
     "lodash": "~4.17.21",
     "mitt": "~3.0.0",
     "mongodb": "~4.5.0",
-    "uuid": "~8.3.2",
-    "why-is-node-running": "^2.2.2"
+    "uuid": "~8.3.2"
   },
   "devDependencies": {
     "chance": "~1.1.8",
-    "mocha": "~9.2.2",
-    "should": "~13.2.3"
+    "mocha": "~10.2.0",
+    "should": "~13.2.3",
+    "why-is-node-running": "^2.2.2"
   },
   "scripts": {
-    "test": "mocha --reporter spec --check-leaks --no-timeouts --exit test/",
-    "test-with-coverage": "nyc --reporter=html -- mocha --reporter spec --check-leaks --no-timeouts --exit test/"
+    "test": "docker compose up -d; sleep 5; mocha --reporter spec --check-leaks --no-timeouts --exit test/ ; docker compose down",
+    "test-with-coverage": "docker compose up -d; sleep 5; nyc --reporter=html -- mocha --reporter spec --check-leaks --no-timeouts --exit ; test/docker compose down"
   }
 }

package/playground/irc.js

@@ -0,0 +1,53 @@
+const async = require ('async');
+const MQ =    require ('../backends/intraorder');
+
+// initialize factory
+MQ ({
+  url: 'mongodb://localhost/keuss'
+}, (err, factory) => {
+  if (err) return console.error(err);
+
+  const q = factory.queue ('test_queue', {});
+/*
+  async.waterfall ([
+    cb =>            q.push ({iid: 123, elem: 1, headline: 'something something', tags: {a: 1, b: 2}}, cb),  
+    (item_id, cb) => q.push ({iid: 123, elem: 2, headline: 'other other', tags: {a: 3, b: 4}}, cb),  
+    (item_id, cb) => q.pop ('consumer-1', {reserve: true}, cb), 
+    (item, cb) =>    q.ko (item, new Date().getTime () + 1500, cb), 
+    (item_id, cb) => q.pop ('consumer-1', {reserve: true}, cb), 
+    (item, cb) =>    {console.log ('%s: got %o', new Date().toISOString (), item.payload); q.ok (item, cb); },
+    (item_id, cb) => q.pop ('consumer-1', {reserve: true}, cb), 
+    (item, cb) =>    q.ko (item, new Date().getTime () + 1500, cb), 
+    (item_id, cb) => q.pop ('consumer-1', {reserve: true}, cb), 
+    (item, cb) =>    {console.log ('%s: got %o', new Date().toISOString (), item.payload); q.ok (item, cb); },
+    (i, cb) =>       setTimeout (cb, 100),
+    cb =>            q.status (cb),
+  ], (err, res) => {
+    if (err) console.error (err);
+    console.log (res)
+    factory.close ();
+  });
+
+*/
+
+
+async.series ([
+  cb => q.status (cb),
+  cb => q.push ({iid: 123, elem: 1, headline: 'something something', tags: {a: 1, b: 2}}, cb), 
+  cb => q.status (cb), 
+  cb => q.push ({iid: 123, elem: 2, headline: 'other other', tags: {a: 3, b: 4}}, cb),  
+  cb => q.status (cb),
+  cb => q.pop ('consumer-1', cb), 
+  cb => q.status (cb),
+  cb => q.pop ('consumer-1', cb), 
+  cb => q.status (cb),
+  cb => setTimeout (cb, 100),
+  cb => q.status (cb),
+], (err, res) => {
+  if (err) console.error (err);
+  factory.close ();
+  res.forEach ((v, i) => console.log ('%d:', i, v ));
+});
+
+});
+

package/test/backends_deadletter.js

@@ -77,12 +77,13 @@ function release_mq_factory (q, factory, cb) {
 
 /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 [
-  {label: 'Simple MongoDB',     mq: require ('../backends/mongo')},
-  {label: 'Pipelined MongoDB',  mq: require ('../backends/pl-mongo')},
-  {label: 'Tape MongoDB',       mq: require ('../backends/ps-mongo')},
-  {label: 'Stream MongoDB',     mq: require ('../backends/stream-mongo')},
-  {label: 'Redis OrderedQueue', mq: require ('../backends/redis-oq')},
-  {label: 'MongoDB SafeBucket', mq: require ('../backends/bucket-mongo-safe')}
+//  {label: 'Simple MongoDB',     mq: require ('../backends/mongo')},
+//  {label: 'Pipelined MongoDB',  mq: require ('../backends/pl-mongo')},
+//  {label: 'Tape MongoDB',       mq: require ('../backends/ps-mongo')},
+//  {label: 'Stream MongoDB',     mq: require ('../backends/stream-mongo')},
+//  {label: 'Redis OrderedQueue', mq: require ('../backends/redis-oq')},
+//  {label: 'MongoDB SafeBucket', mq: require ('../backends/bucket-mongo-safe')},
+  {label: 'Mongo IntraOrder',   mq: require ('../backends/intraorder')},
 ].forEach(function (MQ_item) {
   describe('rollback and deadletters with ' + MQ_item.label + ' queue backend', function () {
     const MQ = MQ_item.mq;

package/test/backends_payload.js

@@ -10,13 +10,14 @@ var MongoClient = require ('mongodb').MongoClient;
 var factory = null;
 
 [
-  {label: 'Simple MongoDB',         mq: require ('../backends/mongo')},
-  {label: 'Pipelined MongoDB',      mq: require ('../backends/pl-mongo')},
-  {label: 'Tape MongoDB',           mq: require ('../backends/ps-mongo')},
-  {label: 'Stream MongoDB',         mq: require ('../backends/stream-mongo')},
-  {label: 'Safe MongoDB Buckets',   mq: require ('../backends/bucket-mongo-safe')},
-  {label: 'Redis List',             mq: require ('../backends/redis-list')},
-  {label: 'Redis OrderedQueue',     mq: require ('../backends/redis-oq')},
+  {label: 'Simple MongoDB',       mq: require ('../backends/mongo')},
+  {label: 'Pipelined MongoDB',    mq: require ('../backends/pl-mongo')},
+  {label: 'Tape MongoDB',         mq: require ('../backends/ps-mongo')},
+  {label: 'Stream MongoDB',       mq: require ('../backends/stream-mongo')},
+  {label: 'Safe MongoDB Buckets', mq: require ('../backends/bucket-mongo-safe')},
+  {label: 'Redis List',           mq: require ('../backends/redis-list')},
+  {label: 'Redis OrderedQueue',   mq: require ('../backends/redis-oq')},
+  {label: 'Mongo IntraOrder',     mq: require ('../backends/intraorder')},
 ].forEach(function (MQ_item) {
   describe('payload aspects on ' + MQ_item.label + ' queue backend, round 1', function () {
     var MQ = MQ_item.mq;