keuss 2.0.7 → 2.2.0

package/backends/bucket-mongo-safe.js

@@ -1,17 +1,17 @@
-var async =     require ('async');
-var _ =         require ('lodash');
-var AsyncLock = require ('async-lock');
+const async =     require ('async');
+const _ =         require ('lodash');
+const AsyncLock = require ('async-lock');
 
 
-var debug = require('debug')('keuss:backend:BucketMongoSafe');
+const debug = require('debug')('keuss:backend:BucketMongoSafe');
 
-var MongoClient = require ('mongodb').MongoClient;
-var mongo =       require ('mongodb');
+const MongoClient = require ('mongodb').MongoClient;
+const mongo =       require ('mongodb');
 
-var Queue =                     require ('../Queue');
-var QFactory_MongoDB_defaults = require ('../QFactory-MongoDB-defaults');
+const Queue =                     require ('../Queue');
+const QFactory_MongoDB_defaults = require ('../QFactory-MongoDB-defaults');
 
-var State = {
+const State = {
   Available: 1,
   Reserved:  2,
   Committed: 3,
@@ -79,9 +79,9 @@ class Bucket {
 
     debug ('Bucket:got_element: look for next available elem starting at pos %d', this._last_b_idx);
 
-    for (var i = this._last_b_idx; i <  this._b_states.length; i++) {
+    for (let i = this._last_b_idx; i <  this._b_states.length; i++) {
       if (this._b_states[i] == State.Available) {
-        var elem = {};
+        const elem = {};
         if (this._b[i].__p) {
           // add-headers: array contains both payload and headers, breaks backwards compat
           elem.payload = this._b[i].__p;
@@ -121,16 +121,11 @@ class Bucket {
 
   /////////////////////////////////////////
   _flush_delete_bucket (cb) {
-    var q = {_id: this._id};
+    const q = {_id: this._id};
     debug ('Bucket: deleting whole bucket %o', q);
 
-    this._col.deleteOne (q, (err, res) => {
-      if (err) return cb({
-        err: 'Bucket flush: mongodb error',
-        e: err,
-        q: q
-      });
-
+    this._col.deleteOne (q)
+    .then (res => {
       if ((res && res.deletedCount) != 1) {
         return cb({
           err: 'Bucket flush: exactly one must be deleted',
@@ -141,23 +136,30 @@ class Bucket {
 
       debug ('Bucket: deleted whole bucket %o', q);
       cb (null, null);
-    });
+    })
+    .catch (err => {
+      cb({
+        err: 'Bucket flush: mongodb error',
+        e: err,
+        q: q
+      });
+    })
   }
 
 
   /////////////////////////////////////////
   _flush_bucket (cb) {
-    var upd = {
+    const upd = {
       $set: {}
     };
 
-    var has_update = false;
-    var committed_pos = [];
-    var mature = null;
-    var timed_out = false;
-    var is_reject = false;
+    let has_update = false;
+    const committed_pos = [];
+    let mature = null;
+    let timed_out = false;
+    let is_reject = false;
 
-    var time_left = this._mature_t - new Date().getTime();
+    const time_left = this._mature_t - new Date().getTime();
     debug ('time left is %d, grace is %d', time_left, this._reject_timeout_grace);
 
     if (time_left < this._reject_timeout_grace) {
@@ -189,13 +191,13 @@ class Bucket {
     ) {
       debug ('Bucket: flushing entire bucket for retry (%o)', this._b_counts);
       is_reject = true;
-      var tries = this._tries || 0;
+      const tries = this._tries || 0;
 
       if (this._rollback_next_t) {
         mature = new Date (this._rollback_next_t);
       }
       else {
-        var delta = this._reject_delta_factor * tries + this._reject_delta_base;
+        const delta = this._reject_delta_factor * tries + this._reject_delta_base;
         mature = new Date (new Date().getTime () + delta)
       }
 
@@ -205,17 +207,12 @@ class Bucket {
     }
 
     if (has_update) {
-      var q = {_id: this._id};
+      const q = {_id: this._id};
 
       debug ('update on bucket: q is %o, upd is %o', q, upd);
 
-      this._col.updateOne (q, upd, (err, res) => {
-        if (err) return cb({
-          err: 'Bucket flush: mongodb error',
-          e: err,
-          q: q
-        });
-
+      this._col.updateOne (q, upd)
+      .then (res => {
         if (res && (res.modifiedCount != 1)) return cb({
           err: 'Bucket flush: exactly one must be updated',
           e: err,
@@ -234,7 +231,14 @@ class Bucket {
 
         debug ('Bucket: flushed ok, states are %o (%o)', this._b_states, this._b_counts);
         cb (null, is_reject ? null : this._b_counts);
-      });
+      })
+      .catch (err => {
+         cb({
+          err: 'Bucket flush: mongodb error',
+          e: err,
+          q: q
+        });
+      })
     }
     else {
       debug ('Bucket: nothing to flush');
@@ -279,11 +283,11 @@ class BucketSet {
 
   ///////////////////////////////
   _read_bucket (cb) {
-    var query = {
+    const query = {
       mature: {$lte: Queue.now ()}
     };
 
-    var update = {
+    const update = {
       $set: {
         mature: Queue.nowPlusSecs (this._reserve_delay),
         reserved: new Date ()
@@ -291,21 +295,21 @@ class BucketSet {
       $inc: {tries: 1}
     };
 
-    var opts = {
+    const opts = {
       sort: {mature : 1},
-      returnDocument: 'after'
+      returnDocument: 'after',
+      includeResultMetadata: true
     };
 
     debug ('reading a new bucket');
 
-    this._col.findOneAndUpdate (query, update, opts, (err, result) => {
-      if (err) return cb (err);
-
-      var val = result && result.value;
+    this._col.findOneAndUpdate (query, update, opts)
+    .then (res => {
+      const val = res && res.value;
 
       if (val) {
         debug ('read a bucket %s with %d elems', val._id.toString(), val.n);
-        var bcket = new Bucket (val, this._col, this._q, this._opts);
+        const bcket = new Bucket (val, this._col, this._q, this._opts);
         this._buckets[bcket.id()] = bcket;
 
         // is already exhausted?
@@ -323,7 +327,8 @@ class BucketSet {
         debug ('no buckets, coll empty');
         cb ();
       }
-    });
+    })
+    .catch (cb)
   }
 
 
@@ -377,7 +382,7 @@ class BucketSet {
         return cb ();  // coll empty or not mature
       }
 
-      var elem = this._active_bucket.get_element (is_reserve);
+      const elem = this._active_bucket.get_element (is_reserve);
 
       if (!elem) {
         debug ('BucketSet:getElement: active bucket exhausted, get another');
@@ -406,11 +411,11 @@ class BucketSet {
 
   /////////////////////////////
   commit_element (id, cb) {
-    var aid = id.split (':');
-    var bucket_id = aid[0];
-    var bucket_idx = parseInt(aid[1]);
+    const aid = id.split (':');
+    const bucket_id = aid[0];
+    const bucket_idx = parseInt(aid[1]);
 
-    var bucket = this._buckets[bucket_id];
+    const bucket = this._buckets[bucket_id];
     if (!bucket) return cb (null, false);
     if (bucket._b_states[bucket_idx] != State.Reserved) return cb (null, false);
 
@@ -425,11 +430,11 @@ class BucketSet {
 
   /////////////////////////////
   rollback_element (id, next_t, cb) {
-    var aid = id.split (':');
-    var bucket_id = aid[0];
-    var bucket_idx = parseInt(aid[1]);
+    const aid = id.split (':');
+    const bucket_id = aid[0];
+    const bucket_idx = parseInt(aid[1]);
 
-    var bucket = this._buckets[bucket_id];
+    const bucket = this._buckets[bucket_id];
     if (!bucket) return cb (null, false);
     if (bucket._b_states[bucket_idx] != State.Reserved) return cb (null, false);
 
@@ -451,7 +456,7 @@ class BucketSet {
 
     debug ('BucketSet:_flush_state_changes: tick');
 
-    var tasks = {};
+    const tasks = {};
     _.each (this._buckets, (bucket, id) => {
       tasks[id] = (cb) => {
         debug ('BucketSet:_flush_state_changes: flushing bucket %s', bucket.id());
@@ -517,7 +522,6 @@ class BucketSet {
 }
 
 
-
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
 class BucketMongoSafeQueue extends Queue {
   /*
@@ -532,13 +536,12 @@ class BucketMongoSafeQueue extends Queue {
   */
   //////////////////////////////////////////////
   constructor (name, factory, opts, orig_opts) {
-  //////////////////////////////////////////////
     super (name, factory, opts, orig_opts);
 
     this._col = factory._db.collection (name);
 
     this._insert_bucket = {
-      _id: new mongo.ObjectID (),
+      _id: new mongo.ObjectId (),
       b: []
     };
 
@@ -553,21 +556,18 @@ class BucketMongoSafeQueue extends Queue {
 
   /////////////////////////////////////////
   static Type () {
-  /////////////////////////////////////////
     return 'mongo:bucket-safe';
   }
 
   /////////////////////////////////////////
   type () {
-  /////////////////////////////////////////
     return 'mongo:bucket-safe';
   }
 
 
   /////////////////////////////////////////
   // add element to queue
-  insert (entry, callback) {
-  /////////////////////////////////////////
+  insert (entry, cb) {
     if (!this._insert_bucket.mature) this._insert_bucket.mature = entry.mature;
     else if (this._insert_bucket.mature.getTime () < entry.mature.getTime ()) this._insert_bucket.mature = entry.mature;
 
@@ -576,7 +576,7 @@ class BucketMongoSafeQueue extends Queue {
       __h: (entry.hdrs || {})
     });
 
-    var id = this._insert_bucket._id.toString () + ':' + (this._insert_bucket.b.length - 1);
+    const id = this._insert_bucket._id.toString () + ':' + (this._insert_bucket.b.length - 1);
     debug ('added to bucket, %s', id);
 
     if (this._insert_bucket.b.length >= this._bucket_max_size) {
@@ -585,7 +585,7 @@ class BucketMongoSafeQueue extends Queue {
 
       debug ('cancelled periodic_flush');
 
-      this._flush_bucket (callback);
+      this._flush_bucket (cb);
     }
     else {
       if (this._insert_bucket.b.length == 1) {
@@ -593,91 +593,95 @@ class BucketMongoSafeQueue extends Queue {
         this._set_periodic_flush ();
       }
 
-      setImmediate (() => callback (null, id));
+      setImmediate (() => cb (null, id));
     }
   }
 
 
   /////////////////////////////////////////
   // get element from queue
-  get (callback) {
+  get (cb) {
     this._read_bucket.get_element ((err, elem) => {
-      if (err) return callback (err);
+      if (err) return cb (err);
       if (elem && elem.payload._bsontype == 'Binary') elem.payload = elem.payload.buffer;
       if (elem && elem.tries > 0) elem.tries--;
-      callback (null, elem);
+      cb (null, elem);
     });
   }
 
 
   /////////////////////////////////////////
   // reserve element from queue
-  reserve (callback) {
+  reserve (cb) {
     this._read_bucket.reserve_element ((err, elem) => {
-      if (err) return callback (err);
+      if (err) return cb (err);
       if (elem && elem.payload._bsontype == 'Binary') elem.payload = elem.payload.buffer;
       if (elem && elem.tries > 0) elem.tries--;
-      callback (null, elem);
+      cb (null, elem);
     });
   }
 
 
   /////////////////////////////////////////
   // commit a reserved element from queue
-  commit (id, callback) {
+  commit (id, cb) {
     this._read_bucket.commit_element (id, (err, elem) => {
-      if (err) return callback (err);
-      callback (null, elem);
+      if (err) return cb (err);
+      cb (null, elem);
     });
   }
 
 
   /////////////////////////////////////////
   // rollback a reserved element from queue
-  rollback (id, next_t, callback) {
+  rollback (id, next_t, cb) {
     this._read_bucket.rollback_element (id, next_t, (err, elem) => {
-      if (err) return callback (err);
-      callback (null, elem);
+      if (err) return cb (err);
+      cb (null, elem);
     });
   }
 
 
   //////////////////////////////////
   // queue size including non-mature elements
-  totalSize (callback) {
+  totalSize (cb) {
     const cursor = this._col.aggregate ([
       {$group:{_id:'t', v: {$sum: '$n'}}}
     ]);
 
-    cursor.toArray ((err, res) => {
+    cursor
+    .toArray ()
+    .then (res => {
       debug ('calculating totalSize: aggregation pipeline returns %o', res);
-      if (err) return callback (err);
-      if (res.length == 0) return callback (null, 0);
-      callback (null, res[0].v);
-    });
+      if (res.length == 0) return cb (null, 0);
+      cb (null, res[0].v);
+    })
+    .catch (cb);
   }
 
 
   //////////////////////////////////
   // queue size NOT including non-mature elements
-  size (callback) {
+  size (cb) {
     const cursor = this._col.aggregate ([
       {$match: {mature: {$lte: Queue.now ()}}},
       {$group:{_id:'t', v: {$sum: '$n'}}}
     ]);
 
-    cursor.toArray ((err, res) => {
+    cursor
+    .toArray ()
+    .then (res => {
       debug ('calculating size: aggregation pipeline returns %o', res);
-      if (err) return callback (err);
-      if (res.length == 0) return callback (null, 0);
-      callback (null, res[0].v);
-    });
+      if (res.length == 0) return cb (null, 0);
+      cb (null, res[0].v);
+    })
+    .catch (cb);
   }
 
 
   //////////////////////////////////
   // queue size of non-mature elements only
-  schedSize (callback) {
+  schedSize (cb) {
     const cursor = this._col.aggregate ([
       {$match: {
         mature: {$gt: Queue.now ()},
@@ -686,18 +690,20 @@ class BucketMongoSafeQueue extends Queue {
       {$group:{_id:'t', v: {$sum: '$n'}}}
     ]);
 
-    cursor.toArray ((err, res) => {
+    cursor
+    .toArray ()
+    .then (res => {
       debug ('calculating schedSize: aggregation pipeline returns %o', res);
-      if (err) return callback (err);
-      if (res.length == 0) return callback (null, 0);
-      callback (null, res[0].v);
-    });
+      if (res.length == 0) return cb (null, 0);
+      cb (null, res[0].v);
+    })
+    .catch (cb);
   }
 
 
   //////////////////////////////////
   // queue size of non-mature elements only
-  resvSize (callback) {
+  resvSize (cb) {
     const cursor = this._col.aggregate ([
       {$match: {
         mature: {$gt: Queue.now ()},
@@ -706,50 +712,55 @@ class BucketMongoSafeQueue extends Queue {
       {$group:{_id:'t', v: {$sum: '$n'}}}
     ]);
 
-    cursor.toArray ((err, res) => {
+    cursor
+    .toArray ()
+    .then (res => {
       debug ('calculating resvSize: aggregation pipeline returns %o', res);
-      if (err) return callback (err);
-      if (res.length == 0) return callback (null, 0);
-      callback (null, res[0].v);
-    });
+      if (res.length == 0) return cb (null, 0);
+      cb (null, res[0].v);
+    })
+    .catch (cb);
   }
 
 
   /////////////////////////////////////////
   // get element from queue
-  next_t (callback) {
+  next_t (cb) {
     this._col.findOne ({}, {
       sort:       {mature:1},
       projection: {mature: 1}
-    },
-    (err, result) => {
-      if (err) return callback (err);
-      debug ('obtaining next_t: got %o', result && result.mature);
-      callback (null, result && result.mature);
-    });
+    })
+    .then (res => {
+      debug ('obtaining next_t: got %o', res && res.mature);
+      cb (null, res && res.mature);
+    })
+    .catch (cb);
   }
 
 
   //////////////////////////////////////////////
   // remove by id
-  remove (id, callback) {
+  remove (id, cb) {
+    let bucket_id;
+    const aid = id.split (':');
+
     try {
-      var aid = id.split (':');
-      var bucket_id =  new mongo.ObjectID (aid[0]);
-      var bucket_idx = parseInt(aid[1]);
+      bucket_id =  new mongo.ObjectId (aid[0]);
     }
     catch (e) {
-      return callback ('id [' + id + '] can not be used as remove id: ' + e);
+      return cb ('id [' + id + '] can not be used as remove id: ' + e);
     }
+     
+    const bucket_idx = parseInt(aid[1]);
 
-    var upd = {
+    const upd = {
       $set: {},
       $inc: {n: -1}
     };
 
     upd.$set['b.' + bucket_idx] = null;
 
-    var q = {
+    const q = {
       _id: bucket_id,
       reserved: {$exists: false},
     };
@@ -757,10 +768,9 @@ class BucketMongoSafeQueue extends Queue {
     q['b.' + bucket_idx] = {$type: 3}; // b.N is an object
 
     debug ('update on bucket: q is %o, upd is %o', q, upd);
-    this._col.updateOne (q, upd, (err, result) => {
-      if (err) return callback (err);
-      callback (null, result && (result.modifiedCount == 1));
-    });
+    this._col.updateOne (q, upd)
+    .then (res => cb (null, res && (res.modifiedCount == 1)))
+    .catch (cb);
   }
 
 
@@ -814,7 +824,9 @@ class BucketMongoSafeQueue extends Queue {
 
   /////////////////////////////////////////
   _ensureIndexes (cb) {
-    this._col.createIndex ({mature : 1}, err => cb (err, this));
+    this._col.createIndex ({mature : 1})
+    .then (res => cb (null, this))
+    .catch (cb);
   }
 
 
@@ -846,44 +858,44 @@ class BucketMongoSafeQueue extends Queue {
 
 
   /////////////////////////////////////////
-  _flush_bucket (callback) {
+  _flush_bucket (cb) {
   /////////////////////////////////////////
-    var bucket = this._insert_bucket;
+    const bucket = this._insert_bucket;
     bucket.n = bucket.b.length;
 
     this._insert_bucket = {
-      _id: new mongo.ObjectID (),
+      _id: new mongo.ObjectId (),
       b: []
     };
 
     debug ('flushing bucket %s with %d elems', bucket._id.toString(), bucket.b.length);
 
-    this._col.insertOne (bucket, {}, (err, result) => {
-      if (err) return callback (err);
-
+    this._col.insertOne (bucket, {})
+    .then (res => {
       this._signal_insertion_own (bucket.mature);
-      callback (null, bucket);
-    });
+      cb (null, bucket);
+    })
+    .catch (cb);
   }
 
 
   /////////////////////////////////////////
   // get element from queue
-  _get_bucket (callback) {
+  _get_bucket (cb) {
   /////////////////////////////////////////
     debug ('need to read a bucket');
 
-    this._col.findOneAndDelete ({}, {sort: {_id : 1}}, (err, result) => {
-      if (err) return callback (err);
-
-      var val = result && result.value;
+    this._col.findOneAndDelete ({}, {sort: {_id : 1}, includeResultMetadata: true})
+    .then (res => {
+      const val = res && res.value;
 
       if (val) {
         debug ('read a bucket %s with %d elems', val._id.toString(), val.n);
       }
 
-      callback (null, val);
-    });
+      cb (null, val);
+    })
+    .catch (cb);
   }
 
 
@@ -901,6 +913,7 @@ class BucketMongoSafeQueue extends Queue {
 };
 
 
+////////////////////////////////////////////////////////////////////////////////
 class Factory extends QFactory_MongoDB_defaults {
   constructor (opts, mongo_conn) {
     super (opts);
@@ -908,6 +921,7 @@ class Factory extends QFactory_MongoDB_defaults {
     this._db = mongo_conn.db();
   }
 
+  ////////////////////////////////////////
   queue (name, opts, cb) {
     if (!cb) {
       cb = opts;
@@ -921,6 +935,7 @@ class Factory extends QFactory_MongoDB_defaults {
     q._ensureIndexes (cb);
   }
 
+  ////////////////////////////////////////
   close (cb) {
     super.close (() => {
       if (this._mongo_conn) {
@@ -932,30 +947,36 @@ class Factory extends QFactory_MongoDB_defaults {
     });
   }
 
+  ////////////////////////////////////////
   type () {
     return BucketMongoSafeQueue.Type ();
   }
 
+  ////////////////////////////////////////
   capabilities () {
     return {
       sched:    true,
       reserve:  true,
       pipeline: false,
       tape:     false,
-      remove:   true
+      remove:   true,
+      id:       false,
     };
   }
 }
 
+
+////////////////////////////////////////
 function creator (opts, cb) {
-  var _opts = opts || {};
-  var m_url = _opts.url || 'mongodb://localhost:27017/keuss';
+  const _opts = opts || {};
+  const m_url = _opts.url || 'mongodb://localhost:27017/keuss';
 
-  MongoClient.connect (m_url, { useNewUrlParser: true }, (err, cl) => {
-    if (err) return cb (err);
-    var F = new Factory (_opts, cl);
+  MongoClient.connect (m_url)
+  .then (cl => {
+    const F = new Factory (_opts, cl);
     F.async_init (err => cb (null, F));
-  });
+  })
+  .catch (cb);
 }
 
 module.exports = creator;