keuss 2.0.7 → 2.2.0

package/backends/stream-mongo.js

@@ -7,10 +7,10 @@ const mongo =       require ('mongodb');
 const Queue =                     require ('../Queue');
 const QFactory_MongoDB_defaults = require ('../QFactory-MongoDB-defaults');
 
-var debug = require('debug')('keuss:Queue:StreamMongo');
+const debug = require('debug')('keuss:Queue:StreamMongo');
 
-class StreamMongoQueue extends Queue {
 
+class StreamMongoQueue extends Queue {
   //////////////////////////////////////////////
   constructor (name, factory, opts, orig_opts) {
     super (name, factory, opts, orig_opts);
@@ -48,7 +48,7 @@ class StreamMongoQueue extends Queue {
 
   /////////////////////////////////////////
   // add element to queue
-  insert (entry, callback) {
+  insert (entry, cb) {
     const mtr = entry.mature;
     const tr =  entry.tries;
 
@@ -58,17 +58,18 @@ class StreamMongoQueue extends Queue {
     
     entry.t = new Date();
 
-    this._col.insertOne (entry, {}, (err, result) => {
-      if (err) return callback (err);
-      callback (null, result.insertedId);
+    this._col.insertOne (entry, {})
+    .then (res => {
+      cb (null, res.insertedId);
       this._groups_vector.forEach (i => this._stats.incr (`stream.${i}.put`));
-    });
+    })
+    .catch (cb);
   }
 
 
   /////////////////////////////////////////
   // get element from queue
-  get (callback) {
+  get (cb) {
     const gid = this._gid;
     const q = {};
 
@@ -82,30 +83,33 @@ class StreamMongoQueue extends Queue {
     updt.$set[`mature.${gid}`] = Queue.nowPlusSecs (100 * this._opts.ttl);
 
     const opts = {
-      sort: {}
+      sort: {},
+      includeResultMetadata: true
     };
+
     opts.sort[`mature.${gid}`] = 1;
 
     debug ('get() with q %O, upd %O, opts %o', q, updt, opts);
 
-    this._col.findOneAndUpdate (q, updt, opts, (err, result) => {
-      if (err) return callback (err);
-      const v = result && result.value;
-      if (!v) return callback ();
+    this._col.findOneAndUpdate (q, updt, opts)
+    .then (res => {
+      const v = res && res.value;
+      if (!v) return cb ();
       if (v.payload._bsontype == 'Binary') v.payload = v.payload.buffer;
       v.mature = v.mature[gid];
       v.tries = v.tries[gid];
       delete v.processed;
       delete v.t;
-      callback (null, v);
+      cb (null, v);
       this._stats.incr (`stream.${gid}.get`);
-    });
+    })
+    .catch (cb);
   }
 
 
   //////////////////////////////////
   // reserve element: call cb (err, pl) where pl has an id
-  reserve (callback) {
+  reserve (cb) {
     const gid = this._gid;
     const delay = this._opts.reserve_delay || 120;
     
@@ -124,41 +128,44 @@ class StreamMongoQueue extends Queue {
 
     const opts = {
       sort: {},
-      returnDocument: 'before'
+      returnDocument: 'before',
+      includeResultMetadata: true
     };
+
     opts.sort[`mature.${gid}`] = 1;
 
     debug ('reserve() with q %O, upd %O, opts %o', q, updt, opts);
 
-    this._col.findOneAndUpdate (q, updt, opts, (err, result) => {
-      if (err) return callback (err);
-      const v = result && result.value;
-      if (!v) return callback ();
+    this._col.findOneAndUpdate (q, updt, opts)
+    .then (res => {
+      const v = res && res.value;
+      if (!v) return cb ();
       if (v.payload._bsontype == 'Binary') v.payload = v.payload.buffer;
       v.mature = v.mature[gid];
       v.tries = v.tries[gid];
       delete v.processed;
       delete v.t;
-      callback (null, v);
+      cb (null, v);
       this._stats.incr (`stream.${gid}.reserve`);
-    });
+    })
+    .catch (cb);
   }
 
 
   //////////////////////////////////
   // commit previous reserve, by p.id
-  commit (id, callback) {
+  commit (id, cb) {
     const gid = this._gid;
     let q;
 
     try {
       q =  {
-        _id: (_.isString(id) ? new mongo.ObjectID (id) : id),
+        _id: (_.isString(id) ? new mongo.ObjectId (id) : id),
       };
       q[`reserved.${gid}`] = {$exists: true};
     }
     catch (e) {
-      return callback ('id [' + id + '] can not be used as rollback id: ' + e);
+      return cb ('id [' + id + '] can not be used as rollback id: ' + e);
     }
 
     const updt = {
@@ -173,33 +180,34 @@ class StreamMongoQueue extends Queue {
 
     debug ('commit() with q %O, upd %O, opts %o', q, updt, opts);
 
-    this._col.updateOne (q, updt, opts, (err, result) => {
-      if (err) return callback (err);
-      callback (null, result && (result.modifiedCount == 1));
+    this._col.updateOne (q, updt, opts)
+    .then (res => {
+      cb (null, res && (res.modifiedCount == 1));
       this._stats.incr (`stream.${gid}.commit`);
-    });
+    })
+    .catch (cb);
   }
 
 
   //////////////////////////////////
   // rollback previous reserve, by p.id
-  rollback (id, next_t, callback) {
+  rollback (id, next_t, cb) {
     const gid = this._gid;
     let q;
 
     if (_.isFunction (next_t)) {
-      callback = next_t;
+      cb = next_t;
       next_t = null;
     }
 
     try {
       q =  {
-        _id: (_.isString(id) ? new mongo.ObjectID (id) : id),
+        _id: (_.isString(id) ? new mongo.ObjectId (id) : id),
       };
       q[`reserved.${gid}`] = {$exists: true};
     }
     catch (e) {
-      return callback ('id [' + id + '] can not be used as rollback id: ' + e);
+      return cb ('id [' + id + '] can not be used as rollback id: ' + e);
     }
 
     const updt = {
@@ -213,30 +221,33 @@ class StreamMongoQueue extends Queue {
 
     debug ('rollback() with q %O, upd %O, opts %o', q, updt, opts);
 
-    this._col.updateOne (q, updt, opts, (err, result) => {
-      if (err) return callback (err);
-      callback (null, result && (result.modifiedCount == 1));
+    this._col.updateOne (q, updt, opts)
+    .then (res => {
+      cb (null, res && (res.modifiedCount == 1));
       this._stats.incr (`stream.${gid}.rollback`);
-    });
+    })
+    .catch (cb);
   }
 
 
   //////////////////////////////////
   // queue size including non-mature elements
-  totalSize (callback, gid) {
+  totalSize (cb, gid) {
     const gr = gid || this._gid;
 
     const q = {};
     q[`processed.${gr}`] = false;
 
     const opts = {};
-    this._col.countDocuments (q, opts, callback);
+    this._col.countDocuments (q, opts)
+    .then (res => cb (null, res))
+    .catch (cb);
   }
 
 
   //////////////////////////////////
   // queue size NOT including non-mature elements
-  size (callback, gid) {
+  size (cb, gid) {
     const gr = gid || this._gid;
 
     const q = {};
@@ -244,13 +255,15 @@ class StreamMongoQueue extends Queue {
     q[`mature.${gr}`] = {$lte: Queue.now()};
 
     const opts = {};
-    this._col.countDocuments (q, opts, callback);
+    this._col.countDocuments (q, opts)
+    .then (res => cb (null, res))
+    .catch (cb);
   }
 
 
   //////////////////////////////////
   // queue size of non-mature elements only
-  schedSize (callback, gid) {
+  schedSize (cb, gid) {
     const gr = gid || this._gid;
 
     const q = {};
@@ -259,13 +272,15 @@ class StreamMongoQueue extends Queue {
     q[`mature.${gr}`] = {$gt: Queue.now()};
 
     const opts = {};
-    this._col.countDocuments (q, opts, callback);
+    this._col.countDocuments (q, opts)
+    .then (res => cb (null, res))
+    .catch (cb);
   }
 
 
   //////////////////////////////////
   // queue size of reserved elements only
-  resvSize (callback, gid) {
+  resvSize (cb, gid) {
     const gr = gid || this._gid;
 
     const q = {};
@@ -274,13 +289,15 @@ class StreamMongoQueue extends Queue {
     q[`mature.${gr}`] = {$gt: Queue.now()};
 
     const opts = {};
-    this._col.countDocuments (q, opts, callback);
+    this._col.countDocuments (q, opts)
+    .then (res => cb (null, res))
+    .catch (cb);
   }
 
 
   /////////////////////////////////////////
   // get element from queue
-  next_t (callback, gid) {
+  next_t (cb, gid) {
     const gr = gid || this._gid;
     
     const q = {};
@@ -294,11 +311,12 @@ class StreamMongoQueue extends Queue {
     .limit(1)
     .sort (sort)
     .project ({mature:1})
-    .next ((err, result) => {
-      if (err) return callback (err);
-      debug ('next_t with git %s: got %o', gr, result);
-      callback (null, result && result.mature && result.mature[gr]);
-    });
+    .next ()
+    .then (res => {
+      debug ('next_t with git %s: got %o', gr, res);
+      cb (null, res && res.mature && res.mature[gr]);
+    })
+    .catch (cb);
   }
 
 
@@ -398,10 +416,14 @@ class StreamMongoQueue extends Queue {
     this._groups_vector.forEach (i => {
       const idx = {};
       idx[`mature.${i}`] = 1;
-      tasks.push (cb => this._col.createIndex (idx, cb));
+      tasks.push (cb => this._col.createIndex (idx).then (res => cb(null, res)).catch (cb));
     });
-    tasks.push (cb => this._col.createIndex ({t: 1}, {expireAfterSeconds: this._opts.ttl}, cb));
-    async.series (tasks, err => cb (err, this));
+
+    tasks.push (cb => this._col.createIndex ({t: 1}, {expireAfterSeconds: this._opts.ttl}).then (res => cb(null, res)).catch (cb));
+    
+    async.series (tasks)
+    .then (() => cb (null, this))
+    .catch (cb);
   }
 }
 
@@ -447,7 +469,8 @@ class Factory extends QFactory_MongoDB_defaults {
       pipeline: false,
       tape:     true,
       remove:   false,
-      stream:   true
+      stream:   true,
+      id:       false,
     };
   }
 }
@@ -456,16 +479,12 @@ function creator (opts, cb) {
   const _opts = opts || {};
   const m_url = _opts.url || 'mongodb://localhost:27017/keuss';
 
-  MongoClient.connect (m_url, { useNewUrlParser: true }, (err, cl) => {
-    if (err) return cb (err);
+  MongoClient.connect (m_url)
+  .then (cl => {
     const F = new Factory (_opts, cl);
     F.async_init (err => cb (null, F));
-  });
+  })
+  .catch (cb);
 }
 
 module.exports = creator;
-
-
-
-
-

package/lib/mubsub/channel.js

@@ -0,0 +1,313 @@
+const EventEmitter = require('events')
+const noop = function () {}
+
+class Channel extends EventEmitter {
+  /**
+   * Channel constructor.
+   *
+   * @param {Connection} connection
+   * @param {String} [name] optional channel/collection name, default is 'mubsub'
+   * @param {Object} [options] optional options
+   *   - `size` max size of the collection in bytes, default is 5mb
+   *   - `max` max amount of documents in the collection
+   *   - `retryInterval` time in ms to wait if no docs found, default is 200ms
+   *   - `recreate` recreate the tailable cursor on error, default is true
+   * @api public
+   */
+  constructor (connection, name, options) {
+    super()
+    options || (options = {})
+    this.options = {
+      recreate: typeof options.recreate === 'boolean' ? options.recreate : true,
+      retryInterval:
+        typeof options.retryInterval === 'number' ? options.retryInterval : 200
+    }
+
+    this.collectionOpts = {
+      capped: true,
+      size: options.size || 1024 * 1024 * 5,
+      max: options.max
+    }
+
+    this.connection = connection
+    this.closed = false
+    this.listening = null
+    this.name = name || 'mubsub'
+
+    this.create().listen()
+    this.setMaxListeners(0)
+  }
+
+  /**
+   * Close the channel.
+   *
+   * @return {Channel} this
+   * @api public
+   */
+  close () {
+    this.closed = true
+
+    return this
+  }
+
+  /**
+   * Publish an event.
+   *
+   * @param {String} event
+   * @param {Object} [message]
+   * @param {Function} [callback]
+   * @return {Channel} this
+   * @api public
+   */
+  publish (event, message, callback) {
+    callback || (callback = noop)
+
+    this.ready(function (collection) {
+      collection
+        .insertOne({ event, message })
+        .then((result) => {
+          callback(null, { _id: result.insertedId })
+        })
+        .catch((err) => {
+          callback(err)
+        })
+    })
+
+    return this
+  }
+
+  /**
+   * Subscribe an event.
+   *
+   * @param {String} [event] if no event passed - all events are subscribed.
+   * @param {Function} callback
+   * @return {Object} unsubscribe function
+   * @api public
+   */
+  subscribe (event, callback) {
+    const self = this
+
+    if (typeof event === 'function') {
+      callback = event
+      event = 'message'
+    }
+
+    this.on(event, callback)
+
+    return {
+      unsubscribe: function () {
+        self.removeListener(event, callback)
+      }
+    }
+  }
+
+  /**
+   * Create a channel collection.
+   *
+   * @return {Channel} this
+   * @api private
+   */
+  create () {
+    const self = this
+
+    function create () {
+      self.connection.db
+        .createCollection(self.name, self.collectionOpts)
+        .then((collection) => {
+          self.collection = collection
+          self.emit('collection', self.collection)
+        })
+        .catch((err) => {
+          const collectionExistsError =
+            err &&
+            (err.codeName === 'NamespaceExists' ||
+              err.message === 'collection already exists' ||
+              err.message.toLowerCase().includes('collection already exists') ||
+              err.message.match(/^a collection.+already exists$/) ||
+              err.message.match(/^Collection.+already exists\.$/))
+          if (collectionExistsError) {
+            const collection = self.connection.db.collection(self.name)
+            collection.isCapped().then((capped) => {
+              if (!capped) {
+                self.emit(
+                  'error',
+                  new Error(`${err.message}, but it's NOT capped.`)
+                )
+              } else {
+                self.collection = collection
+                self.emit('collection', self.collection)
+              }
+            })
+          } else if (err) {
+            self.emit('error', err)
+          }
+        })
+    }
+
+    this.connection.db ? create() : this.connection.once('connect', create)
+
+    return this
+  }
+
+  /**
+   * Create a listener which will emit events for subscribers.
+   * It will listen to any document with event property.
+   *
+   * @param {Object} [latest] latest document to start listening from
+   * @return {Channel} this
+   * @api private
+   */
+  listen (latest) {
+    const self = this
+
+    this.latest(
+      latest,
+      this.handle(true, function (latest, collection) {
+        const cursor = collection.find(
+          { _id: { $gt: latest._id } },
+          {
+            tailable: true,
+            awaitData: true,
+            timeout: false,
+            maxAwaitTimeMS: self.options.retryInterval
+          }
+        ).hint({ $natural: 1 })
+        const next = self.handle(function (doc) {
+          // There is no document only if the cursor is closed by accident.
+          // F.e. if collection was dropped or connection died.
+          if (!doc) {
+            return setTimeout(function () {
+              self.emit('error', new Error('Mubsub: broken cursor.'))
+              if (self.options.recreate) {
+                self.create().listen(latest)
+              }
+            }, 1000)
+          }
+          latest = doc
+          if (doc.event) {
+            self.emit(doc.event, doc.message)
+            self.emit('message', doc.message)
+          }
+
+          self.emit('document', doc)
+          process.nextTick(more)
+        })
+
+        const more = function () {
+          cursor
+            .next()
+            .then((doc) => next(undefined, doc))
+            .catch((err) => next(err))
+        }
+
+        more()
+        self.listening = collection
+        self.emit('ready', collection)
+      })
+    )
+
+    return this
+  }
+
+  /**
+   * Get the latest document from the collection. Insert a dummy object in case
+   * the collection is empty, because otherwise we don't get a tailable cursor
+   * and need to poll in a loop.
+   *
+   * @param {Object} [latest] latest known document
+   * @param {Function} callback
+   * @return {Channel} this
+   * @api private
+   */
+  latest (latest, callback) {
+    function onCollection (collection) {
+      const cursor = collection
+        .find(latest ? { _id: latest._id } : {}, { timeout: false })
+        .hint({ $natural: -1 })
+        .limit(1)
+      cursor
+        .next()
+        .then((doc) => {
+          cursor.close() // Is this required?
+          if (doc) { // further check if there are larger _id than this doc.
+            collection.findOne({ _id: { $gt: doc._id } }, { sort: { _id: -1 } }).then((record) => {
+              if (record) {
+                return callback(undefined, record, collection)
+              }
+              return callback(undefined, doc, collection)
+            }).catch(err => {
+              console.warn(`failed to find the largest _id: ${err.message}, ignore it.`)
+              return callback(undefined, doc, collection)
+            })
+          } else {
+            collection
+              .insertOne({ dummy: true })
+              .then((result) => {
+                callback(null, { _id: result.insertedId }, collection)
+              })
+              .catch((err) => {
+                callback(err, undefined, collection)
+              })
+          }
+        })
+        .catch((err) => {
+          callback(err, undefined, collection)
+        })
+    }
+
+    this.collection
+      ? onCollection(this.collection)
+      : this.once('collection', onCollection)
+
+    return this
+  }
+
+  /**
+   * Return a function which will handle errors and consider channel and connection
+   * state.
+   *
+   * @param {Boolean} [exit] if error happens and exit is true, callback will not be called
+   * @param {Function} callback
+   * @return {Function}
+   * @api private
+   */
+  handle (exit, callback) {
+    const self = this
+
+    if (typeof exit === 'function') {
+      callback = exit
+      exit = null
+    }
+
+    return function () {
+      if (self.closed || self.connection.destroyed) {
+        return
+      }
+
+      const args = [].slice.call(arguments)
+      const err = args.shift()
+      if (err) self.emit('error', err)
+      if (err && exit) return
+      callback.apply(self, args)
+    }
+  }
+
+  /**
+   * Call back if collection is ready for publishing.
+   *
+   * @param {Function} callback
+   * @return {Channel} this
+   * @api private
+   */
+  ready (callback) {
+    if (this.listening) {
+      callback(this.listening)
+    } else {
+      this.once('ready', callback)
+    }
+
+    return this
+  }
+}
+
+module.exports = Channel