keuss 1.7.4 → 2.0.1

package/Pipeline/Queue.js

@@ -37,7 +37,6 @@ class PipelinedMongoQueue extends Queue {
 
     this._col.insertOne (entry, {}, (err, result) => {
       if (err) return callback (err);
-      // TODO result.insertedCount must be 1
       callback (null, result.insertedId);
     });
   }

package/QFactory.js

@@ -1,12 +1,13 @@
-var async = require ('async');
-var _ =     require ('lodash');
+const async = require ('async');
+const _ =     require ('lodash');
 
-var LocalSignal = require ('./signal/local');
-var MemStats =    require ('./stats/mem');
+const LocalSignal = require ('./signal/local');
+const MemStats =    require ('./stats/mem');
 
-var debug = require('debug')('keuss:QFactory');
+const debug = require('debug')('keuss:QFactory');
 
 
+/////////////////////////////////////////////////////////////////////////////
 class QFactory {
   constructor (opts) {
     this._opts = opts || {};
@@ -15,6 +16,8 @@ class QFactory {
     debug ('created QFactory %s with opts %o', this._name, this._opts);
   }
 
+
+  /////////////////////////////////////////////////////////////////////////////
   async_init (cb) {
     if (!this._opts.stats)      this._opts.stats = {};
     if (!this._opts.stats.opts) this._opts.stats.opts = {};
@@ -22,8 +25,8 @@ class QFactory {
     if (!this._opts.signaller)      this._opts.signaller = {};
     if (!this._opts.signaller.opts) this._opts.signaller.opts = {};
 
-    var signal_provider = this._opts.signaller.provider || LocalSignal;
-    var stats_provider = this._opts.stats.provider || MemStats;
+    const signal_provider = this._opts.signaller.provider || LocalSignal;
+    const stats_provider = this._opts.stats.provider || MemStats;
 
     async.parallel ([
       cb => signal_provider (this._opts.signaller.opts, cb),
@@ -36,27 +39,40 @@ class QFactory {
       this._stats_factory = res[1];
 
       if (this._opts.deadletter) {
-        this._deadletter_queue = this.queue (this._opts.deadletter.queue || '__deadletter__');
         this._max_ko = this._opts.deadletter.max_ko || 0;
-        debug('%s: uses deadletter queue %s, max KO is %d', this._name, this._deadletter_queue.name(), this._max_ko);
+        const qname = this._opts.deadletter.queue || '__deadletter__';
+        debug('%s: uses deadletter queue %s, max KO is %d', this._name, qname, this._max_ko);
+        this.queue (qname, (err, dlq) => {
+          this._deadletter_queue = dlq;
+          cb (err);
+        });
+      }
+      else {
+        cb();
       }
-
-      cb();
     });
   }
 
+
+  /////////////////////////////////////////////////////////////////////////////
   signaller_factory () {
     return this._signaller_factory;
   }
 
+
+  /////////////////////////////////////////////////////////////////////////////
   stats_factory () {
     return this._stats_factory;
   }
 
-  queue (name, opts) {
-    return null;
+
+  /////////////////////////////////////////////////////////////////////////////
+  queue (name, opts, cb) {
+    return cb ();
   }
 
+
+  /////////////////////////////////////////////////////////////////////////////
   close (cb) {
     debug ('%s: closing', this._name);
     async.parallel ([
@@ -65,50 +81,66 @@ class QFactory {
     ], cb);
   }
 
+  
+  /////////////////////////////////////////////////////////////////////////////
   name () {
     return this._name;
   }
 
+
+  /////////////////////////////////////////////////////////////////////////////
   type () {
     return 'none';
   }
 
+
+  /////////////////////////////////////////////////////////////////////////////
   capabilities () {
     return {};
   }
 
+
+  /////////////////////////////////////////////////////////////////////////////
   deadletter_queue () {
     return this._deadletter_queue;
   }
 
+
+  /////////////////////////////////////////////////////////////////////////////
   max_ko () {
     return this._max_ko;
   }
 
+
+  /////////////////////////////////////////////////////////////////////////////
   list (opts, cb) {
     // use stats factory
     this._stats_factory.queues (this.name (), opts, cb);
   }
 
+
+  /////////////////////////////////////////////////////////////////////////////
   recreate_topology (cb) {
     debug ('%s: recreating topology', this._name);
 
     this.list ({full: true}, (err, ql) => {
       if (err) return cb (err);
 
-      var ret = {};
+      const tasks = {};
 
       _.each (ql, (v, k) => {
         debug ('%s: recreating topology: adding queue %s with opts %o', this._name, k, v.opts);
-        ret[k] = this.queue (k, v.opts);
+        ret[k] = cb => this.queue (k, v.opts, cb); // TODO fix to async
       });
 
-      cb (null, ret);
+      async.parallel (tasks, cb);
     });
   }
 
+
+  /////////////////////////////////////////////////////////////////////////////
   to_descriptor_obj () {
-    let obj = {
+    const obj = {
       name: this.name (),
       type: this.type (),
       opts: _.omit (this._opts, ['stats', 'signaller']),

package/Queue.js

@@ -342,8 +342,6 @@ class Queue {
   //////////////////////////////////
   // obtain element from queue
   pop (cid, opts, callback) {
-    // TODO fail if too many consumers?
-
     if (this._drained) return setImmediate (() => {
       debug ('%s: pop while drained, return error', this._name);
       if (callback) callback ('drain');
@@ -461,7 +459,7 @@ class Queue {
     if (
       (obj.tries) &&                            // only if we got tries
       (this._factory.deadletter_queue ()) &&    // AND the factory has a deadletter queue
-      (this._factory.max_ko ()) &&              // AND thee's a max ko attempts 
+      (this._factory.max_ko ()) &&              // AND there's a max ko attempts 
       (obj.tries > this._factory.max_ko ()) &&  // AND we got enough tries
       (this.name () != '__deadletter__')        // and this queue is not deadletter already
     ) {
@@ -592,7 +590,6 @@ class Queue {
         }
       }
 
-      // TODO eat up errors?
       if (err) {
         // get/reserve in error
         debug ('%s - tid %s: getOrReserve_cb in error: err %o', this._name, consumer.tid, err);

package/README.md

@@ -1,5 +1,6 @@
 # Keuss
-Enterprise-grade Job Queues for node.js, backed by redis and/or MongoDB
+
+Enterprise-grade Job Queues for node.js backed by redis, MongoDB or PostgreSQL
 
 * [Quickstart](https://pepmartinez.github.io/keuss/docs/quickstart)
 * [Documentation](https://pepmartinez.github.io/keuss/docs/)

package/TODO

@@ -8,16 +8,7 @@ prio 1
 * pipelines
   + add some commonplace transforms (etl-like)
   + create a server for that, allow processor code defined externally
-* documentation
-  + blog entries:
-    - motivations on using mongodb+redis. comparatives, pros&cons
-    - buckets
-    - pipelines
-    - tape
-    - redis-oq
-    - exotic experiments: intraorder
 
 prio 0
 -----------------------------------------------------------------
-* bridges?
 * signal queue creation or read queues on refresh?

package/backends/bucket-mongo-safe.js

@@ -96,8 +96,6 @@ class Bucket {
         elem.mature = this._mature;
         elem._id = this.id () + ':' + i;
 
-        // TODO optimization: set this._b[i] = null
-
         if (is_reserve) {
           this._b_states[i] = State.Reserved;
           this._b_counts.Reserved++;
@@ -314,7 +312,6 @@ class BucketSet {
         if (bcket.exhausted ()) {
           debug ('bucket %s already exhausted, read another', bcket.id());
           this._active_bucket = null;
-          // TODO loop internally!!!!!
           cb ();
         }
         else {
@@ -523,9 +520,7 @@ class BucketSet {
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
 class BucketMongoSafeQueue extends Queue {
-
   /*
-
   options:
     bucket_max_size    || 1024;
     bucket_max_wait    || 500;
@@ -540,9 +535,7 @@ class BucketMongoSafeQueue extends Queue {
   //////////////////////////////////////////////
     super (name, factory, opts, orig_opts);
 
-    this._factory = factory;
     this._col = factory._db.collection (name);
-    this._ensureIndexes (err => {});
 
     this._insert_bucket = {
       _id: new mongo.ObjectID (),
@@ -821,7 +814,7 @@ class BucketMongoSafeQueue extends Queue {
 
   /////////////////////////////////////////
   _ensureIndexes (cb) {
-    this._col.createIndex ({mature : 1}, err => cb (err));
+    this._col.createIndex ({mature : 1}, err => cb (err, this));
   }
 
 
@@ -915,10 +908,17 @@ class Factory extends QFactory_MongoDB_defaults {
     this._db = mongo_conn.db();
   }
 
-  queue (name, opts) {
-    var full_opts = {}
+  queue (name, opts, cb) {
+    if (!cb) {
+      cb = opts;
+      opts = {};
+    }
+    
+    const full_opts = {};
     _.merge(full_opts, this._opts, opts);
-    return new BucketMongoSafeQueue (name, this, full_opts, opts);
+
+    const q = new BucketMongoSafeQueue (name, this, full_opts, opts);
+    q._ensureIndexes (cb);
   }
 
   close (cb) {

package/backends/intraorder.js

@@ -12,14 +12,10 @@ const debug = require('debug')('keuss:Queue:intraqueue');
 
 
 class IntraOrderedQueue extends Queue {
-
   //////////////////////////////////////////////
   constructor (name, factory, opts, orig_opts) {
     super (name, factory, opts, orig_opts);
-
-    this._factory = factory;
     this._col = factory._db.collection (name);
-    this.ensureIndexes (err => {});
   }
 
 
@@ -49,7 +45,6 @@ class IntraOrderedQueue extends Queue {
     this._col.updateOne (q, upd, opts, (err, result) => {
       debug ('insert: updateOne (%j, %j, %j) => (%j, %j)', q, upd, opts, err, result);
       if (err) return callback (err);
-      // TODO result.insertedCount must be 1
       callback (null, result.upsertedId || q._id);
     });
   }
@@ -308,10 +303,10 @@ class IntraOrderedQueue extends Queue {
 
   //////////////////////////////////////////////////////////////////
   // create needed indexes for O(1) functioning
-  ensureIndexes (cb) {
+  _ensureIndexes (cb) {
     async.series ([
-      cb => this._col.createIndex ({mature : 1, qcnt: 1}, cb),
-    ], cb);
+      cb => this._col.createIndex ({mature : 1, qcnt: 1}, err => cb (err)),
+    ], err => cb (err, this));
   }
 }
 
@@ -323,10 +318,17 @@ class Factory extends QFactory_MongoDB_defaults {
     this._db = mongo_conn.db();
   }
 
-  queue (name, opts) {
-    var full_opts = {};
+  queue (name, opts, cb) {
+    if (!cb) {
+      cb = opts;
+      opts = {};
+    }
+
+    const full_opts = {};
     _.merge(full_opts, this._opts, opts);
-    return new IntraOrderedQueue (name, this, full_opts, opts);
+
+    const q = new IntraOrderedQueue (name, this, full_opts, opts);
+    q._ensureIndexes (cb);
   }
 
   close (cb) {

package/backends/mongo.js

@@ -11,10 +11,7 @@ class SimpleMongoQueue extends Queue {
   //////////////////////////////////////////////
   constructor (name, factory, opts, orig_opts) {
     super (name, factory, opts, orig_opts);
-
-    this._factory = factory;
     this._col = factory._db.collection (name);
-    this.ensureIndexes (function (err) {});
   }
 
 
@@ -33,7 +30,6 @@ class SimpleMongoQueue extends Queue {
   insert (entry, callback) {
     this._col.insertOne (entry, {}, (err, result) => {
       if (err) return callback (err);
-      // TODO result.insertedCount must be 1
       callback (null, result.insertedId);
     });
   }
@@ -213,8 +209,8 @@ class SimpleMongoQueue extends Queue {
 
   //////////////////////////////////////////////////////////////////
   // create needed indexes for O(1) functioning
-  ensureIndexes (cb) {
-    this._col.createIndex ({mature : 1}, err => cb (err));
+  _ensureIndexes (cb) {
+    this._col.createIndex ({mature : 1}, err => cb (err, this));
   }
 };
 
@@ -226,10 +222,17 @@ class Factory extends QFactory_MongoDB_defaults {
     this._db = mongo_conn.db();
   }
 
-  queue (name, opts) {
-    var full_opts = {};
+  queue (name, opts, cb) {
+    if (!cb) {
+      cb = opts;
+      opts = {};
+    }
+
+    const full_opts = {};
     _.merge(full_opts, this._opts, opts);
-    return new SimpleMongoQueue (name, this, full_opts, opts);
+    
+    const q = new SimpleMongoQueue (name, this, full_opts, opts);
+    q._ensureIndexes (cb);
   }
 
   close (cb) {

package/backends/pl-mongo.js

@@ -119,18 +119,21 @@ class Factory extends QFactory_MongoDB_defaults {
 
 
   ///////////////////////////////////////////////////////////
-  queue (name, opts) {
-    if (!opts) opts = {};
+  queue (name, opts, cb) {
+    if (!cb) {
+      cb = opts;
+      opts = {};
+    }
 
-    var pl_name = opts.pipeline || 'default';
-    var pipeline = this._pipelines[pl_name];
+    const pl_name = opts.pipeline || 'default';
+    let pipeline = this._pipelines[pl_name];
 
     if (!pipeline) {
       this._pipelines[pl_name] = new Pipeline (pl_name, this);
       pipeline = this._pipelines[pl_name];
     }
 
-    return this._queue_from_pipeline (name, pipeline, opts);
+    return setImmediate (() => cb (null, this._queue_from_pipeline (name, pipeline, opts)));
   }