keuss 1.7.4 → 2.0.0

package/Pipeline/Queue.js

@@ -37,7 +37,6 @@ class PipelinedMongoQueue extends Queue {
 
     this._col.insertOne (entry, {}, (err, result) => {
       if (err) return callback (err);
-      // TODO result.insertedCount must be 1
       callback (null, result.insertedId);
     });
   }

package/QFactory.js

@@ -36,12 +36,17 @@ class QFactory {
       this._stats_factory = res[1];
 
       if (this._opts.deadletter) {
-        this._deadletter_queue = this.queue (this._opts.deadletter.queue || '__deadletter__');
         this._max_ko = this._opts.deadletter.max_ko || 0;
-        debug('%s: uses deadletter queue %s, max KO is %d', this._name, this._deadletter_queue.name(), this._max_ko);
+        const qname = this._opts.deadletter.queue || '__deadletter__';
+        debug('%s: uses deadletter queue %s, max KO is %d', this._name, qname, this._max_ko);
+        this.queue (qname, (err, dlq) => {
+          this._deadletter_queue = dlq;
+          cb (err);
+        });
+      }
+      else {
+        cb();
       }
-
-      cb();
     });
   }
 
@@ -53,8 +58,8 @@ class QFactory {
     return this._stats_factory;
   }
 
-  queue (name, opts) {
-    return null;
+  queue (name, opts, cb) {
+    return cb ();
   }
 
   close (cb) {

package/Queue.js

@@ -342,8 +342,6 @@ class Queue {
   //////////////////////////////////
   // obtain element from queue
   pop (cid, opts, callback) {
-    // TODO fail if too many consumers?
-
     if (this._drained) return setImmediate (() => {
       debug ('%s: pop while drained, return error', this._name);
       if (callback) callback ('drain');
@@ -461,7 +459,7 @@ class Queue {
     if (
       (obj.tries) &&                            // only if we got tries
       (this._factory.deadletter_queue ()) &&    // AND the factory has a deadletter queue
-      (this._factory.max_ko ()) &&              // AND thee's a max ko attempts 
+      (this._factory.max_ko ()) &&              // AND there's a max ko attempts 
       (obj.tries > this._factory.max_ko ()) &&  // AND we got enough tries
       (this.name () != '__deadletter__')        // and this queue is not deadletter already
     ) {
@@ -592,7 +590,6 @@ class Queue {
         }
       }
 
-      // TODO eat up errors?
       if (err) {
         // get/reserve in error
         debug ('%s - tid %s: getOrReserve_cb in error: err %o', this._name, consumer.tid, err);

package/README.md

@@ -1,5 +1,6 @@
 # Keuss
-Enterprise-grade Job Queues for node.js, backed by redis and/or MongoDB
+
+Enterprise-grade Job Queues for node.js backed by redis, MongoDB or PostgreSQL
 
 * [Quickstart](https://pepmartinez.github.io/keuss/docs/quickstart)
 * [Documentation](https://pepmartinez.github.io/keuss/docs/)

package/TODO

@@ -8,16 +8,7 @@ prio 1
 * pipelines
   + add some commonplace transforms (etl-like)
   + create a server for that, allow processor code defined externally
-* documentation
-  + blog entries:
-    - motivations on using mongodb+redis. comparatives, pros&cons
-    - buckets
-    - pipelines
-    - tape
-    - redis-oq
-    - exotic experiments: intraorder
 
 prio 0
 -----------------------------------------------------------------
-* bridges?
 * signal queue creation or read queues on refresh?

package/backends/bucket-mongo-safe.js

@@ -96,8 +96,6 @@ class Bucket {
         elem.mature = this._mature;
         elem._id = this.id () + ':' + i;
 
-        // TODO optimization: set this._b[i] = null
-
         if (is_reserve) {
           this._b_states[i] = State.Reserved;
           this._b_counts.Reserved++;
@@ -314,7 +312,6 @@ class BucketSet {
         if (bcket.exhausted ()) {
           debug ('bucket %s already exhausted, read another', bcket.id());
           this._active_bucket = null;
-          // TODO loop internally!!!!!
           cb ();
         }
         else {
@@ -523,9 +520,7 @@ class BucketSet {
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
 class BucketMongoSafeQueue extends Queue {
-
   /*
-
   options:
     bucket_max_size    || 1024;
     bucket_max_wait    || 500;
@@ -540,9 +535,7 @@ class BucketMongoSafeQueue extends Queue {
   //////////////////////////////////////////////
     super (name, factory, opts, orig_opts);
 
-    this._factory = factory;
     this._col = factory._db.collection (name);
-    this._ensureIndexes (err => {});
 
     this._insert_bucket = {
       _id: new mongo.ObjectID (),
@@ -821,7 +814,7 @@ class BucketMongoSafeQueue extends Queue {
 
   /////////////////////////////////////////
   _ensureIndexes (cb) {
-    this._col.createIndex ({mature : 1}, err => cb (err));
+    this._col.createIndex ({mature : 1}, err => cb (err, this));
   }
 
 
@@ -915,10 +908,17 @@ class Factory extends QFactory_MongoDB_defaults {
     this._db = mongo_conn.db();
   }
 
-  queue (name, opts) {
-    var full_opts = {}
+  queue (name, opts, cb) {
+    if (!cb) {
+      cb = opts;
+      opts = {};
+    }
+    
+    const full_opts = {};
     _.merge(full_opts, this._opts, opts);
-    return new BucketMongoSafeQueue (name, this, full_opts, opts);
+
+    const q = new BucketMongoSafeQueue (name, this, full_opts, opts);
+    q._ensureIndexes (cb);
   }
 
   close (cb) {

package/backends/intraorder.js

@@ -12,14 +12,10 @@ const debug = require('debug')('keuss:Queue:intraqueue');
 
 
 class IntraOrderedQueue extends Queue {
-
   //////////////////////////////////////////////
   constructor (name, factory, opts, orig_opts) {
     super (name, factory, opts, orig_opts);
-
-    this._factory = factory;
     this._col = factory._db.collection (name);
-    this.ensureIndexes (err => {});
   }
 
 
@@ -49,7 +45,6 @@ class IntraOrderedQueue extends Queue {
     this._col.updateOne (q, upd, opts, (err, result) => {
       debug ('insert: updateOne (%j, %j, %j) => (%j, %j)', q, upd, opts, err, result);
       if (err) return callback (err);
-      // TODO result.insertedCount must be 1
       callback (null, result.upsertedId || q._id);
     });
   }
@@ -308,10 +303,10 @@ class IntraOrderedQueue extends Queue {
 
   //////////////////////////////////////////////////////////////////
   // create needed indexes for O(1) functioning
-  ensureIndexes (cb) {
+  _ensureIndexes (cb) {
     async.series ([
-      cb => this._col.createIndex ({mature : 1, qcnt: 1}, cb),
-    ], cb);
+      cb => this._col.createIndex ({mature : 1, qcnt: 1}, err => cb (err)),
+    ], err => cb (err, this));
   }
 }
 
@@ -323,10 +318,17 @@ class Factory extends QFactory_MongoDB_defaults {
     this._db = mongo_conn.db();
   }
 
-  queue (name, opts) {
-    var full_opts = {};
+  queue (name, opts, cb) {
+    if (!cb) {
+      cb = opts;
+      opts = {};
+    }
+
+    const full_opts = {};
     _.merge(full_opts, this._opts, opts);
-    return new IntraOrderedQueue (name, this, full_opts, opts);
+
+    const q = new IntraOrderedQueue (name, this, full_opts, opts);
+    q._ensureIndexes (cb);
   }
 
   close (cb) {

package/backends/mongo.js

@@ -11,10 +11,7 @@ class SimpleMongoQueue extends Queue {
   //////////////////////////////////////////////
   constructor (name, factory, opts, orig_opts) {
     super (name, factory, opts, orig_opts);
-
-    this._factory = factory;
     this._col = factory._db.collection (name);
-    this.ensureIndexes (function (err) {});
   }
 
 
@@ -33,7 +30,6 @@ class SimpleMongoQueue extends Queue {
   insert (entry, callback) {
     this._col.insertOne (entry, {}, (err, result) => {
       if (err) return callback (err);
-      // TODO result.insertedCount must be 1
       callback (null, result.insertedId);
     });
   }
@@ -213,8 +209,8 @@ class SimpleMongoQueue extends Queue {
 
   //////////////////////////////////////////////////////////////////
   // create needed indexes for O(1) functioning
-  ensureIndexes (cb) {
-    this._col.createIndex ({mature : 1}, err => cb (err));
+  _ensureIndexes (cb) {
+    this._col.createIndex ({mature : 1}, err => cb (err, this));
   }
 };
 
@@ -226,10 +222,17 @@ class Factory extends QFactory_MongoDB_defaults {
     this._db = mongo_conn.db();
   }
 
-  queue (name, opts) {
-    var full_opts = {};
+  queue (name, opts, cb) {
+    if (!cb) {
+      cb = opts;
+      opts = {};
+    }
+
+    const full_opts = {};
     _.merge(full_opts, this._opts, opts);
-    return new SimpleMongoQueue (name, this, full_opts, opts);
+    
+    const q = new SimpleMongoQueue (name, this, full_opts, opts);
+    q._ensureIndexes (cb);
   }
 
   close (cb) {

package/backends/pl-mongo.js

@@ -119,18 +119,21 @@ class Factory extends QFactory_MongoDB_defaults {
 
 
   ///////////////////////////////////////////////////////////
-  queue (name, opts) {
-    if (!opts) opts = {};
+  queue (name, opts, cb) {
+    if (!cb) {
+      cb = opts;
+      opts = {};
+    }
 
-    var pl_name = opts.pipeline || 'default';
-    var pipeline = this._pipelines[pl_name];
+    const pl_name = opts.pipeline || 'default';
+    let pipeline = this._pipelines[pl_name];
 
     if (!pipeline) {
       this._pipelines[pl_name] = new Pipeline (pl_name, this);
       pipeline = this._pipelines[pl_name];
     }
 
-    return this._queue_from_pipeline (name, pipeline, opts);
+    return setImmediate (() => cb (null, this._queue_from_pipeline (name, pipeline, opts)));
   }
 
 

package/backends/postgres.js

@@ -0,0 +1,380 @@
+const _ =    require ('lodash');
+const uuid = require ('uuid');
+const pg =   require ('pg');
+
+const Queue =    require ('../Queue');
+const QFactory = require ('../QFactory');
+
+const debug = require('debug')('keuss:Queue:postgres');
+
+class PGQueue extends Queue {
+
+
+	// TODO escape table name properly , as identifier, and check for reserved names using pg_get_keywords(), if the name is in select * from pg_get_keywords() where catdesc = 'R', then it should not be used
+  //////////////////////////////////////////////
+  constructor (name, factory, opts, orig_opts) {
+    super (name, factory, opts, orig_opts);
+    this._pool = factory._pool;
+    this._tbl_name = this._name; 
+  }
+
+
+  /////////////////////////////////////////
+  static Type () {
+    return 'postgres:simple';
+  }
+
+
+  /////////////////////////////////////////
+  type () {
+    return 'postgres:simple';
+  }
+
+
+  //////////////////////////////////////////////
+  // ensure table and indexes exists
+  _init (cb) {
+    this._pool.query (`
+    CREATE TABLE IF NOT EXISTS ${this._tbl_name} (
+      _id      VARCHAR(64) PRIMARY KEY,
+      _pl      JSONB,
+      mature   TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP NOT NULL,
+      tries    INTEGER DEFAULT 0 NOT NULL,
+      reserved TIMESTAMPTZ
+    );
+    CREATE INDEX IF NOT EXISTS idx_${this._tbl_name}_mature ON ${this._tbl_name} (mature);
+    `, err => cb (err));
+  }
+
+
+  /////////////////////////////////////////
+  // add element to queue
+  insert (entry, cb) {
+    const _id =     entry.id || uuid.v4();
+    const tries =   entry.tries || 0;
+    const mature =  new Date (entry.mature);
+
+    const pl = {
+      payload: entry.payload,
+      hdrs:    entry.hdrs || {}
+    }
+
+    if (Buffer.isBuffer (pl.payload)) {
+      pl.payload = pl.payload.toString ('base64');
+      pl.type = 'buffer';
+    }
+
+    this._pool.query (`INSERT INTO ${this._tbl_name} VALUES($1, $2, $3, $4)`, [_id, pl, mature, tries], (err, res) => {
+      if (err) return cb (err);
+      cb (null, _id)
+    })
+  }
+
+
+  /////////////////////////////////////////
+  // get element from queue
+  get (cb) {    
+    this._pool.query (`
+      DELETE FROM ${this._tbl_name}
+      WHERE _id = (
+        SELECT _id
+        FROM ${this._tbl_name}
+        WHERE mature < now()
+        ORDER BY mature
+        FOR UPDATE SKIP LOCKED
+        LIMIT 1
+      )
+      RETURNING *;
+    `, (err, res) => {
+      if (err) return cb (err);
+
+      if (_.size (res.rows) == 0) return cb (null, null); // not found
+      
+      const pl = res.rows[0];
+
+      _.merge (pl, pl._pl);
+      delete (pl._pl);
+
+      if (pl.type == 'buffer') {
+        try {
+          pl.payload = Buffer.from (pl.payload, 'base64');
+        } catch (e) {
+          // TODO maybe log some error/Statistic?
+        }
+      }
+
+      cb (null, pl);
+    })
+  }
+
+
+  //////////////////////////////////
+  // reserve element: call cb (err, pl) where pl has an id
+  reserve (cb) {
+    const delay = this._opts.reserve_delay || 120;
+
+    this._pool.query (`
+      UPDATE ${this._tbl_name}
+      SET
+        tries = tries + 1,
+        mature = now() + make_interval(secs => ${delay}),
+        reserved = now()
+      WHERE _id = (
+        SELECT _id
+        FROM ${this._tbl_name}
+        WHERE mature < now()
+        ORDER BY mature
+        FOR UPDATE SKIP LOCKED
+        LIMIT 1
+      )
+      RETURNING *;
+    `, (err, res) => {
+      if (err) return cb (err);
+      
+      if (_.size (res.rows) == 0) return cb (null, null); // not found
+      
+      const pl = res.rows[0];
+
+      _.merge (pl, pl._pl);
+      delete (pl._pl);
+
+      // adjust tries to pre-update
+      pl.tries--;
+
+      if (pl.type == 'buffer') {
+        try {
+          pl.payload = Buffer.from (pl.payload, 'base64');
+        } catch (e) {
+          // TODO maybe log some error/Statistic?
+        }
+      }
+
+      cb (null, pl);
+    })
+  }
+
+
+  //////////////////////////////////
+  // commit previous reserve, by p.id
+  commit (id, cb) {
+    if (!uuid.validate (id)) return cb ('id [' + id + '] can not be used as commit id: not a valid UUID');
+
+    this._pool.query (`
+      DELETE FROM ${this._tbl_name}
+      WHERE _id = $1
+      AND reserved IS NOT NULL
+    `, 
+    [id],
+    (err, res) => {
+      if (err) return cb (err);
+      cb (null, res && (res.rowCount == 1));
+    })
+  }
+
+
+  //////////////////////////////////
+  // rollback previous reserve, by p.id
+  rollback (id, next_t, cb) {
+    if (_.isFunction (next_t)) {
+      cb = next_t;
+      next_t = null;
+    }
+
+    if (!uuid.validate (id)) return cb ('id [' + id + '] can not be used as rollback id: not a valid UUID');
+
+    const nxt = (next_t ? new Date (next_t) : Queue.now ());
+
+    this._pool.query (`
+      UPDATE ${this._tbl_name}
+      SET
+        reserved = NULL,
+        mature = $1
+      WHERE _id = $2
+      AND reserved IS NOT NULL
+    `, 
+    [nxt, id],
+    (err, res) => {
+      if (err) return cb (err);
+      cb (null, res && (res.rowCount == 1));
+    })
+  }
+
+
+  //////////////////////////////////
+  // queue size including non-mature elements
+  totalSize (cb) {
+    this._pool.query (`
+      SELECT COUNT(*)
+      FROM ${this._tbl_name}
+    `, 
+    (err, res) => {
+      if (err) return cb (err);
+      cb (null, parseInt (res.rows[0].count));
+    })
+  }
+
+
+  //////////////////////////////////
+  // queue size NOT including non-mature elements
+  size (cb) {
+    this._pool.query (`
+      SELECT COUNT(*)
+      FROM ${this._tbl_name}
+      WHERE mature < now()
+    `, 
+    (err, res) => {
+      if (err) return cb (err);
+      cb (null, parseInt (res.rows[0].count));
+    })
+  }
+
+
+  //////////////////////////////////
+  // queue size of non-mature elements only
+  schedSize (cb) {
+    this._pool.query (`
+      SELECT COUNT(*)
+      FROM ${this._tbl_name}
+      WHERE mature >= now()
+      AND reserved IS NULL
+    `, 
+    (err, res) => {
+      if (err) return cb (err);
+      cb (null, parseInt (res.rows[0].count));
+    })
+  }
+
+
+  //////////////////////////////////
+  // queue size of reserved elements only
+  resvSize (cb) {
+    this._pool.query (`
+      SELECT COUNT(*)
+      FROM ${this._tbl_name}
+      WHERE mature >= now()
+      AND reserved IS NOT NULL
+    `, 
+    (err, res) => {
+      if (err) return cb (err);
+      cb (null, parseInt (res.rows[0].count));
+    })
+  }
+
+
+  /////////////////////////////////////////
+  // get element from queue
+  next_t (cb) {  
+    this._pool.query (`
+      SELECT mature
+      FROM ${this._tbl_name}
+      ORDER BY mature
+      LIMIT 1
+    `, (err, res) => {
+      if (err)                    return cb (err);
+      if (_.size (res.rows) == 0) return cb (null, null); // not found
+      
+      cb (null, res.rows[0].mature);
+    });
+  }
+
+
+  //////////////////////////////////////////////
+  // remove by id
+  remove (id, cb) {
+    if (!uuid.validate (id)) return cb ('id [' + id + '] can not be used as remove id: not a valid UUID');
+
+    this._pool.query (`
+      DELETE FROM ${this._tbl_name}
+      WHERE _id = $1
+      AND reserved IS NULL
+    `, 
+    [id],
+    (err, res) => {
+      if (err) return cb (err);
+      cb (null, res && (res.rowCount == 1));
+    })
+  }
+
+
+  ///////////////////////////////////////////////////////////////////////////////
+  // private parts
+
+};
+
+
+
+///////////////////////////////////////////////////////////////////////////////
+class Factory extends QFactory {
+  constructor (opts, pg_pool) {
+    super (opts);
+    this._pool = pg_pool;
+    debug ('crated Factory with options %j', opts);
+  }
+
+  queue (name, opts, cb) {
+    if (!cb) {
+      cb = opts;
+      opts = {};
+    }
+    
+    const full_opts = {};
+    _.merge(full_opts, this._opts, opts);
+    debug ('creating queue with full_opts %j', full_opts);
+
+    const q = new PGQueue (name, this, full_opts, opts);
+    q._init (err => {
+      if (err) return cb (err);
+      cb (null, q);
+    });
+  }
+
+  close (cb) {
+    super.close (() => {
+      debug ('closing pool...');
+      this._pool.end (cb);
+    });
+  }
+
+  type () {
+    return PGQueue.Type ();
+  }
+
+  capabilities () {
+    return {
+      sched:    true,
+      reserve:  true,
+      pipeline: false,
+      tape:     false,
+      remove:   true
+    };
+  }
+}
+
+
+///////////////////////////////////////////////////////////////////////////////
+function creator (opts, cb) {
+  const _opts = opts || {};
+
+  debug ('Creator: creating pool with %j', _opts);
+
+  const dflt = {
+    user:     'pg', 
+    password: 'pg',
+    host:     'localhost',
+    port:     5432,
+    database: 'pg'
+  }
+
+  const pool = new pg.Pool(_.merge ({}, dflt, _opts.postgres));
+
+  // ensure the pool can connect
+  pool.query ('select 1', err => {
+    if (err) return cb (err);
+
+    debug ('pool created, can connect. Creating Factory...');
+    const F = new Factory (_opts, pool);
+    F.async_init (err => cb (err, F));
+  });
+}
+
+module.exports = creator;