keuss 1.6.14 → 1.7.0

package/examples/snippets/07-stream-simple.js

@@ -0,0 +1,53 @@
+/*
+ *
+ * very simple example of stream-mongo: one element pushed, consumed three times
+ *
+ */
+
+const async = require ('async');
+const MQ =    require ('../../backends/stream-mongo');
+
+// initialize factory
+MQ ({
+  url: 'mongodb://localhost/keuss_test_stream'
+}, (err, factory) => {
+  if (err) return console.error(err);
+
+  // factory ready, create one queue
+  const q0 = factory.queue ('test_stream', {groups: 'G1, G2, G4'});
+  const q1 = factory.queue ('test_stream', {group: 'G1'});
+  const q2 = factory.queue ('test_stream', {group: 'G2'});
+
+  async.series ([
+    // push element
+    cb => q0.push (
+      {elem: 1, headline: 'something something', tags: {a: 1, b: 2}}, // this is the payload
+      {
+        hdrs: {h1: 'aaa', h2: 12, h3: false}  // let's add some headers too
+      },
+      cb
+    ),
+    cb => setTimeout (cb, 1000),  // wait a bit
+    cb => q1.pop ('consumer-1', cb), // pop element in group G1
+    cb => q2.pop ('consumer-2', cb), // pop element in group G2
+  ], (err, res) => {
+    if (err) {
+      console.error (err);
+    }
+    else {
+      console.log ('element popped for group G1:', res[2]);
+      console.log ('element popped for group G2:', res[3]);
+      // this should print twice something like:
+      // {
+      //   _id: <some id>,
+      //   mature: <some date>,
+      //   payload: { elem: 1, headline: 'something something', tags: { a: 1, b: 2 } },
+      //   tries: 0
+      //   hdrs: {h1: 'aaa', h2: 12, h3: false}
+      // }
+    }
+
+    factory.close ();
+  });
+});
+

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "keuss",
-  "version": "1.6.14",
+  "version": "1.7.0",
   "keywords": [
     "queue",
     "persistent",
@@ -34,7 +34,8 @@
     "lodash": "~4.17.21",
     "mitt": "~3.0.0",
     "mongodb": "~4.5.0",
-    "uuid": "~8.3.2"
+    "uuid": "~8.3.2",
+    "why-is-node-running": "^2.2.2"
   },
   "devDependencies": {
     "chance": "~1.1.8",
@@ -43,6 +44,6 @@
   },
   "scripts": {
     "test": "mocha --reporter spec --check-leaks --no-timeouts --exit test/",
-    "test-with-coverage": "nyc --reporter=text -- mocha --reporter spec --check-leaks --no-timeouts --exit test/"
+    "test-with-coverage": "nyc --reporter=html -- mocha --reporter spec --check-leaks --no-timeouts --exit test/"
   }
 }

package/signal/local.js

@@ -4,6 +4,7 @@ var Signal = require ('../Signal');
 var debug = require('debug')('keuss:Signal:local');
 
 
+//////////////////////////////////////////////////////////////////////
 class LocalSignal extends Signal {
   constructor (queue, factory, opts) {
     super (queue, opts);
@@ -39,38 +40,110 @@ class LocalSignal extends Signal {
     debug ('created LocalSignal for channel %s', this._channel);
   }
 
-  type () {return LocalSignalFactory.Type ()}
 
+  //////////////////////////////////////////////////////////////////////
+  type () {
+    return LocalSignalFactory.Type ();
+  }
+
+
+  //////////////////////////////////////////////////////////////////////
   emitInsertion (mature, cb) {
     debug ('got insertion event [%o], relay on local mitt', mature);
     this._factory._emitter.emit (this._channel, mature.getTime () + '');
   }
 
+  //////////////////////////////////////////////////////////////////////
   emitPaused (paused, cb) {
     debug ('got paused event [%d], relay on local mitt', paused);
     this._factory._emitter.emit (this._channel, `p ${paused}`);
   }
+
+
+  //////////////////////////////////////////////////////////////////////
+  subscribe_extra (topic, on_cb) {
+    return this._factory.subscribe_extra (this._master.ns (), topic, on_cb);
+  }
+
+
+  //////////////////////////////////////////////////////////////////////
+  unsubscribe_extra (subscr) {
+    this._factory.unsubscribe_extra (subscr);
+  }
+
+
+  //////////////////////////////////////////////////////////////////////
+  emit_extra (topic, ev, cb) {
+    this._factory.emit_extra (this._master.ns (), topic, ev, cb);
+  }
 }
 
 
+//////////////////////////////////////////////////////////////////////
 class LocalSignalFactory {
   constructor (opts) {
     this._emitter = mitt();
+    debug ('created local factory');
+  }
+
+
+  //////////////////////////////////////////////////////////////////////
+  static Type () {
+    return 'signal:local';
   }
 
-  static Type () {return 'signal:local'}
-  type () {return LocalSignalFactory.Type ()}
 
+  //////////////////////////////////////////////////////////////////////
+  type () {
+    return LocalSignalFactory.Type ();
+  }
+
+
+  //////////////////////////////////////////////////////////////////////
   signal (queue, opts) {
     return new LocalSignal (queue, this, opts);
   }
 
+
+  //////////////////////////////////////////////////////////////////////
+  subscribe_extra (ns, topic, on_cb) {
+    const t = `keuss:signal:${ns}:extra:${topic}`;
+    debug ('subscribing to ns [%s], topic [%s]', ns, t);
+
+    const s = {
+      n: ns,
+      t: t,
+      f: (msg => on_cb (msg))
+    };
+
+    this._emitter.on (s.t, s.f); 
+    return s;
+  }
+
+
+  //////////////////////////////////////////////////////////////////////
+  unsubscribe_extra (s) {
+    this._emitter.off (s.t, s.f); 
+    debug ('unsubscribed on %s', s.t);
+  }
+
+
+  //////////////////////////////////////////////////////////////////////
+  emit_extra (ns, topic, ev, cb) {
+    const t = `keuss:signal:${ns}:extra:${topic}`;
+    debug ('emit extra on topic [%s], value [%j]', t, ev);
+    this._emitter.emit (t, ev);
+  }
+
+
+  //////////////////////////////////////////////////////////////////////
   close (cb) {
     cb ();
   }
 }
 
 
+//////////////////////////////////////////////////////////////////////
 function creator (opts, cb) {
   return cb (null, new LocalSignalFactory (opts));
 }

package/signal/mongo-capped.js

@@ -5,7 +5,7 @@ var Signal = require ('../Signal');
 
 var debug = require('debug')('keuss:Signal:MongoCapped');
 
-
+//////////////////////////////////////////////////////////////////////
 class MCSignal extends Signal {
   constructor (queue, factory, opts) {
     super (queue, opts);
@@ -45,24 +45,52 @@ class MCSignal extends Signal {
     debug ('created mongo-capped signaller for topic %s with opts %o', this._topic_name, opts);
   }
 
+
+  //////////////////////////////////////////////////////////////////////
   type () {return MCSignalFactory.Type ()}
 
+
+  //////////////////////////////////////////////////////////////////////
   emitInsertion (mature, cb) {
     debug ('emit insertion on topic [%s] value [%d])', this._topic_name, mature);
     this._factory._channel.publish (this._topic_name, mature.getTime());
   }
 
+
+  //////////////////////////////////////////////////////////////////////
   emitPaused (paused, cb) {
     debug ('emit paused on topic [%s], value [%b]', this._topic_name, paused);
     this._factory._channel.publish (this._topic_name, `p ${paused}`);
   }
 
+
+  //////////////////////////////////////////////////////////////////////
   _insertionEvent (mature) {
     debug ('got insertion event on ch [%s], mature is %s, calling master.emitInsertion()', this._channel, mature);
     this._master.signalInsertion (new Date (mature));
   }
+
+
+  //////////////////////////////////////////////////////////////////////
+  subscribe_extra (topic, on_cb) {
+    return this._factory.subscribe_extra (this._master.ns (), topic, on_cb);
+  }
+
+
+  //////////////////////////////////////////////////////////////////////
+  unsubscribe_extra (subscr) {
+    this._factory.unsubscribe_extra (subscr);
+  }
+
+
+  //////////////////////////////////////////////////////////////////////
+  emit_extra (topic, ev, cb) {
+    this._factory.emit_extra (this._master.ns (), topic, ev, cb);
+  }
 }
 
+
+//////////////////////////////////////////////////////////////////////
 class MCSignalFactory {
   constructor (opts) {
     var defaults = {
@@ -79,19 +107,56 @@ class MCSignalFactory {
     debug ('created mongo-capped factory with opts %o', opts);
   }
 
-  static Type () {return 'signal:mongo-capped'}
-  type () {return MCSignalFactory.Type ()}
 
+  //////////////////////////////////////////////////////////////////////
+  static Type () {
+    return 'signal:mongo-capped';
+  }
+  
+
+  //////////////////////////////////////////////////////////////////////
+  type () {
+    return MCSignalFactory.Type ();
+  }
+
+
+  //////////////////////////////////////////////////////////////////////
   signal (channel, opts) {
     return new MCSignal (channel, this, opts);
   }
 
+
+  //////////////////////////////////////////////////////////////////////
+  subscribe_extra (ns, topic, on_cb) {
+    const t = `keuss:signal:${ns}:extra:${topic}`;
+    debug ('subscribing to ns [%s], topic [%s]', ns, t);
+    return this._channel.subscribe (t, on_cb);
+  }
+
+
+  //////////////////////////////////////////////////////////////////////
+  unsubscribe_extra (subscr) {
+    subscr.unsubscribe ();
+    debug ('unsubscribed on %j', subscr);
+  }
+
+
+  //////////////////////////////////////////////////////////////////////
+  emit_extra (ns, topic, ev, cb) {
+    const t = `keuss:signal:${ns}:extra:${topic}`;
+    debug ('emit extra on ns [%s], topic [%s], value [%j]', ns, t, ev);
+    this._channel.publish (t, ev);
+  }
+
+
+  //////////////////////////////////////////////////////////////////////
   close (cb) {
     this._mubsub.close (cb);
   }
 }
 
 
+//////////////////////////////////////////////////////////////////////
 function creator (opts, cb) {
   return cb (null, new MCSignalFactory (opts));
 }

package/signal/redis-pubsub.js

@@ -8,6 +8,7 @@ var Signal =    require ('../Signal');
 var debug = require('debug')('keuss:Signal:RedisPubsub');
 
 
+//////////////////////////////////////////////////////////////////////
 class RPSSignal extends Signal {
   constructor (queue, factory, opts) {
     super (queue, opts);
@@ -53,24 +54,54 @@ class RPSSignal extends Signal {
     debug ('created redis-pubsub signaller for topic %s with opts %o', this._topic_name, opts);
   }
 
-  type () {return RPSSignalFactory.Type ()}
 
+  //////////////////////////////////////////////////////////////////////
+  type () {
+    return RPSSignalFactory.Type ();
+  }
+
+
+  //////////////////////////////////////////////////////////////////////
   emitInsertion (mature, cb) {
     debug ('emit insertion on channel [%s] value [%d])', this._channel, mature);
     this._rediscl_pub.publish (this._channel, mature.getTime());
   }
 
+
+  //////////////////////////////////////////////////////////////////////
   emitPaused (paused, cb) {
     debug ('emit paused on channel [%s], value [%b]', this._channel, paused);
     this._rediscl_pub.publish (this._channel, `p ${paused}`);
   }
 
+
+  //////////////////////////////////////////////////////////////////////
   _insertionEvent (mature) {
     debug ('got insertion event on ch [%s], mature is %s, calling master.emitInsertion()', this._channel, mature);
     this._master.signalInsertion (new Date (mature));
   }
+
+
+  //////////////////////////////////////////////////////////////////////
+  subscribe_extra (topic, on_cb) {
+    return this._factory.subscribe_extra (this._master.ns (), topic, on_cb);
+  }
+
+
+  //////////////////////////////////////////////////////////////////////
+  unsubscribe_extra (subscr) {
+    this._factory.unsubscribe_extra (subscr);
+  }
+
+
+  //////////////////////////////////////////////////////////////////////
+  emit_extra (topic, ev, cb) {
+    this._factory.emit_extra (this._master.ns (), topic, ev, cb);
+  }
 }
 
+
+//////////////////////////////////////////////////////////////////////
 class RPSSignalFactory {
   constructor (opts) {
     this._opts = opts || {};
@@ -86,14 +117,61 @@ class RPSSignalFactory {
     debug ('created redis-pubsub factory with opts %o', opts);
   }
 
-  static Type () {return 'signal:redis-pubsub'}
-  type () {return RPSSignalFactory.Type ()}
 
+  //////////////////////////////////////////////////////////////////////
+  static Type () {
+    return 'signal:redis-pubsub';
+  }
+
+
+  //////////////////////////////////////////////////////////////////////
+  type () {
+    return RPSSignalFactory.Type ();
+  }
+
+
+  //////////////////////////////////////////////////////////////////////
   signal (channel, opts) {
     debug ('creating redis-pubsub signaller with opts %o', opts);
     return new RPSSignal (channel, this, opts);
   }
 
+
+  //////////////////////////////////////////////////////////////////////
+  subscribe_extra (ns, topic, on_cb) {
+    const t = `keuss:signal:${ns}:extra:${topic}`;
+    debug ('subscribing to ns [%s], topic [%s]', ns, t);
+
+    const s = {
+      n: ns,
+      t: t,
+      f: (msg => on_cb (JSON.parse (msg)))
+    };
+
+    this._emitter.on (s.t, s.f);
+    this._rediscl_sub.subscribe (s.t);
+    return s;
+  }
+
+
+  //////////////////////////////////////////////////////////////////////
+  unsubscribe_extra (subscr) {
+    this._rediscl_sub.unsubscribe (subscr.t);
+    this._emitter.off (subscr.t, subscr.f); 
+    debug ('unsubscribed on %j', subscr);
+  }
+
+
+  //////////////////////////////////////////////////////////////////////
+  emit_extra (ns, topic, ev, cb) {
+    const t = `keuss:signal:${ns}:extra:${topic}`;
+    const v = JSON.stringify(ev);
+    debug ('emit extra on ns [%s], topic [%s], value [%j]', ns, t, ev);
+    this._rediscl_pub.publish (t, v);
+  }
+
+
+  //////////////////////////////////////////////////////////////////////
   close (cb) {
     async.parallel ([
       cb => this._rediscl_pub.quit (cb),
@@ -103,6 +181,7 @@ class RPSSignalFactory {
 }
 
 
+  //////////////////////////////////////////////////////////////////////
 function creator (opts, cb) {
   return cb (null, new RPSSignalFactory (opts));
 }

package/stats/mem.js

@@ -36,15 +36,12 @@ class MemStats extends Stats {
   }
 
   incr (v, delta, cb) {
-    if (!this._s.counters[v]) {
-      this._s.counters[v] = 0;
-    }
-    
     if ((delta == null) || (delta == undefined)) delta = 1;
-    var old_v = this._s.counters[v]
-    debug ('incr %s by %d: %d --> %d', v, delta, old_v, this._s.counters[v]);
-    this._s.counters[v] = this._s.counters[v] + delta;
-    if (cb) cb(null, this._s.counters[v]);
+    const old_v = _.get (this._s.counters, v, 0);
+    _.set (this._s.counters, v, old_v + delta);
+    const new_v = _.get (this._s.counters, v);
+    debug ('incr %s by %d: %d --> %d', v, delta, old_v, new_v);
+    if (cb) cb(null, new_v);
   }
 
   decr (v, delta, cb) {

package/stats/redis.js

@@ -37,13 +37,27 @@ class RedisStats extends Stats {
       var ret = {};
       for (let k in v) {
         // filter counters
-        if (k.startsWith ('counter_')) ret[k.substr (8)] = parseInt(v[k]);
+        if (k.startsWith ('counter_')) _.set (ret, k.substring (8), parseInt(v[k]));
       }
 
       cb (null, ret);
     });
   }
 
+  
+  _raw_values (cb) {
+    this._rediscl.hgetall (this._id, (err, v) => {
+      if (err) return cb(err);
+      var ret = {};
+      for (let k in v) {
+        if (k.startsWith ('counter_')) ret[k.substring (8)] = v[k];
+      }
+
+      cb (null, ret);
+    });
+  }
+
+
   paused (val, cb) {
     if (!cb) {
       // get, val is cb
@@ -144,7 +158,7 @@ class RedisStats extends Stats {
       cb => this._rediscl.hdel (this._id, 'opts', cb)
     ];
 
-    this.values ((err, vals) => {
+    this._raw_values ((err, vals) => {
       _.forEach (vals, (v, k) => {
         tasks.push (cb => this._rediscl.hdel (this._id, 'counter_' + k, cb));
       });

package/test/backends_deadletter.js

@@ -80,6 +80,7 @@ function release_mq_factory (q, factory, cb) {
   {label: 'Simple MongoDB',     mq: require ('../backends/mongo')},
   {label: 'Pipelined MongoDB',  mq: require ('../backends/pl-mongo')},
   {label: 'Tape MongoDB',       mq: require ('../backends/ps-mongo')},
+  {label: 'Stream MongoDB',     mq: require ('../backends/stream-mongo')},
   {label: 'Redis OrderedQueue', mq: require ('../backends/redis-oq')},
   {label: 'MongoDB SafeBucket', mq: require ('../backends/bucket-mongo-safe')}
 ].forEach(function (MQ_item) {
@@ -159,8 +160,8 @@ function release_mq_factory (q, factory, cb) {
             cb => stats (q, cb),
             cb => stats (factory.deadletter_queue(), cb),
           ], (err, res) => {
-            res[1].tsize.should.equal (0);
-            res[2].tsize.should.equal (0);
+              res[1].tsize.should.equal (0);
+              res[2].tsize.should.equal (0);
 
             cb (err, q, factory);
           });
@@ -209,8 +210,8 @@ function release_mq_factory (q, factory, cb) {
             cb => stats (q, cb),
             cb => stats (factory.deadletter_queue(), cb),
           ], (err, res) => {
-            res[1].tsize.should.equal (0);
-            res[2].tsize.should.equal (0);
+              res[1].tsize.should.equal (0);
+              res[2].tsize.should.equal (0);
 
             cb (err, q, factory);
           });

package/test/backends_payload.js

@@ -13,7 +13,7 @@ var factory = null;
   {label: 'Simple MongoDB',         mq: require ('../backends/mongo')},
   {label: 'Pipelined MongoDB',      mq: require ('../backends/pl-mongo')},
   {label: 'Tape MongoDB',           mq: require ('../backends/ps-mongo')},
-  {label: 'Simple MongoDB Buckets', mq: require ('../backends/bucket-mongo')},
+  {label: 'Stream MongoDB',         mq: require ('../backends/stream-mongo')},
   {label: 'Safe MongoDB Buckets',   mq: require ('../backends/bucket-mongo-safe')},
   {label: 'Redis List',             mq: require ('../backends/redis-list')},
   {label: 'Redis OrderedQueue',     mq: require ('../backends/redis-oq')},
@@ -264,6 +264,7 @@ var factory = null;
   {label: 'Simple MongoDB',       mq: require ('../backends/mongo')},
   {label: 'Pipelined MongoDB',    mq: require ('../backends/pl-mongo')},
   {label: 'Tape MongoDB',         mq: require ('../backends/ps-mongo')},
+  {label: 'Stream MongoDB',       mq: require ('../backends/stream-mongo')},
   {label: 'Safe MongoDB Buckets', mq: require ('../backends/bucket-mongo-safe')},
   {label: 'Redis OrderedQueue',   mq: require ('../backends/redis-oq')},
 ].forEach(function (MQ_item) {