keuss 1.6.8 → 1.6.11

package/Pipeline/Queue.js

@@ -167,6 +167,10 @@ class PipelinedMongoQueue extends Queue {
       this._embed_update_for_payload (upd, opts.update);
     }
 
+    if (opts.hdrs) {
+      _.each (opts.hdrs, (v, k) => upd.$set['hdrs.' + k] = v);
+    }
+
     this._col.updateOne (q, upd, {}, (err, result) => {
       if (err) return callback (err);
       callback (null, result && (result.modifiedCount == 1));
@@ -274,9 +278,15 @@ class PipelinedMongoQueue extends Queue {
 
 
   //////////////////////////////////////////////
-  // redefnition
+  // redefinition from Queue
   _move_to_deadletter (obj, cb) {
-    this.pl_step (obj._id, this._factory.deadletter_queue (), {}, (err, res) => {
+    const hdrs = {
+      'x-dl-from-queue': this.name (),
+      'x-dl-t': new Date().toISOString (),
+      'x-dl-tries': obj.tries
+    };
+
+    this.pl_step (obj._id, this._factory.deadletter_queue (), {hdrs}, (err, res) => {
       if (err) return cb (err);
       this._stats.incr ('get');
       this._factory.deadletter_queue ()._stats.incr ('put');

package/Queue.js

@@ -46,6 +46,7 @@ class Queue {
       this._stats.incr ('reserve', 0);
       this._stats.incr ('commit', 0);
       this._stats.incr ('rollback', 0);
+      this._stats.incr ('deadletter', 0);
     }
 
     // if true, queue is being drained just before shutdown
@@ -432,14 +433,14 @@ class Queue {
     let id = (obj._id ? obj._id : obj);
 
     if (
-      (obj.tries) &&
-      (this._factory.deadletter_queue ()) &&
-      (this._factory.max_ko ()) &&
-      (obj.tries > this._factory.max_ko ())
+      (obj.tries) &&                            // only if we got tries
+      (this._factory.deadletter_queue ()) &&    // AND the factory has a deadletter queue
+      (this._factory.max_ko ()) &&              // AND thee's a max ko attempts 
+      (obj.tries > this._factory.max_ko ()) &&  // AND we got enough tries
+      (this.name () != '__deadletter__')        // and this queue is not deadletter already
     ) {
       debug ('%s: too many retries (%d), moving to deadletter', obj._id, obj.tries);
       this._move_to_deadletter (obj, cb);
-      // TODO add from-what-queue to deadletter elements
     }
     else {
       this.rollback (id, next_t, (err, res) => {
@@ -709,21 +710,30 @@ class Queue {
   /////////////////////////////////////////////
   _move_to_deadletter (obj, cb) {
     // commit and move to deadletter
-    // ALSO NOT IN deadletter queue (to void loop)
     // commit element in origin queue, push in deadletter afterwards
+    const opts = {
+      hdrs: _.clone (obj.hdrs || {})
+    };
+
+    // add some extra x-dl-* headers
+    opts.hdrs['x-dl-from-queue'] = this.name ();
+    opts.hdrs['x-dl-t'] = new Date().toISOString ();
+    opts.hdrs['x-dl-tries'] = obj.tries;
+
     this.commit (obj._id, err => {
       if (err) {
         debug ('while committing %s prior to moving to deadletter: %j', obj._id, err);
         return cb (err);
       }
 
-      this._factory.deadletter_queue ().push (obj.payload, (err, res) => {
+      this._factory.deadletter_queue ().push (obj.payload, opts, (err, res) => {
         if (err) {
           debug ('while moving %s to deadletter: %j', obj._id, err);
           return cb (err, 'deadletter');
         }
         else {
           debug ('moved %s to deadletter with _id %s', obj._id, res);
+          this._stats.incr ('deadletter');
           return cb (null, 'deadletter');
         }
       });

package/TODO

@@ -1,5 +1,6 @@
 bugs
 -------------------------------------------------------------
+* deadlettering on all backend but mongo-pipeline does not preserve hdrs
 
 prio 1
 -------------------------------------------------------------

package/backends/ps-mongo.js

@@ -217,6 +217,7 @@ class PersistentMongoQueue extends Queue {
     try {
       query =  {
         _id: (_.isString(id) ? new mongo.ObjectID (id) : id),
+        processed: {$exists: false},
         reserved: {$exists: false}
       };
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "keuss",
-  "version": "1.6.8",
+  "version": "1.6.11",
   "keywords": [
     "queue",
     "persistent",
@@ -27,10 +27,10 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@nodebb/mubsub": "~1.7.1",
-    "async": "~3.2.1",
-    "async-lock": "~1.3.0",
-    "debug": "~4.3.2",
-    "ioredis": "~4.27.10",
+    "async": "~3.2.3",
+    "async-lock": "~1.3.1",
+    "debug": "~4.3.3",
+    "ioredis": "~4.28.5",
     "lodash": "~4.17.21",
     "mitt": "~3.0.0",
     "mongodb": "~3.6.12",
@@ -38,7 +38,7 @@
   },
   "devDependencies": {
     "chance": "~1.1.8",
-    "mocha": "~9.1.2",
+    "mocha": "~9.2.1",
     "should": "~13.2.3"
   },
   "scripts": {

package/test/backends_bucket-at-least-once.js

@@ -52,7 +52,7 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
       function (cb) {q.totalSize (cb)},
       function (cb) {q.next_t (cb)},
     ], function(err, results) {
-      results.should.eql ([{get: 0, put: 0, reserve: 0, commit: 0, rollback: 0}, 0, 0, null])
+      results.should.eql ([{get: 0, put: 0, reserve: 0, commit: 0, rollback: 0, deadletter: 0}, 0, 0, null])
       done();
     });
   });
@@ -70,7 +70,7 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
         cb();
       })},
       function (cb) {q.stats (function (err, res) {
-        res.should.eql ({get: 0, put: 2, reserve: 0, commit: 0, rollback: 0});
+        res.should.eql ({get: 0, put: 2, reserve: 0, commit: 0, rollback: 0, deadletter: 0});
         cb();
       })},
       function (cb) {q.next_t (function (err, res) {
@@ -86,7 +86,7 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
         cb();
       })},
       function (cb) {q.stats (function (err, res) {
-        res.should.eql ({get: 1, put: 2, reserve: 0, commit: 0, rollback: 0});
+        res.should.eql ({get: 1, put: 2, reserve: 0, commit: 0, rollback: 0, deadletter: 0});
         cb();
       })},
       function (cb) {q.pop ('c2', function (err, ret) {
@@ -98,7 +98,7 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
         cb();
       })},
       function (cb) {q.stats (function (err, res) {
-        res.should.eql ({get: 2, put: 2, reserve: 0, commit: 0, rollback: 0});
+        res.should.eql ({get: 2, put: 2, reserve: 0, commit: 0, rollback: 0, deadletter: 0});
         cb();
       })},
     ], function(err, results) {
@@ -139,7 +139,7 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
         cb();
       })},
       function (cb) {q.stats (function (err, res) {
-        res.should.eql ({get: 2, put: 2, reserve: 0, commit: 0, rollback: 0});
+        res.should.eql ({get: 2, put: 2, reserve: 0, commit: 0, rollback: 0, deadletter: 0});
         cb();
       })},
     ], function(err, results) {
@@ -160,7 +160,7 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
       (cb) => setTimeout (cb, 666),
 
       (cb) => q.size ((err, size) => {size.should.equal (4); cb(); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 4, reserve: 0, commit: 0, rollback: 0}); cb(); }),
+      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 4, reserve: 0, commit: 0, rollback: 0, deadletter: 0}); cb(); }),
 
       (cb) => q.push ({elem:5, pl:'twetrwte'}, cb),
       (cb) => q.push ({elem:6, pl:'twetrwte'}, cb),
@@ -168,7 +168,7 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
       (cb) => setTimeout (cb, 666),
 
       (cb) => q.size ((err, size) => {size.should.equal (7); cb(); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 7, reserve: 0, commit: 0, rollback: 0}); cb(); }),
+      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 7, reserve: 0, commit: 0, rollback: 0, deadletter: 0}); cb(); }),
 
       (cb) => q.next_t ((err, res) => {res.getTime().should.be.approximately(new Date().getTime(), 2000); cb(); }),
 
@@ -181,7 +181,7 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
       (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:7, pl:'twetrwte'}); cb (err); })},
 
       (cb) => q.size ((err, size) => {size.should.equal (0); cb(); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 7, put: 7, reserve: 0, commit: 0, rollback: 0}); cb(); }),
+      (cb) => q.stats ((err, res) => {res.should.eql ({get: 7, put: 7, reserve: 0, commit: 0, rollback: 0, deadletter: 0}); cb(); }),
     ], function(err, results) {
       q.drain();
       done();
@@ -211,7 +211,7 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
       (cb) => q.size ((err, size) => {size.should.equal (0); cb(); }),
       (cb) => q.totalSize ((err, size) => {size.should.equal (7); cb(); }),
       (cb) => q.schedSize ((err, size) => {size.should.equal (7); cb(); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 7, reserve: 0, commit: 0, rollback: 0}); cb(); }),
+      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 7, reserve: 0, commit: 0, rollback: 0, deadletter: 0}); cb(); }),
 
       (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:5, pl:'twetrwte'}); cb (err); })},
       (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:6, pl:'twetrwte'}); cb (err); })},
@@ -222,7 +222,7 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
       (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:4, pl:'twetrwte'}); cb (err); })},
 
       (cb) => q.size ((err, size) => {size.should.equal (0); cb(); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 7, put: 7, reserve: 0, commit: 0, rollback: 0}); cb(); }),
+      (cb) => q.stats ((err, res) => {res.should.eql ({get: 7, put: 7, reserve: 0, commit: 0, rollback: 0, deadletter: 0}); cb(); }),
     ], function(err, results) {
       q.drain();
       done();
@@ -239,13 +239,13 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
       (cb) => setTimeout (cb, 700),
 
       (cb) => q.size ((err, size) => {size.should.equal (2); cb(); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 2, reserve: 0, commit: 0, rollback: 0}); cb(); }),
+      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 2, reserve: 0, commit: 0, rollback: 0, deadletter: 0}); cb(); }),
 
       (cb) => q.push ({elem:3, pl:'twetrwte'}, cb),
       (cb) => setTimeout (cb, 700),
 
       (cb) => q.size ((err, size) => {size.should.equal (3); cb(); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 3, reserve: 0, commit: 0, rollback: 0}); cb(); }),
+      (cb) => q.stats ((err, res) => {res.should.eql ({get: 0, put: 3, reserve: 0, commit: 0, rollback: 0, deadletter: 0}); cb(); }),
 
       (cb) => q.next_t ((err, res) => {res.getTime().should.be.approximately(new Date().getTime() - 1400, 100); cb(); }),
 
@@ -283,7 +283,7 @@ describe ('bucket-at-least-once with ' + MQ_item.label + ' queue backend', funct
       (cb) => {q.pop ('c1', (err, ret) => {ret.payload.should.eql ({elem:1, pl:'twetrwte'}); cb (err); })},
 
       (cb) => q.size ((err, size) => {size.should.equal (0); cb(); }),
-      (cb) => q.stats ((err, res) => {res.should.eql ({get: 3, put: 3, reserve: 2, commit: 0, rollback: 2}); cb(); }),
+      (cb) => q.stats ((err, res) => {res.should.eql ({get: 3, put: 3, reserve: 2, commit: 0, rollback: 2, deadletter: 0}); cb(); }),
     ], function(err, results) {
       q.drain();
       done();

package/test/backends_deadletter.js

@@ -111,6 +111,8 @@ function release_mq_factory (q, factory, cb) {
         pl: 'twetrwte'
       };
 
+      const hdrs = {aaa: 'qw', bbb: '666'};
+
       async.waterfall ([
         cb => get_mq_factory (MQ, factory_opts, cb),
         (factory, cb) => {
@@ -120,7 +122,7 @@ function release_mq_factory (q, factory, cb) {
 
           async.race ([
             cb => async.series([
-              cb => q.push (pl, cb),
+              cb => q.push (pl, {hdrs}, cb),
               cb => pop (q, stage, cb),
               cb => reject (q, stage, (err) => {tries++;cb()}),
               cb => pop (q, stage, cb),
@@ -139,6 +141,13 @@ function release_mq_factory (q, factory, cb) {
             }),
             cb => factory.deadletter_queue().pop('c2', (err, res) => {
               res.payload.should.eql (pl);
+              res.hdrs.should.match ({
+                aaa: "qw",
+                bbb: "666",
+                'x-dl-from-queue': "test_queue_deadletter",
+                'x-dl-t': /.+/,
+                'x-dl-tries': 4
+              });
               tries.should.equal (5);
               cb (err);
             })

package/test/backends_remove.js

@@ -19,39 +19,6 @@ function stats (q, cb) {
   });
 }
 
-function pop (q, stage, cb) {
-  q.pop('c1', { reserve: true }, (err, res) => {
-    stage.obj = res;
-//    console.log('reserved element %o', res);
-    cb(err);
-  });
-}
-
-function reject (q, stage, cb) {
-  var next_t = new Date().getTime() + 2000;
-
-  q.ko (stage.obj, next_t, (err, res) => {
-    if (err) {
-//      console.error ('error in rollback of %s: %j', stage.obj._id, err);
-      return cb (err);
-    }
-
-//    console.log('rolled back element %s: %o', stage.obj._id, res);
-    cb();
-  });
-}
-
-function accept (q, stage, cb) {
-  q.ok (stage.obj, (err, res) => {
-    if (err) {
-//      console.error ('error in rollback of %s: %j', stage.obj._id, err);
-      return cb (err);
-    }
-
-//    console.log('commited element %s: %j', stage.obj._id, res);
-    cb();
-  });
-}
 
 function get_mq_factory (MQ, opts, cb) {
   const common_opts = {
@@ -163,7 +130,7 @@ function release_mq_factory (q, factory, cb) {
           ctx.id = res[0];
           ctx.id.should.not.be.null();
           res[2].should.eql ({
-            stats: { get: 0, put: 1, reserve: 0, commit: 0, rollback: 0 },
+            stats: { get: 0, put: 1, reserve: 0, commit: 0, rollback: 0, deadletter: 0 },
             tsize: 1,
             rsize: is_redis ? null : 0
           })
@@ -181,7 +148,7 @@ function release_mq_factory (q, factory, cb) {
           cb => setTimeout (cb, 1000),
         ], (err, res) => {
           res[0].should.eql ({
-            stats: { get: 0, put: 1, reserve: 0, commit: 0, rollback: 0 },
+            stats: { get: 0, put: 1, reserve: 0, commit: 0, rollback: 0, deadletter: 0 },
             tsize: 0,
             rsize: is_redis ? null : 0
           });
@@ -228,7 +195,7 @@ function release_mq_factory (q, factory, cb) {
           cb => setTimeout (cb, 1000),
         ], (err, res) => {
           res[0].should.eql ({
-            stats: { get: 0, put: 1, reserve: 1, commit: 0, rollback: 0 },
+            stats: { get: 0, put: 1, reserve: 1, commit: 0, rollback: 0, deadletter: 0 },
             tsize: 1,
             rsize: is_redis ? null : 1
           });
@@ -282,7 +249,61 @@ function release_mq_factory (q, factory, cb) {
           cb => setTimeout (cb, 1000),
         ], (err, res) => {
           res[0].should.eql ({
-            stats: { get: 0, put: 1, reserve: 1, commit: 0, rollback: 1 },
+            stats: { get: 0, put: 1, reserve: 1, commit: 0, rollback: 1, deadletter: 0 },
+            tsize: 0,
+            rsize: is_redis ? null : 0
+          });
+          cb (err, q, factory);
+        }),
+        (q, factory, cb) => {
+          release_mq_factory (q, factory, cb);
+        }
+      ], done);
+    });
+
+
+    it('does fail trying to delete an already-deleted element', done => {
+      const ctx = {};
+      async.waterfall ([
+        cb => get_mq_factory (MQ, {}, cb),
+        (factory, cb) => {
+          const q = factory.queue('test_queue_remove', {});
+          cb (null, q, factory);
+        },
+        (q, factory, cb) => async.series ([
+          cb => q.push ({a: 1, b: 'oo'}, cb),
+          cb => setTimeout (cb, 1000),
+          cb => stats (q, cb)
+        ], (err, res) => {
+          ctx.id = res[0];
+          ctx.id.should.not.be.null();
+          res[2].should.eql ({
+            stats: { get: 0, put: 1, reserve: 0, commit: 0, rollback: 0, deadletter: 0 },
+            tsize: 1,
+            rsize: is_redis ? null : 0
+          })
+          cb (err, q, factory);
+        }),
+        (q, factory, cb) => {
+          q.remove (ctx.id, (err, res) => {
+            should (err).be.null();
+            res.should.be.true();
+            cb (err, q, factory);
+          });
+        },
+        (q, factory, cb) => {
+          q.remove (ctx.id, (err, res) => {
+            should (err).be.null();
+            res.should.be.false();
+            cb (err, q, factory);
+          });
+        },
+        (q, factory, cb) => async.series ([
+          cb => stats (q, cb),
+          cb => setTimeout (cb, 1000),
+        ], (err, res) => {
+          res[0].should.eql ({
+            stats: { get: 0, put: 1, reserve: 0, commit: 0, rollback: 0, deadletter: 0 },
             tsize: 0,
             rsize: is_redis ? null : 0
           });

package/test/backends_reserve-commit-rollback.js

@@ -59,7 +59,8 @@ var factory = null;
           put: 0,
           reserve: 0,
           commit: 0,
-          rollback: 0
+          rollback: 0, 
+          deadletter: 0
         }, 0, 0, 0, null]);
         done();
       });
@@ -83,7 +84,8 @@ var factory = null;
             put: 2,
             reserve: 0,
             commit: 0,
-            rollback: 0
+            rollback: 0, 
+            deadletter: 0
           });
           cb();
         }),
@@ -102,7 +104,8 @@ var factory = null;
             put: 2,
             reserve: 0,
             commit: 0,
-            rollback: 0
+            rollback: 0,
+            deadletter: 0
           });
           cb();
         }),
@@ -120,7 +123,8 @@ var factory = null;
             put: 2,
             reserve: 0,
             commit: 0,
-            rollback: 0
+            rollback: 0,
+            deadletter: 0
           });
           cb();
         }),
@@ -171,7 +175,8 @@ var factory = null;
               put: 2,
               reserve: 0,
               commit: 0,
-              rollback: 0
+              rollback: 0,
+              deadletter: 0
             });
             cb();
           })
@@ -204,7 +209,8 @@ var factory = null;
               put: 2,
               reserve: 0,
               commit: 0,
-              rollback: 0
+              rollback: 0,
+              deadletter: 0
             });
             cb();
           });
@@ -237,7 +243,8 @@ var factory = null;
               put: 2,
               reserve: 0,
               commit: 0,
-              rollback: 0
+              rollback: 0,
+              deadletter: 0
             });
             cb();
           })
@@ -291,7 +298,8 @@ var factory = null;
               put: 2,
               reserve: 0,
               commit: 0,
-              rollback: 0
+              rollback: 0,
+              deadletter: 0
             });
             cb();
           })
@@ -317,7 +325,8 @@ var factory = null;
               put: 2,
               reserve: 0,
               commit: 0,
-              rollback: 0
+              rollback: 0,
+              deadletter: 0
             });
             cb();
           })
@@ -343,7 +352,8 @@ var factory = null;
               put: 2,
               reserve: 0,
               commit: 0,
-              rollback: 0
+              rollback: 0,
+              deadletter: 0
             });
             cb();
           })
@@ -383,7 +393,8 @@ var factory = null;
               put: 2,
               reserve: 0,
               commit: 0,
-              rollback: 0
+              rollback: 0,
+              deadletter: 0
             });
             cb();
           })
@@ -463,7 +474,8 @@ var factory = null;
               put: 2,
               reserve: 0,
               commit: 0,
-              rollback: 0
+              rollback: 0,
+              deadletter: 0
             });
             cb();
           })
@@ -519,7 +531,8 @@ var factory = null;
               put: 3,
               reserve: 0,
               commit: 0,
-              rollback: 0
+              rollback: 0,
+              deadletter: 0
             });
             cb();
           })
@@ -576,7 +589,8 @@ var factory = null;
               put: 3,
               reserve: 0,
               commit: 0,
-              rollback: 0
+              rollback: 0,
+              deadletter: 0
             });
             cb();
           })
@@ -849,7 +863,8 @@ var factory = null;
             put: 1,
             reserve: 1,
             commit: 1,
-            rollback: 0
+            rollback: 0,
+            deadletter: 0
           });
           cb();
         }),
@@ -986,7 +1001,8 @@ var factory = null;
             put: 1,
             reserve: 3,
             commit: 1,
-            rollback: 2
+            rollback: 2,
+            deadletter: 0
           });
           cb();
         }),