keuss 1.7.3 → 2.0.0

package/examples/snippets/05-insert-reserve-rollback-deadletter.js

@@ -1,105 +0,0 @@
-/*
- * 
- * deadletter simple demo: an item is repeatedly reserved and rejected, and 
- * after some rejections, the element goes to the __deadletter__ instead of 
- * going back to its queue (which is caught at the pop at line 98)
- * 
- */
-
-// var MQ = require('../../backends/bucket-mongo-safe');
-// var MQ = require('../../backends/redis-oq');
-var MQ = require('../../backends/ps-mongo');
-
-var async = require('async');
-
-
-function stats (q, cb) {
-  q.stats((err, res) => {
-    console.log('queue stats now: %o', res);
-    cb(err);
-  });
-}
-
-function pop (q, stage, cb) {
-  q.pop('c1', { reserve: true }, (err, res) => {
-    stage.obj = res;
-    console.log('reserved element %o', res);
-    cb(err);
-  });
-}
-
-function reject (q, stage, cb) {
-  var next_t = new Date().getTime() + 2000;
-
-  q.ko (stage.obj, next_t, (err, res) => {
-    if (err) {
-      console.error ('error in rollback of %s: %o', stage.obj._id, err);
-      return cb (err);
-    }
-
-    console.log('rolled back element %s: %o', stage.obj._id, res);
-    cb();
-  });
-}
-
-function accept (q, stage, cb) {
-  q.ok (stage.obj, (err, res) => {
-    if (err) {
-      console.error ('error in rollback of %s: %o', stage.obj._id, err);
-      return cb (err);
-    }
-
-    console.log('commited element %s: %o', stage.obj._id, res);
-    cb();
-  });
-}
-
-
-var factory_opts = {
-  url: 'mongodb://localhost/qeus',
-  deadletter: {
-    max_ko: 3
-  }
-};
-
-// initialize factory
-MQ(factory_opts, (err, factory) => {
-  if (err) return console.error(err);
-
-  // factory ready, create one queue
-  var q = factory.queue('test_queue_rcr', {});
-  
-  // state is kept here
-  var stage = {};
-
-  async.series([
-    cb => q.push({ elem: 1, pl: 'twetrwte' }, (err, res) => {
-      console.log('pushed element');
-      cb(err);
-    }),
-    cb => pop (q, stage, cb),
-    cb => reject (q, stage, cb),
-    cb => pop (q, stage, cb),
-    cb => reject (q, stage, cb),
-    cb => pop (q, stage, cb),
-    cb => reject (q, stage, cb),
-    cb => pop (q, stage, cb),
-    cb => reject (q, stage, cb),
-    cb => pop (q, stage, cb),
-    cb => reject (q, stage, cb),
-    cb => pop (q, stage, cb),
-    cb => reject (q, stage, cb)
-  ], err => {
-    // all done
-    if (err) console.error ('reject_line error: ', err);
-    else console.log ('reject_line done');
-  });
-
-  factory.deadletter_queue().pop('c2', (err, res) => {
-    console.log('from deadletter_queue, got element %o', res);
-    setTimeout (() => {
-      q.cancel ();
-      factory.close (() => console.log ('done'));
-    }, 1000);
-  });
-});

package/examples/snippets/06-random-consumer-producer.js

@@ -1,270 +0,0 @@
-/*
- * 
- * Runs a set of producers and consumers, where consumers do a reserve-commit-rollback cycle. In each cycle a 
- * consumer would randomly choose whether to commit or rollback (10% chance of rollback). No deadletter is used
- * so elements are retried ad infinitum until processed ok
- * 
- * Upon completion (all items generated and consumed ok) some stats will be shown
- * 
- * It uses bucket-mongo-safe backend for high throughput
- * 
-*/
-
-var MQ = require ('../../backends/bucket-mongo-safe');
-
-var _ =      require ('lodash');
-var async =  require ('async');
-var Chance = require ('chance');
-var chance = new Chance();
-
-
-var factory_opts = {
-  url: 'mongodb://localhost/qeus',
-  name: 'Random-NS',
-  reject_delta_base: 2000,
-  reject_delta_factor: 2000
-};
-
-// test dimensions: elems to produce and consume, number of consumers, number of producers
-const num_elems = 1000000;
-const num_producers = 3;
-const num_consumers = 7;
-const commit_likelihood = 75;
-
-// stats holder
-var selfs = {
-  consumers: {},
-  producers: {}
-};
-
-var shareds = {};
-
-
-//////////////////////////////////////////////////////////////////////////////////////////////////////////////
-function consume_one (shared_ctx, self_ctx, cb) {
-  if (shared_ctx.pop_count > shared_ctx.pop_max) {
-    console.log ('pop consumer %s ended', self_ctx.id);
-    shared_ctx.q.cancel ();
-    return cb ();
-  }
-
-  shared_ctx.pop_count ++;
-  self_ctx.pop_count ++;
-
-  shared_ctx.q.pop (self_ctx.id, shared_ctx.pop_opts, (err, res) => {
-    if (err) return cb (err);
-    if ((shared_ctx.pop_count % 100000) == 0) console.log ('%s : consumed #%d elems', self_ctx.id, shared_ctx.pop_count);
-    consume_one (shared_ctx, self_ctx, cb);
-  });
-}
-
-
-function run_a_pop_consumer (shared_ctx, cb) {
-  var self_ctx = {
-    id: chance.word (),
-    pop_count: 0
-  };
-
-  selfs.consumers[self_ctx.id] = self_ctx;
-
-  consume_one (shared_ctx, self_ctx, cb);
-}
-
-
-function run_consumers (q, cnt, cb) {
-  var shared_ctx = {
-    q: q,
-    pop_count: 0,
-    pop_max: cnt,
-    pop_opts: {}
-  };
-
-  shareds.pc = shared_ctx;
-
-  var tasks = [];
-  for (var i = 0; i < num_consumers; i++) tasks.push ((cb) => run_a_pop_consumer (shared_ctx, cb));
-  async.parallel (tasks, cb);
-}
-
-//////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-var das_pop = 0;
-var das_ko = 0;
-var das_ok = 0;
-
-//////////////////////////////////////////////////////////////////////////////////////////////////////////////
-function reserve_and_commit_one (shared_ctx, self_ctx, cb) {
-  self_ctx.pop_count ++;
-
-  shared_ctx.q.pop (self_ctx.consumer, shared_ctx.pop_opts, (err, res) => {
-    if (err) return cb (err);
-    self_ctx.resv_count ++;
-
-    das_pop++;
-
-    if (chance.bool({likelihood: commit_likelihood})) {
-      shared_ctx.q.ok (res._id, (err) => {
-        if (err) return cb (err);
-        self_ctx.ok_count ++;
-        shared_ctx.pop_count ++;
-        das_ok++;
-
-        if (shared_ctx.pop_count > shared_ctx.pop_max) {
-          console.log ('rcr consumer %s ended', self_ctx.id);
-          shared_ctx.q.cancel ();
-          return cb ();
-        }
-
-//        if ((shared_ctx.pop_count % 100000) == 0) console.log ('%s : consumed #%d elems', self_ctx.id, shared_ctx.pop_count);
-        reserve_and_commit_one (shared_ctx, self_ctx, cb);
-      });
-    }
-    else {
-      shared_ctx.q.ko (res._id, (err) => {
-        if (err) return cb (err);
-        self_ctx.ko_count ++;
-        das_ko++;
-
-//        if ((shared_ctx.pop_count % 100000) == 0) console.log ('%s : consumed #%d elems', self_ctx.id, shared_ctx.pop_count);
-        reserve_and_commit_one (shared_ctx, self_ctx, cb);
-      });
-    }
-  });
-}
-
-function run_a_rcr_consumer (shared_ctx, cb) {
-  var self_ctx = {
-    id: chance.word (),
-    pop_count: 0,
-    resv_count: 0,
-    ko_count: 0,
-    ok_count: 0
-  };
-
-  selfs.consumers[self_ctx.id] = self_ctx;
-
-  reserve_and_commit_one (shared_ctx, self_ctx, cb);
-}
-
-function run_rcr_consumers (q, cnt, cb) {
-  var shared_ctx = {
-    q: q,
-    pop_count: 0,
-    pop_max: cnt,
-    pop_opts: {
-      reserve: true
-    }
-  };
-
-  shareds.rcrc = shared_ctx;
-
-  var tasks = [];
-  for (var i = 0; i < num_consumers; i++) 
-    tasks.push ((cb) => run_a_rcr_consumer (shared_ctx, cb));
-  
-  async.parallel (tasks, cb);
-}
-
-//////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-
-
-//////////////////////////////////////////////////////////////////////////////////////////////////////////////
-function produce_one (shared_ctx, self_ctx, cb) {
-  if (shared_ctx.push_count > shared_ctx.push_max) {
-    console.log ('producer %s ended', self_ctx.id);
-    return cb ();
-  }
-  
-  shared_ctx.push_count ++;
-  self_ctx.push_count ++;
-
-  shared_ctx.q.push ({a: chance.integer (), b: chance.word (), n: shared_ctx.push_count}, function (err, res) {
-    if (err) return console.error (err);
-//    if ((shared_ctx.push_count % 100000) == 0) console.log ('%s : produced #%d elems', self_ctx.id, shared_ctx.push_count);
-    produce_one (shared_ctx, self_ctx, cb);
-  });
-}
-
-function run_a_producer (shared_ctx, cb) {
-  var self_ctx = {
-    id: chance.word (),
-    push_count: 0
-  };
-
-  selfs.producers[self_ctx.id] = self_ctx;
-
-  produce_one (shared_ctx, self_ctx, cb);
-}
-
-function run_producers (q, cnt, cb) {
-  var shared_ctx = {
-    q: q,
-    push_count: 0,
-    push_max: cnt
-  };
-
-  shareds.prod = shared_ctx;
-
-  var tasks = [];
-  for (var i = 0; i < num_producers; i++) tasks.push ((cb) => run_a_producer (shared_ctx, cb));
-  async.parallel (tasks, cb);
-}
-
-//////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-
-// initialize factory 
-MQ (factory_opts, function (err, factory) {
-  if (err) return console.error (err);
-
-  // factory ready, create one queue
-  var q_opts = {};
-  var q = factory.queue ('test_queue_456', q_opts);
-
-  var timer = setInterval (() => {
-    console.log ('**** state: %o', _.map (shareds, (v, k) => {return {cnt: v.push_count || v.pop_count, max: v.push_max || v.pop_max }}));
-   
-    console.log ('das pop %d, ko %d, ok %d', das_pop, das_ko, das_ok);
-
-    q.status ((err ,res) => {
-      console.log ('**** status: %o', res);
-    });
-  }, 2000);
-
-  async.parallel ([
-    (cb) => run_producers (q, num_elems, cb),
-//    (cb) => run_consumers (q, 250000, cb),
-    (cb) => run_rcr_consumers (q, num_elems, cb),
-  ], (err) => {
-    if (err) {
-      console.error (err);
-    }
-    
-    var tot_push = 0;
-    var tot_pop = 0;
-    var tot_resv = 0;
-    var tot_ok = 0;
-    var tot_ko = 0;
-
-    console.log ('\nProducers: ');
-    _.each (selfs.producers, (v, k) => {console.log ('  %s: pushed %d', v.id, v.push_count), tot_push += v.push_count});
-    
-    console.log ('\nConsumers: ');
-    _.each (selfs.consumers, (v, k) => {
-      console.log ('  %s: popped %d, resvd %d, committed %d, rollbacked %d', v.id, v.pop_count, v.resv_count, v.ok_count, v.ko_count);
-      tot_pop += v.pop_count;
-      tot_resv += v.resv_count;
-      tot_ok += v.ok_count;
-      tot_ko += v.ko_count;
-    });
-    
-    console.log ('  \nTotals: %d popped, %d pushed, resvd %d, committed %d, rollbacked %d', tot_pop, tot_push, tot_resv, tot_ok, tot_ko);
-    clearInterval (timer);
-    
-    q.status ((err ,res) => {
-      console.log ('**** status: %o', res);
-      q.drain (() => factory.close ());
-    });
-  });
-});

package/examples/snippets/07-stream-simple.js

@@ -1,53 +0,0 @@
-/*
- *
- * very simple example of stream-mongo: one element pushed, consumed three times
- *
- */
-
-const async = require ('async');
-const MQ =    require ('../../backends/stream-mongo');
-
-// initialize factory
-MQ ({
-  url: 'mongodb://localhost/keuss_test_stream'
-}, (err, factory) => {
-  if (err) return console.error(err);
-
-  // factory ready, create one queue
-  const q0 = factory.queue ('test_stream', {groups: 'G1, G2, G4'});
-  const q1 = factory.queue ('test_stream', {group: 'G1'});
-  const q2 = factory.queue ('test_stream', {group: 'G2'});
-
-  async.series ([
-    // push element
-    cb => q0.push (
-      {elem: 1, headline: 'something something', tags: {a: 1, b: 2}}, // this is the payload
-      {
-        hdrs: {h1: 'aaa', h2: 12, h3: false}  // let's add some headers too
-      },
-      cb
-    ),
-    cb => setTimeout (cb, 1000),  // wait a bit
-    cb => q1.pop ('consumer-1', cb), // pop element in group G1
-    cb => q2.pop ('consumer-2', cb), // pop element in group G2
-  ], (err, res) => {
-    if (err) {
-      console.error (err);
-    }
-    else {
-      console.log ('element popped for group G1:', res[2]);
-      console.log ('element popped for group G2:', res[3]);
-      // this should print twice something like:
-      // {
-      //   _id: <some id>,
-      //   mature: <some date>,
-      //   payload: { elem: 1, headline: 'something something', tags: { a: 1, b: 2 } },
-      //   tries: 0
-      //   hdrs: {h1: 'aaa', h2: 12, h3: false}
-      // }
-    }
-
-    factory.close ();
-  });
-});
-

package/examples/snippets/redislabs-consumer-producer.js

@@ -1,44 +0,0 @@
-// redis-list consumer & producer on a redislabs Redis
-
-var MQ = require('../../backends/redis-list');
-
-var factory_opts = {
-  redis: {
-    Redis: {
-      port: 12345,
-      host: 'redis-12345.c3.eu-west-1-2.ec2.cloud.redislabs.com',
-      family: 4,
-      password: 'xxxxxxxx',
-      db: 0
-    }
-  }
-};
-
-// initialize factory 
-MQ(factory_opts, function (err, factory) {
-  if (err) {
-    return console.error(err);
-  }
-
-  // factory ready, create one queue
-  var q_opts = {};
-  var q = factory.queue('test_queue', q_opts);
-
-  // insert element
-  q.push({ a: 1, b: '666' }, function (err, res) {
-    if (err) {
-      return console.error(err);
-    }
-
-    // element inserted at this point. pop it again
-    var pop_opts = {};
-    q.pop('consumer-one', pop_opts, function (err, res) {
-      if (err) {
-        return console.error(err);
-      }
-
-      console.log('got this: ', res.payload);
-    });
-  });
-
-});

package/examples/snippets/with-redis-stats-and-signaller-consumer-producer.js

@@ -1,52 +0,0 @@
-// mongodb: create a consumer and a producer, use redis signaller and redis stats
-var MQ = require ('../backends/mongo');
-var signal_redis_pubsub = require ('../signal/redis-pubsub');
-var stats_redis = require ('../stats/redis');
-
-var local_redis_opts = {
-  Redis: {
-    port: 6379,
-    host: 'localhost',
-    db: 6
-  }
-};
-
-var factory_opts = {
-  url: 'mongodb://localhost/qeus',
-  signaller: {
-    provider: signal_redis_pubsub,
-    opts: local_redis_opts
-  },
-  stats: {
-    provider: stats_redis,
-    opts: local_redis_opts
-  }
-};
-    
-// initialize factory 
-MQ (factory_opts, function (err, factory) {
-  if (err) {
-    return console.error (err);
-  }
-
-  // factory ready, create one queue
-  var q_opts = {};
-  var q = factory.queue ('test_queue_123', q_opts);
-
-  // insert element
-  q.push ({a:1, b:'666'}, function (err, res) {
-    if (err) {
-      return console.error (err);
-    }
-
-    // element inserted at this point. pop it again
-    var pop_opts = {};
-    q.pop ('consumer-one', pop_opts, function (err, res) {
-      if (err) {
-        return console.error (err);
-      }
-
-      console.log ('got this: ', res.payload);
-    });
-  });
-});

package/examples/webhooks/README.md

@@ -1,36 +0,0 @@
-# Keuss example app: WebHook delivery
-
-This is a small but fully functional WebHook Delivery System using keuss as job-queue middleware. It can be certainly improved in many ways, but it is perfectly usable in its current state
-
-## install & run
-You need a mongodb instance running locally.
-
-Then, run the usual `npm install` and then `node .`
-
-## How it works
-This system works as a store-and-forward http relay:
-* you make HTTP calls (any method) to `http://localhost:6677/wh`. The whole HTTP request will be queued for later. You'll receive a `HTTP 201 Created` response, immediately
-* The queued requests are extracted by means of a reserve from the queue and *executed*.
-  * If they fail with a retriable error (http 5xx, non-http errors) they are rolled back with a delay of `tries^2 * 3 + tries * 3 + 3` seconds.
-  * If they fail with a non-retriable error (http 4xx) they are committed
-  * If they succeed (http 2xx) they are committed
-* Also, deadletter is used. If a webhook is retried over 14 times, it is moved to `__deadletter__`
-
-In order to work, you need to pass along the destination url as `x-dest-url` header. This header is not passed along. Also, you can specify an initial delay in secons in the header `x-delay`, which is not passed along either
-
-A set of testing urls is also provided for a self-contained testing:
-* `/test/200`: returns a HTTP 200
-* `/test/400`: returns a HTTP 400
-* `/test/404`: returns a HTTP 404
-* `/test/500`: returns a HTTP 500
-* `/test/drop`: no response, and close the socket
-
-Then, you can issue a POST webhook which would be retried 14 times and then moved to deadletter like this:
-
-```bash
-curl -X POST --data-bin @wh-payload.json -v -H 'x-dest-url: http://localhost:6677/test/500' -H 'content-type: text/plain' -H 'x-delay: 1' http://localhost:6677/wh
-```
-
-You would need to first create a file `wh-payload.json` with the webhook payload or content. Also, it will be issued with an initial delay of 1 second.
-The provided sample in the `wh-payload.json` file is a sample content for posting a message to a Microsoft Teams channel using webhooks connector. You can find the sample and more  concrete explanation of how to build your own cards for Microsoft Teams here:
-https://docs.microsoft.com/en-us/microsoftteams/platform/task-modules-and-cards/cards/cards-reference#thumbnail-card

package/examples/webhooks/app.js

@@ -1,70 +0,0 @@
-const express =    require ('express');
-const bodyParser = require ('body-parser');
-
-
-module.exports = (context) => {
-  var app = express();
-
-  // parse everything as text. A more robust and generic solution shoudl use raw() and manage Buffers, though
-  app.use (bodyParser.text ({type: () => true}));
-
-  // main server entry point: anything here is queued for async delivery
-  app.all ('/wh', (req, res) => {
-    const url = req.headers['x-dest-url'];
-    let delay = 0;
-
-    // we expect a header x-dest-url to specif the webhook's url
-    if (!url) return res.status (400).send ('no x-dest-url, ignoring request');
-
-    // one can specify the initial delay, if desired
-    const delay_str = req.headers['x-delay'];
-    if (delay_str) {
-      delete req.headers['x-delay'];
-      delay = parseInt (delay_str);
-    }
-
-    // build the payload...
-    delete req.headers['x-dest-url'];
-    const pl = {
-      url: url,
-      method: req.method,
-      headers: req.headers,
-      body: req.body
-    };
-
-    // ...and queue it
-    context.q.push (pl, {delay: delay}, (err, id) => {
-      // error while queuing?
-      if (err) {
-        console.error ('error whipe pushing payload:', err);
-        return res.status (500).send (err);
-      }
-
-      // no errors, return a 201 Created...
-      console.log ('inserted element with id %s', id);
-      return res.status (201).send ({res: 'ok', id: id});
-    });
-  });
-
-
-  // test respnses for various http response codes
-  app.all ('/test/200', (req, res) => res.status (200).send ('a 200'));
-  app.all ('/test/400', (req, res) => res.status (400).send ('a 400'));
-  app.all ('/test/404', (req, res) => res.status (404).send ('a 404'));
-  app.all ('/test/500', (req, res) => res.status (500).send ('a 500'));
-
-  // do not respond
-  app.all ('/test/noresponse', (req, res) => {});
-
-  // close socket
-  app.all ('/test/drop', (req, res) => req.socket.destroy());
-
-  // express error manager
-  app.use (function (err, req, res, next) {
-    console.error ('error caught: %s', err.stack);
-    res.status (err.status || 500).send (err.stack);
-  });
-
-  return app;
-};
-