keuss 1.7.3 → 2.0.0

package/examples/pipelines/builder/index.js

@@ -1,99 +0,0 @@
-const Chance = require ('chance');
-const chance = new Chance();
-
-const MQ = require ('../../../backends/pl-mongo');
-
-function loop (n, fn, cb) {
-  if (n == 0) return cb ();
-  fn (n, err => {
-    if (err) return cb (err);
-    setImmediate (() => loop (n-1, fn, cb));
-  });
-}
-
-const factory_opts = {
-  url: 'mongodb://localhost/qeus'
-};
-
-function get_a_delay (min, max) {
-  return chance.integer ({min, max});
-}
-
-const num_elems = 37;
-let   processed = 0;
-
-function sink_process (elem, done) {
-  setTimeout (() => {
-    console.log ('%s: sunk elem [a %d, choice %d, tries %d]', this.name(), elem.payload.a, elem.payload.choice, elem.payload.choice_try);
-    done();
-
-    processed++;
-    if (processed == num_elems) {
-      setTimeout (() =>{
-        console.log ('processing done, exiting');
-        process.exit (0);
-      }, 1000);
-    }
-  }, get_a_delay (10, 20));
-}
-
-function dl_process (elem, done) {
-  // pass element to next queue, set payload.passed to true
-  done (null, {
-    update: {
-      $set: {stamp_0: 'passed'}
-    }
-  });
-}
-
-function choice_process (elem, done) {
-  setTimeout (() => {
-    if (chance.bool ({likelihood: 9})) {
-      console.log ('%s: failing on elem %o on try [%d]', this.name(), elem.payload, elem.tries);
-      return done ({e: 'cl1 induced a failure'});
-    }
-
-    const idx = chance.pickone([0, 1, 2]);
-    console.log ('%s: passing elem %o to %d on try [%d]', this.name(), elem.payload, idx, elem.tries);
-    done (null, {
-      dst: idx,
-      update: {
-        $set: {stamp_1: 'passed', choice: idx, choice_try: elem.tries}
-      }
-    });
-  }, get_a_delay (10, 20));
-}
-
-// initialize factory
-MQ (factory_opts, (err, factory) => {
-  if (err) return console.error (err);
-
-  const q_opts = {};
-
-  factory
-  .builder ()
-  .pipeline ('the-pipeline')
-  .queue ('test_pl_1', q_opts)
-  .queue ('test_pl_2', q_opts)
-  .queue ('test_pl_3', q_opts)
-  .queue ('test_pl_4', q_opts)
-  .queue ('test_pl_5', q_opts)
-  .directLink ('test_pl_1', 'test_pl_2', dl_process)
-  .choiceLink ('test_pl_2', ['test_pl_3', 'test_pl_4', 'test_pl_5'], choice_process)
-  .sink ('test_pl_3', sink_process)
-  .sink ('test_pl_4', sink_process)
-  .sink ('test_pl_5', sink_process)
-  .onError (console.log)
-  .done ((err, pl) => {
-    if (err) return console.error (err);
-    console.log ('pipeline IS READY')
-    pl.start ();
-    console.log ('pipeline IS RUNNING, inserting load')
-
-    loop (
-      num_elems,
-      (n, next) => setTimeout (() => pl.queues()['test_pl_1'].push ({a:n, b:'see it fail...'}, next), chance.integer ({min:0, max: 20})),
-      err => console.log (err || 'done')
-    );
-  });
-});

package/examples/pipelines/fromRecipe/index.js

@@ -1,127 +0,0 @@
-const Chance = require ('chance');
-
-
-const chance = new Chance();
-
-const MQ = require    ('../../../backends/pl-mongo');
-
-function loop (n, fn, cb) {
-  if (n == 0) return cb ();
-  fn (n, err => {
-    if (err) return cb (err);
-    setImmediate (() => loop (n-1, fn, cb));
-  });
-}
-
-const factory_opts = {
-  url: 'mongodb://localhost/qeus'
-};
-
-const bs_src_array = [
-  `
-  const Chance = require ('chance');
-  const chance = new Chance();
-  `,
-  `
-  function get_a_delay (min, max) {
-    return chance.integer ({min, max});
-  }
-  `,
-  `
-  function sink_process (elem, done) {
-    setTimeout (() => {
-      console.log ('%s: sunk elem [a %d, choice %d, tries %d]', this.name(), elem.payload.a, elem.payload.choice, elem.payload.choice_try);
-      done();
-
-      processed++;
-      if (processed == num_elems) {
-        setTimeout (() =>{
-          console.log ('processing done, exiting');
-          process.exit (0);
-        }, 1000);
-      }
-
-    }, get_a_delay (10, 20));
-  }
-
-  function dl_process (elem, done) {
-    // pass element to next queue, set payload.passed to true
-    done (null, {
-      update: {
-        $set: {stamp_0: 'passed'}
-      }
-    });
-  }
-
-  function choice_process (elem, done) {
-    setTimeout (() => {
-      if (chance.bool ({likelihood: 9})) {
-        console.log ('%s: failing on elem %o on try [%d]', this.name(), elem.payload, elem.tries);
-        return done ({e: 'cl1 induced a failure'});
-      }
-
-      const idx = chance.pickone([0, 1, 2]);
-      console.log ('%s: passing elem %o to %d on try [%d]', this.name(), elem.payload, idx, elem.tries);
-      done (null, {
-        dst: idx,
-        update: {
-          $set: {stamp_1: 'passed', choice: idx, choice_try: elem.tries}
-        }
-      });
-    }, get_a_delay (10, 20));
-  }
-  `
-];
-
-const setup_src_array = [
-  `
-  const q_opts = {};
-
-  builder
-  .queue ('test_pl_1', q_opts)
-  .queue ('test_pl_2', q_opts)
-  .queue ('test_pl_3', q_opts)
-  .queue ('test_pl_4', q_opts)
-  .queue ('test_pl_5', q_opts)
-  .directLink ('test_pl_1', 'test_pl_2', dl_process)
-  .choiceLink ('test_pl_2', ['test_pl_3', 'test_pl_4', 'test_pl_5'], choice_process)
-  .sink ('test_pl_3', sink_process)
-  .sink ('test_pl_4', sink_process)
-  .sink ('test_pl_5', sink_process)
-  .onError (console.log)
-  .done (done);
-  `
-];
-
-const num_elems = 3;
-let   processed = 0;
-
-// initialize factory
-MQ (factory_opts, (err, factory) => {
-  if (err) return console.error (err);
-
-  factory.pipelineFromRecipe (
-    'a_test_etcher',
-    bs_src_array,
-    setup_src_array,
-    {
-      context: {
-        num_elems,
-        processed,
-        process,
-        console
-      }
-    }, (err, pl) => {
-      if (err) return console.error (err);
-      console.log ('pipeline IS READY')
-      pl.start ();
-      console.log ('pipeline IS RUNNING, inserting load')
-
-      loop (
-        num_elems,
-        (n, next) => setTimeout (() => pl.queues()['test_pl_1'].push ({a:n, b:'see it fail...'}, next), chance.integer ({min:0, max: 20})),
-        err => console.log (err || 'done')
-      );
-    }
-  );
-});

package/examples/pipelines/simplest/index.js

@@ -1,38 +0,0 @@
-var MQ = require    ('../../../backends/pl-mongo');
-var PDL = require   ('../../../Pipeline/DirectLink');
-var async = require ('async');
-
-var factory_opts = {
-  url: 'mongodb://localhost/qeus'
-};
-
-// initialize factory
-MQ (factory_opts, (err, factory) => {
-  if (err) return console.error (err);
-
-  // factory ready, create 2 queues on default pipeline
-  var q_opts = {};
-  var q1 = factory.queue ('test_pl_1', q_opts);
-  var q2 = factory.queue ('test_pl_2', q_opts);
-
-  // tie them up, q1 -> q2
-  var pdl = new PDL (q1, q2);
-
-  pdl.start (function (elem, done) {
-    // pass element to next queue, set payload.passed to true
-    done (null, {
-      update: {
-        $set: {passed: true}
-      }
-    });
-  });
-
-  // insert elements in the entry queue
-  async.timesLimit (3, 3, (n, next) => q1.push ({a:n, b:'see it spin...'}, next));
-
-  // read elements at the outer end
-  async.timesLimit (3, 3, (n, next) => q2.pop ('exit', (err, res) => {
-    console.log ('end point get', res);
-    next ();
-  }));
-});

package/examples/pipelines/simulation-fork/index.js

@@ -1,115 +0,0 @@
-// mongodb: create a consumer and a producer
-const Chance = require ('chance');
-
-const MQ =  require ('../../../backends/pl-mongo');
-const DCT = require ('../../../Pipeline/DirectLink');
-const SNK = require ('../../../Pipeline/Sink');
-const CHC = require ('../../../Pipeline/ChoiceLink');
-
-
-const chance = new Chance();
-
-const num_elems = 100;
-let   processed = 0;
-
-
-function get_a_delay (min, max) {
-  return chance.integer ({min, max});
-}
-
-function loop (n, fn, cb) {
-  if (n == 0) return cb ();
-  fn (n, err => {
-    if (err) return cb (err);
-    setImmediate (() => loop (n-1, fn, cb));
-  });
-}
-
-function sink_process (elem, done) {
-  setTimeout (() => {
-    console.log ('%s: sunk elem [a %d, choice %d]', this.name(), elem.payload.a, elem.payload.choice);
-    done();
-
-    processed++;
-    if (processed == num_elems) {
-      setTimeout (() =>{
-        console.log ('processing done, exiting');
-        process.exit (0);
-      }, 1000);
-    }
-
-  }, get_a_delay (10, 20));
-}
-
-
-const factory_opts = {
-  url: 'mongodb://localhost/qeus_pl',
-  options: { useNewUrlParser: true },
-  deadletter: {
-    max_ko: 3
-  }
-};
-
-// initialize factory
-MQ (factory_opts, (err, factory) => {
-  if (err) return console.error (err);
-
-  // factory ready, create queues on default pipeline
-  const q_opts = {aaa: 666, b: 'yy'};
-  const q1 = factory.queue ('pl_many_q_1', q_opts);
-  const q2 = factory.queue ('pl_many_q_2', q_opts);
-  const q3 = factory.queue ('pl_many_q_3', q_opts);
-  const q4 = factory.queue ('pl_many_q_4', q_opts);
-  const q5 = factory.queue ('pl_many_q_5', q_opts);
-
-  // tie them up:
-  const dl1 = new DCT (q1, q2);
-  const cl1 = new CHC (q2, [q3, q4, q5]);
-  const sk1 = new SNK (q3);
-  const sk2 = new SNK (q4);
-  const sk3 = new SNK (q5);
-
-  sk1.on_data (sink_process);
-  sk2.on_data (sink_process);
-  sk3.on_data (sink_process);
-
-  cl1.on_data (function (elem, done) {
-    setTimeout (() => {
-      if (chance.bool ({likelihood: 90})) {
-        return done ({e: 'cl1 induced a failure'});
-      }
-
-      const idx = chance.pickone([0, 1, 2]);
-      console.log ('%s: passing elem %o to %d on try [%d]', this.name(), elem.payload, idx, elem.tries);
-      done (null, {
-        dst: idx,
-        update: {
-          $set: {stamp_1: 'passed', choice: idx}
-        }
-      });
-    }, get_a_delay (10, 100));
-  });
-
-  dl1.on_data (function (elem, done) {
-    setTimeout (() => {
-      console.log ('%s: passing elem %o', this.name(), elem.payload);
-      done (null, {
-        update: {
-          $set: {stamp_0: 'passed'}
-        }
-      });
-    }, get_a_delay (10, 100));
-  });
-
-  sk1.start ();
-  sk2.start ();
-  sk3.start ();
-  cl1.start ();
-  dl1.start ();
-
-  loop (
-    num_elems,
-    (n, next) => setTimeout (() => q1.push ({a:n, b:'see it fail...'}, next), chance.integer ({min:0, max: 20})),
-    err => console.log (err || 'done')
-  );
-});

package/examples/snippets/01-simplest-pop-push.js

@@ -1,49 +0,0 @@
-/*
- *
- * very simple example of push & pop: an element is pushed, and then popped
- *
- */
-
-const async = require ('async');
-const MQ =    require ('../../backends/mongo');
-
-// initialize factory
-MQ ({
-  url: 'mongodb://localhost/keuss_test'
-}, (err, factory) => {
-  if (err) return console.error(err);
-
-  // factory ready, create one queue
-  const q = factory.queue ('test_queue', {});
-
-  async.series([
-  // push element
-    cb => q.push (
-      {elem: 1, headline: 'something something', tags: {a: 1, b: 2}}, // this is the payload
-      {
-        hdrs: {h1: 'aaa', h2: 12, h3: false}  // let's add some headers too
-      },
-      cb
-    ),
-  // pop element
-    cb => q.pop ('consumer-1', cb)
-  ], (err, res) => {
-    if (err) {
-      console.error (err);
-    }
-    else {
-      console.log (res[1]);
-      // this should print something like:
-      // {
-      //   _id: <some id>,
-      //   mature: <some date>,
-      //   payload: { elem: 1, headline: 'something something', tags: { a: 1, b: 2 } },
-      //   tries: 0
-      //   hdrs: {h1: 'aaa', h2: 12, h3: false}
-      // }
-    }
-
-    factory.close ();
-  });
-});
-

package/examples/snippets/02-simplest-reserve-rollback-commit.js

@@ -1,44 +0,0 @@
-/*
- * 
- * very simple example of push-reserve-rollback-reserve-commit: one element is pushed,
- * then reserved, then rolled back (that is, rejected ad back to queue with a delay)
- * Then, it is reserved again and then, finally, committed (and removed fro the queue)
- * 
- */
-
-const async = require ('async');
-const MQ =    require ('../../backends/mongo');
-
-// initialize factory
-MQ ({
-  url: 'mongodb://localhost/keuss_test'
-}, (err, factory) => {
-  if (err) return console.error(err);
-
-  // factory ready, create one queue
-  const q = factory.queue ('test_queue', {});
-
-  async.waterfall ([
-  // element is pushed to queue
-    cb => q.push ({elem: 1, headline: 'something something', tags: {a: 1, b: 2}}, cb),  
-  // element is reserved...
-    (item_id, cb) => q.pop ('consumer-1', {reserve: true}, cb),                         
-  // ... and rejected back to queue with a delay of 1.5 sec
-    (item, cb) => {
-      console.log ('%s: got %o', new Date().toString (), item);                        
-      const next_t = new Date().getTime () + 1500;
-      q.ko (item, next_t, cb);                                                        
-    },
-  // Element is reserved gain. It will not finish until 1.5 secs have passed
-    (ko_res, cb) => q.pop ('consumer-1', {reserve: true}, cb),              
-  // and finally, it is committed
-    (item, cb) => {
-      console.log ('%s: got %o', new Date().toString (), item);                     
-      q.ok (item, cb);                                                             
-    },
-  ], (err, res) => {
-    if (err) console.error (err);
-    factory.close ();
-  });
-});
-

package/examples/snippets/03-simplest-producer-consumer-loops.js

@@ -1,77 +0,0 @@
-/*
- * 
- * simplest-possible producer and consumer loops: a loop of parallel producers push N elements in a queue; 
- * another loop of parallel consumers pop the elements out of the queue 
- * 
- * Queue stats are also shown every second
- * 
- */
-
-
-const async = require ('async');
-
-// choice of backend
-const MQ =    require ('../../backends/bucket-mongo-safe');
-//const MQ =    require ('../../backends/redis-oq');
-//const MQ =    require ('../../backends/mongo');
-//const MQ =    require ('../../backends/ps-mongo');
-
-// initialize factory
-MQ ({
-  url: 'mongodb://localhost/keuss_test'
-}, (err, factory) => {
-  if (err) return console.error(err);
-
-  const consumers = 3;
-  const producers = 3;
-  const msgs = 100000;
-
-  // factory ready, create one queue
-  const q = factory.queue ('test_queue', {});
-
-  // show stats every sec
-  const timer = setInterval (() => {
-    q.stats ((err, res) => console.log ('  --> stats now: %o', res));
-  }, 1000);
-
-  async.parallel ([
-    // producers' loop, with 'producers' parallel producers
-    cb => async.timesLimit (msgs, producers, (n, next) => {
-      q.push ({elem: n, headline: 'something something', tags: {a: 1, b: 2}}, next);
-    }, err => {
-      console.log ('producer loop ended');
-      cb (err);
-    }),
-    // consumers' loop, with 'consumers' parallel consumers
-    cb => async.timesLimit (msgs, consumers, (n, next) => {
-      q.pop ('theconsumer', {reserve: true}, (err, item) => {
-        if (err) return cb (err);
-        q.ok (item, next);
-      });
-    }, err => {
-      console.log ('consumer loop ended');
-      cb (err);
-    })
-  ], err => {
-    if (err) return console.error (err);
-
-    clearInterval (timer);
-
-    // all loops completed, cleanup & show stats
-    async.series ([
-      cb => q.drain (cb),
-      cb => q.stats (cb),
-      cb => setTimeout (cb, 1000),
-      cb => q.stats (cb),
-    ], (err, res) => {
-      if (err) console.error (err);
-      else {
-        console.log ('stats right after drain: %o', res[1]);
-        console.log ('stats once dust settled: %o', res[3]);
-      }
-
-      factory.close ();
-    });
-  });
-});
-

package/examples/snippets/04-bucket-mongo-safe-insert-reserve-commit.js

@@ -1,78 +0,0 @@
-/*
- * 
- * simple sequence of insert, reserve, rollback and commits over a bucket-mongo-safe queue
- * every now and then queue stats are shown; a delay is added to ensure stats' flow happens
- * 
- */
-
-const async = require('async');
-const MQ = require('../../backends/bucket-mongo-safe');
-
-const factory_opts = {
-  url: 'mongodb://localhost/qeus',
-  reserve_delay: 7
-};
-
-// initialize factory 
-MQ(factory_opts, (err, factory) => {
-  if (err) return console.error(err);
-
-  // factory ready, create one queue
-  const q = factory.queue('test_queue_rcr', {});
-
-  let id = null;
-
-  async.series([
-    cb => q.push({ elem: 1, pl: 'twetrwte' }, cb),
-    cb => q.push({ elem: 2, pl: 'twetrwte' }, cb),
-    cb => q.push({ elem: 3, pl: 'twetrwte' }, cb),
-    cb => setTimeout (cb, 500),
-    cb => q.stats ((err, res) => {
-        console.log('queue stats now: %o', res);
-        cb(err);
-    }),
-   cb => q.pop ('c1', {reserve: true}, (err, res) => {
-        id = res._id;
-        console.log('reserved element %o, id is %s', res, id)
-        cb(err);
-    }),
-    cb => setTimeout (cb, 500),
-    cb => q.stats ((err, res) => {
-        console.log('queue stats now: %o', res);
-        cb(err);
-    }),
-    cb => q.ko(id, (err, res) => {
-      console.log('rolled back element %s -> %s', id, res);
-      cb();
-    }),
-    cb => setTimeout (cb, 500),
-    cb => q.stats ((err, res) => {
-        console.log('queue stats now: %o', res);
-        cb(err);
-    }),
-    cb => q.pop ('c1', {reserve: true}, (err, res) => {
-      id = res._id;
-      console.log('reserved element %o, id is %s', res, id)
-      cb(err);
-    }),
-    cb => setTimeout (cb, 500),
-    cb => q.stats ((err, res) => {
-      console.log('queue stats now: %o', res);
-      cb(err);
-    }),
-    cb => q.ok (id, (err, res) => {
-      console.log('commited element %s -> %s', id, res);
-      cb();
-    }),
-    cb => setTimeout (cb, 1000),
-    cb => q.stats ((err, res) => {
-      console.log('queue stats now: %o', res);
-      cb(err);
-    }),
-    cb => q.drain (cb),
-    cb => {factory.close(); cb ();},
-  ], err => {
-    // all done
-    if (err) console.log ('error: ', err)
-  });
-});