keuss 1.7.4 → 2.0.1

package/bench/all-mongo.js

@@ -1,108 +0,0 @@
-
-var async = require ('async');
-
-var MQ = require('../backends/mongo');
-//var signal_mongo_capped = require('../signal/mongo-capped');
-var signal_mongo_capped = require('../signal/local');
-var stats_mongo = require('../stats/mongo');
-
-
-var factory_opts = {
-  url: 'mongodb://localhost/qeus',
-  signaller: {
-    provider: signal_mongo_capped,
-    opts: {
-      url: 'mongodb://localhost/qeus_signal',
-      channel: 'das_channel'
-    }
-  },
-  stats: {
-    provider: stats_mongo,
-    opts: {
-      url: 'mongodb://localhost/qeus_stats'
-    }
-  }
-};
-
-
-// initialize factory
-MQ(factory_opts, (err, factory) => {
-  if (err) {
-    return console.error(err);
-  }
-
-  var consumed = 0;
-  var produced = 0;
-
-  function run_consumer(q) {
-    q.pop('c1', {}, (err, res) => {
-//      console.log('consumer[%s]: got res %j', q.name(), res, {});
-      consumed++;
-
-      if (consumed > 100) {
-        factory.close();
-      } else {
-        if (!(consumed % 10)) console.log('< %d', consumed);
-
-        setTimeout(function () {
-          run_consumer(q);
-        }, 222);
-      }
-    });
-  }
-
-  function run_producer(q) {
-    q.push({
-      a: 1,
-      b: '666',
-      t: new Date(),
-      n: produced
-    }, (err, res) => {
-      produced++;
-
-      if (produced > 3) {
-
-      } else {
-        if (!(produced % 10)) console.log('> %d', produced);
-
-        setTimeout(function () {
-          run_producer(q);
-        }, 33);
-      }
-    });
-  }
-
-  var opts = {};
-
-  var q = factory.queue('bench_test_queue_0', opts);
-//  run_consumer(q);
-//  run_producer(q);
-
-  async.series ([
-    cb => q.push ({q:0, a: 'ryetyeryre 0'}, cb),
-    cb => q.push ({q:1, a: 'ryetyeryre 1'}, cb),
-    cb => q.pop ('me', (err, res) => {console.log ('>>>>>>> POP c0:', res); cb ();}),
-    cb => q.pop ('me', (err, res) => {console.log ('>>>>>>> POP c1:', res); cb ();}),
-
-    cb => {
-      q.pop ('me', (err, res) => console.log ('>>>>>>> POP c2:', res));
-      q.pop ('me', (err, res) => console.log ('>>>>>>> POP c3:', res));
-      q.pop ('me', (err, res) => console.log ('>>>>>>> POP c4:', res));
-      q.pop ('me', (err, res) => console.log ('>>>>>>> POP c5:', res));
-      q.pop ('me', (err, res) => console.log ('>>>>>>> POP c6:', res));
-      cb ();
-    },
-    cb => setTimeout (cb, 2000),
-    cb => {q.pause(true); cb ();},
-    cb => q.push ({q:2, a: 'ryetyeryre 2'}, cb),
-    cb => q.push ({q:3, a: 'ryetyeryre 3'}, cb),
-    cb => q.push ({q:4, a: 'ryetyeryre 4'}, cb),
-    cb => q.push ({q:5, a: 'ryetyeryre 5'}, cb),
-    cb => setTimeout (cb, 2000),
-    cb => {q.pause(false); cb ();},
-    cb => setTimeout (cb, 2000),
-  ], () => {
-    q.cancel();
-    factory.close();
-  });
-});

package/bench/multi-q.js

@@ -1,85 +0,0 @@
-var async =  require ('async');
-var should = require ('should');
-var Chance = require ('chance');
-
-var produced = 0;
-var consumed = 0;
-
-var chance = new Chance();
-
-
-//var MQ = require ('../backends/redis-oq');
-//var MQ = require ('../backends/redis-list');
-//var MQ = require ('../backends/mongo');
-var MQ = require ('../backends/pl-mongo');
-
-var redis_signaller = require ('../signal/redis-pubsub');
-var redis_stats =     require ('../stats/redis');
-
-var opts = {
-  signaller: {
-    provider: new redis_signaller ()
-  },
-  stats: {
-    provider: new redis_stats ()
-  }
-};
-    
-MQ (opts, function (err, factory) {
-  if (err) {
-    return console.error (err);
-  }
-
-  
-function run_consumer (q) {
-  q.pop ('c1', {}, function (err, res) {
-//    console.log ('consumer[%s]: got res %j', q.name(), res, {});
-    consumed++;
-
-    if (consumed > 1000000) {
-      factory.close ();
-    }
-    else {
-      if (!(consumed % 10000)) console.log ('< %d', consumed)
-//    setTimeout (function () {
-      run_consumer (q);
-//    }, chance.integer({ min: 200, max: 2000 }));
-    }
-  });
-}
-
-function run_producer (q) {
-  q.push ({a:1, b:'666'}, function (err, res) {
-    produced++;
-
-    if (produced > 1000000) {
-      
-    }
-    else {
-      if (!(produced % 10000)) console.log ('> %d', produced)
-//    setTimeout (function () {
-      run_producer (q);
-//    }, chance.integer({ min: 200, max: 2000 }));
-    }
-  });
-}
-
-
-
-  var q0 = factory.queue('bench_test_queue_0', opts);
-  run_consumer (q0);
-  run_producer (q0);
-/*
-  var q1 = factory.queue('bench_test_queue_1', opts);
-  run_consumer (q1);
-  run_producer (q1);
-
-  var q2 = factory.queue('bench_test_queue_2', opts);
-  run_consumer (q2);
-  run_producer (q2);
-
-  var q3 = factory.queue('bench_test_queue_3', opts);
-  run_consumer (q3);
-  run_producer (q3);
-*/
-});

package/bench/redis-oq-consumer-producer.js

@@ -1,52 +0,0 @@
-var async =  require ('async');
-var should = require ('should');
-var Chance = require ('chance');
-
-var chance = new Chance();
-
-function run_consumer (q) {
-  q.pop ('c1', {}, function (err, res) {
-    console.log ('consumer: got err %j', err, {});
-    console.log ('consumer: got res %j', res, {});
-
-//    setTimeout (function () {
-      run_consumer (q);
-//    }, chance.integer({ min: 200, max: 2000 }));
-  });
-}
-
-function run_producer (q) {
-  q.push ({a:1, b:'666'}, function (err, res) {
-    console.log ('producer: got err %j', err, {});
-    console.log ('producer: got res %j', res, {});
-
-    setTimeout (function () {
-      run_producer (q);
-    }, (random.from0to (10) + 1) * 1000);
-  });
-}
-
-
-var MQ = require ('../backends/redis-oq');
-var redis_signaller = require ('../signal/redis-pubsub');
-var redis_stats = require ('../stats/redis');
-
-var opts = {
-  signaller: {
-    provider: new redis_signaller ()
-  },
-  stats: {
-    provider: new redis_stats ()
-  }
-};
-    
-MQ (opts, function (err, factory) {
-  if (err) {
-    return logger.error (err);
-  }
-
-  var q = factory.queue('bench_test_queue', opts);
-
-  run_consumer (q);
-  run_producer (q);
-});

package/bench/redis-oq-consumer.js

@@ -1,40 +0,0 @@
-var async =  require ('async');
-var should = require ('should');
-
-var counter = 0;
-
-function run_consumer (q) {
-  q.pop ('c1', {}, function (err, res) {
-    console.log ('consumer: got err %j', err, {});
-    console.log ('consumer: got res %j', res, {});
-
-    counter++;
-    logger.info ('consumer: got %d', counter);
-    run_consumer (q);
-  });
-}
-
-var MQ = require ('../backends/redis-oq');
-
-var opts = {
-};
-    
-MQ (opts, function (err, factory) {
-  if (err) {
-    return logger.error (err);
-  }
-
-  var q_opts = {
-    logger: logger,
-    signaller: {
-      provider: require ('../signal/redis-pubsub')
-    },
-    stats: {
-      provider: require ('../stats/redis')
-    }
-  };
-
-  var q = factory.queue ('test_queue', q_opts);
-
-  run_consumer (q);
-});

package/bench/redis-oq-producer.js

@@ -1,43 +0,0 @@
-var async =  require ('async');
-var should = require ('should');
-
-var counter = 0;
-
-function run_producer (q) {
-  q.push ({a:1, b:'666'}, function (err, res) {
-    console.log ('producer: got err %j', err, {});
-    console.log('producer: got res %j', res, {});
-
-    counter++;
-    console.log('producer: got %d', counter);
-    setTimeout (function () {
-      run_producer (q);
-    }, 333);
-  });
-}
-
-
-var MQ = require ('../backends/redis-oq');
-
-var opts = {
-};
-    
-MQ (opts, function (err, factory) {
-  if (err) {
-    return logger.error (err);
-  }
-
-  var q_opts = {
-    logger: logger,
-    signaller: {
-      provider: require ('../signal/redis-pubsub')
-    },
-    stats: {
-      provider: require ('../stats/redis')
-    }
-  };
-
-  var q = factory.queue ('test_queue', q_opts);
-
-  run_producer (q);
-});

package/docker-compose/docker-compose.yaml

@@ -1,18 +0,0 @@
-version: '3'
-
-services:
-  mongo:
-    image: mongo:4.4.2-bionic
-    command: --logpath /dev/null
-    restart: on-failure
-    ports:
-      - 27017:27017
-
-  redis:
-    image: redis:alpine
-    ports:
-     - 6379:6379
-
-networks:
-  default:
-

package/docker-compose.yaml

@@ -1,18 +0,0 @@
-version: '3'
-
-services:
-  mongo:
-    image: mongo:4.4.2-bionic
-    command: --logpath /dev/null
-    restart: on-failure
-    ports:
-      - 27017:27017
-
-  redis:
-    image: redis:alpine
-    ports:
-     - 6379:6379
-
-networks:
-  default:
-

package/examples/pipelines/builder/index.js

@@ -1,99 +0,0 @@
-const Chance = require ('chance');
-const chance = new Chance();
-
-const MQ = require ('../../../backends/pl-mongo');
-
-function loop (n, fn, cb) {
-  if (n == 0) return cb ();
-  fn (n, err => {
-    if (err) return cb (err);
-    setImmediate (() => loop (n-1, fn, cb));
-  });
-}
-
-const factory_opts = {
-  url: 'mongodb://localhost/qeus'
-};
-
-function get_a_delay (min, max) {
-  return chance.integer ({min, max});
-}
-
-const num_elems = 37;
-let   processed = 0;
-
-function sink_process (elem, done) {
-  setTimeout (() => {
-    console.log ('%s: sunk elem [a %d, choice %d, tries %d]', this.name(), elem.payload.a, elem.payload.choice, elem.payload.choice_try);
-    done();
-
-    processed++;
-    if (processed == num_elems) {
-      setTimeout (() =>{
-        console.log ('processing done, exiting');
-        process.exit (0);
-      }, 1000);
-    }
-  }, get_a_delay (10, 20));
-}
-
-function dl_process (elem, done) {
-  // pass element to next queue, set payload.passed to true
-  done (null, {
-    update: {
-      $set: {stamp_0: 'passed'}
-    }
-  });
-}
-
-function choice_process (elem, done) {
-  setTimeout (() => {
-    if (chance.bool ({likelihood: 9})) {
-      console.log ('%s: failing on elem %o on try [%d]', this.name(), elem.payload, elem.tries);
-      return done ({e: 'cl1 induced a failure'});
-    }
-
-    const idx = chance.pickone([0, 1, 2]);
-    console.log ('%s: passing elem %o to %d on try [%d]', this.name(), elem.payload, idx, elem.tries);
-    done (null, {
-      dst: idx,
-      update: {
-        $set: {stamp_1: 'passed', choice: idx, choice_try: elem.tries}
-      }
-    });
-  }, get_a_delay (10, 20));
-}
-
-// initialize factory
-MQ (factory_opts, (err, factory) => {
-  if (err) return console.error (err);
-
-  const q_opts = {};
-
-  factory
-  .builder ()
-  .pipeline ('the-pipeline')
-  .queue ('test_pl_1', q_opts)
-  .queue ('test_pl_2', q_opts)
-  .queue ('test_pl_3', q_opts)
-  .queue ('test_pl_4', q_opts)
-  .queue ('test_pl_5', q_opts)
-  .directLink ('test_pl_1', 'test_pl_2', dl_process)
-  .choiceLink ('test_pl_2', ['test_pl_3', 'test_pl_4', 'test_pl_5'], choice_process)
-  .sink ('test_pl_3', sink_process)
-  .sink ('test_pl_4', sink_process)
-  .sink ('test_pl_5', sink_process)
-  .onError (console.log)
-  .done ((err, pl) => {
-    if (err) return console.error (err);
-    console.log ('pipeline IS READY')
-    pl.start ();
-    console.log ('pipeline IS RUNNING, inserting load')
-
-    loop (
-      num_elems,
-      (n, next) => setTimeout (() => pl.queues()['test_pl_1'].push ({a:n, b:'see it fail...'}, next), chance.integer ({min:0, max: 20})),
-      err => console.log (err || 'done')
-    );
-  });
-});

package/examples/pipelines/fromRecipe/index.js

@@ -1,127 +0,0 @@
-const Chance = require ('chance');
-
-
-const chance = new Chance();
-
-const MQ = require    ('../../../backends/pl-mongo');
-
-function loop (n, fn, cb) {
-  if (n == 0) return cb ();
-  fn (n, err => {
-    if (err) return cb (err);
-    setImmediate (() => loop (n-1, fn, cb));
-  });
-}
-
-const factory_opts = {
-  url: 'mongodb://localhost/qeus'
-};
-
-const bs_src_array = [
-  `
-  const Chance = require ('chance');
-  const chance = new Chance();
-  `,
-  `
-  function get_a_delay (min, max) {
-    return chance.integer ({min, max});
-  }
-  `,
-  `
-  function sink_process (elem, done) {
-    setTimeout (() => {
-      console.log ('%s: sunk elem [a %d, choice %d, tries %d]', this.name(), elem.payload.a, elem.payload.choice, elem.payload.choice_try);
-      done();
-
-      processed++;
-      if (processed == num_elems) {
-        setTimeout (() =>{
-          console.log ('processing done, exiting');
-          process.exit (0);
-        }, 1000);
-      }
-
-    }, get_a_delay (10, 20));
-  }
-
-  function dl_process (elem, done) {
-    // pass element to next queue, set payload.passed to true
-    done (null, {
-      update: {
-        $set: {stamp_0: 'passed'}
-      }
-    });
-  }
-
-  function choice_process (elem, done) {
-    setTimeout (() => {
-      if (chance.bool ({likelihood: 9})) {
-        console.log ('%s: failing on elem %o on try [%d]', this.name(), elem.payload, elem.tries);
-        return done ({e: 'cl1 induced a failure'});
-      }
-
-      const idx = chance.pickone([0, 1, 2]);
-      console.log ('%s: passing elem %o to %d on try [%d]', this.name(), elem.payload, idx, elem.tries);
-      done (null, {
-        dst: idx,
-        update: {
-          $set: {stamp_1: 'passed', choice: idx, choice_try: elem.tries}
-        }
-      });
-    }, get_a_delay (10, 20));
-  }
-  `
-];
-
-const setup_src_array = [
-  `
-  const q_opts = {};
-
-  builder
-  .queue ('test_pl_1', q_opts)
-  .queue ('test_pl_2', q_opts)
-  .queue ('test_pl_3', q_opts)
-  .queue ('test_pl_4', q_opts)
-  .queue ('test_pl_5', q_opts)
-  .directLink ('test_pl_1', 'test_pl_2', dl_process)
-  .choiceLink ('test_pl_2', ['test_pl_3', 'test_pl_4', 'test_pl_5'], choice_process)
-  .sink ('test_pl_3', sink_process)
-  .sink ('test_pl_4', sink_process)
-  .sink ('test_pl_5', sink_process)
-  .onError (console.log)
-  .done (done);
-  `
-];
-
-const num_elems = 3;
-let   processed = 0;
-
-// initialize factory
-MQ (factory_opts, (err, factory) => {
-  if (err) return console.error (err);
-
-  factory.pipelineFromRecipe (
-    'a_test_etcher',
-    bs_src_array,
-    setup_src_array,
-    {
-      context: {
-        num_elems,
-        processed,
-        process,
-        console
-      }
-    }, (err, pl) => {
-      if (err) return console.error (err);
-      console.log ('pipeline IS READY')
-      pl.start ();
-      console.log ('pipeline IS RUNNING, inserting load')
-
-      loop (
-        num_elems,
-        (n, next) => setTimeout (() => pl.queues()['test_pl_1'].push ({a:n, b:'see it fail...'}, next), chance.integer ({min:0, max: 20})),
-        err => console.log (err || 'done')
-      );
-    }
-  );
-});

package/examples/pipelines/simplest/index.js

@@ -1,38 +0,0 @@
-var MQ = require    ('../../../backends/pl-mongo');
-var PDL = require   ('../../../Pipeline/DirectLink');
-var async = require ('async');
-
-var factory_opts = {
-  url: 'mongodb://localhost/qeus'
-};
-
-// initialize factory
-MQ (factory_opts, (err, factory) => {
-  if (err) return console.error (err);
-
-  // factory ready, create 2 queues on default pipeline
-  var q_opts = {};
-  var q1 = factory.queue ('test_pl_1', q_opts);
-  var q2 = factory.queue ('test_pl_2', q_opts);
-
-  // tie them up, q1 -> q2
-  var pdl = new PDL (q1, q2);
-
-  pdl.start (function (elem, done) {
-    // pass element to next queue, set payload.passed to true
-    done (null, {
-      update: {
-        $set: {passed: true}
-      }
-    });
-  });
-
-  // insert elements in the entry queue
-  async.timesLimit (3, 3, (n, next) => q1.push ({a:n, b:'see it spin...'}, next));
-
-  // read elements at the outer end
-  async.timesLimit (3, 3, (n, next) => q2.pop ('exit', (err, res) => {
-    console.log ('end point get', res);
-    next ();
-  }));
-});