keuss 1.7.4 → 2.0.0

package/backends/ps-mongo.js

@@ -16,9 +16,7 @@ class PersistentMongoQueue extends Queue {
 
     if (!this._opts.ttl) this._opts.ttl = 3600;
 
-    this._factory = factory;
     this._col = factory._db.collection (name);
-    this.ensureIndexes (function (err) {});
   }
 
 
@@ -37,7 +35,6 @@ class PersistentMongoQueue extends Queue {
   insert (entry, callback) {
     this._col.insertOne (entry, {}, (err, result) => {
       if (err) return callback (err);
-      // TODO result.insertedCount must be 1
       callback (null, result.insertedId);
     });
   }
@@ -273,10 +270,10 @@ class PersistentMongoQueue extends Queue {
 
   //////////////////////////////////////////////////////////////////
   // create needed indexes for O(1) functioning
-  ensureIndexes (cb) {
+  _ensureIndexes (cb) {
     this._col.createIndex ({mature : 1}, err => {
       if (err) return cb (err);
-      this._col.createIndex({processed: 1}, {expireAfterSeconds: this._opts.ttl}, err => cb (err));
+      this._col.createIndex({processed: 1}, {expireAfterSeconds: this._opts.ttl}, err => cb (err, this));
     });
   }
 }
@@ -289,10 +286,16 @@ class Factory extends QFactory_MongoDB_defaults {
     this._db = mongo_conn.db();
   }
 
-  queue (name, opts) {
-    var full_opts = {};
+  queue (name, opts, cb) {
+    if (!cb) {
+      cb = opts;
+      opts = {};
+    }
+    
+    const full_opts = {};
     _.merge(full_opts, this._opts, opts);
-    return new PersistentMongoQueue (name, this, full_opts, opts);
+    const q = new PersistentMongoQueue (name, this, full_opts, opts);
+    q._ensureIndexes (cb);
   }
 
   close (cb) {

package/backends/redis-list.js

@@ -106,10 +106,15 @@ class Factory extends QFactory {
     this._rediscl = rediscl;
   }
 
-  queue (name, opts) {
-    var full_opts = {};
+  queue (name, opts, cb) {
+    if (!cb) {
+      cb = opts;
+      opts = {};
+    }
+    
+    const full_opts = {};
     _.merge(full_opts, this._opts, opts);
-    return new RedisListQueue (name, this, full_opts, opts);
+    return setImmediate(() => cb (null, new RedisListQueue (name, this, full_opts, opts)));
   }
 
   close (cb) {
@@ -143,8 +148,3 @@ function creator (opts, cb) {
 }
 
 module.exports = creator;
-
-
-
-
-

package/backends/redis-oq.js

@@ -128,10 +128,15 @@ class Factory extends QFactory {
     this._roq_factory = roq_factory;
   }
 
-  queue (name, opts) {
-    var full_opts = {};
+  queue (name, opts, cb) {
+    if (!cb) {
+      cb = opts;
+      opts = {};
+    }
+    
+    const full_opts = {};
     _.merge(full_opts, this._opts, opts);
-    return new RedisOQ (name, this, full_opts, opts);
+    return setImmediate (() => cb (null, new RedisOQ (name, this, full_opts, opts)));
   }
 
   close (cb) {

package/backends/stream-mongo.js

@@ -17,21 +17,11 @@ class StreamMongoQueue extends Queue {
 
     if (!this._opts.ttl) this._opts.ttl = 3600;
 
-    this._factory = factory;
     this._col = factory._db.collection (name);
     this._groups_str = this._opts.groups || 'A,B:C';
     this._groups_vector = this._groups_str.split (/[:,;.-]/).map (i => i.trim());
     this._gid = this._opts.group || this._groups_vector[0];
 
-    this.ensureIndexes (err => {
-      if (err) {
-        console.error ('keuss:Queue:StreamMongo: index creation failed, queues performance will be severely impacted:', err);
-      }
-      else {
-        debug ('indexes created');
-      }
-    });
-
     debug ('created with groups %j and gid %s (used for pop/reserve only)', this._groups_vector, this._gid);
   }
 
@@ -70,7 +60,6 @@ class StreamMongoQueue extends Queue {
 
     this._col.insertOne (entry, {}, (err, result) => {
       if (err) return callback (err);
-      // TODO result.insertedCount must be 1
       callback (null, result.insertedId);
       this._groups_vector.forEach (i => this._stats.incr (`stream.${i}.put`));
     });
@@ -403,7 +392,7 @@ class StreamMongoQueue extends Queue {
 
   //////////////////////////////////////////////////////////////////
   // create needed indexes for O(1) functioning
-  ensureIndexes (cb) {
+  _ensureIndexes (cb) {
     const tasks = [];
 
     this._groups_vector.forEach (i => {
@@ -412,7 +401,7 @@ class StreamMongoQueue extends Queue {
       tasks.push (cb => this._col.createIndex (idx, cb));
     });
     tasks.push (cb => this._col.createIndex ({t: 1}, {expireAfterSeconds: this._opts.ttl}, cb));
-    async.series (tasks, cb);
+    async.series (tasks, err => cb (err, this));
   }
 }
 
@@ -424,10 +413,16 @@ class Factory extends QFactory_MongoDB_defaults {
     this._db = mongo_conn.db();
   }
 
-  queue (name, opts) {
+  queue (name, opts, cb) {
+    if (!cb) {
+      cb = opts;
+      opts = {};
+    }
+    
     const full_opts = {};
     _.merge(full_opts, this._opts, opts);
-    return new StreamMongoQueue (name, this, full_opts, opts);
+    const q = new StreamMongoQueue (name, this, full_opts, opts);
+    q._ensureIndexes (cb);
   }
 
   close (cb) {

package/package.json

@@ -1,11 +1,13 @@
 {
   "name": "keuss",
-  "version": "1.7.4",
+  "version": "2.0.0",
   "keywords": [
     "queue",
     "persistent",
     "job",
     "mongodb",
+    "postgres",
+    "postgresql",
     "redis",
     "HA",
     "pipeline",
@@ -27,19 +29,21 @@
   "license": "MIT",
   "dependencies": {
     "@nodebb/mubsub": "~1.8.0",
-    "async": "~3.2.4",
-    "async-lock": "~1.3.1",
-    "debug": "~4.3.4",
-    "ioredis": "~5.3.2",
+    "async": "~3.2.5",
+    "async-lock": "~1.4.1",
+    "debug": "~4.3.5",
+    "ioredis": "~5.4.1",
     "lodash": "~4.17.21",
-    "mitt": "~3.0.0",
+    "mitt": "~3.0.1",
     "mongodb": "~4.17.0",
-    "uuid": "~8.3.2"
+    "uuid": "~8.3.2",
+    "pg": "~8.12.0"
   },
   "devDependencies": {
     "chance": "~1.1.11",
-    "mocha": "~10.2.0",
+    "mocha": "~10.4.0",
     "should": "~13.2.3",
+    "nyc": "~15.1.0",
     "why-is-node-running": "^2.2.2"
   },
   "scripts": {

package/stats/mem.js

@@ -68,7 +68,6 @@ class MemStats extends Stats {
     this._s.paused = false;
 
     // TODO remove from factory
-
     if (cb) cb();
   }
 

package/.github/workflows/codeql-analysis.yml

@@ -1,72 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL"
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ master ]
-  schedule:
-    - cron: '26 13 * * 1'
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'javascript' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        
-        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-        # queries: security-extended,security-and-quality
-
-        
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-
-    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
-    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
-
-    # - run: |
-    #   echo "Run, Build Application using script"
-    #   ./location_of_script_within_repo/buildscript.sh
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2

package/bench/all-mongo.js

@@ -1,108 +0,0 @@
-
-var async = require ('async');
-
-var MQ = require('../backends/mongo');
-//var signal_mongo_capped = require('../signal/mongo-capped');
-var signal_mongo_capped = require('../signal/local');
-var stats_mongo = require('../stats/mongo');
-
-
-var factory_opts = {
-  url: 'mongodb://localhost/qeus',
-  signaller: {
-    provider: signal_mongo_capped,
-    opts: {
-      url: 'mongodb://localhost/qeus_signal',
-      channel: 'das_channel'
-    }
-  },
-  stats: {
-    provider: stats_mongo,
-    opts: {
-      url: 'mongodb://localhost/qeus_stats'
-    }
-  }
-};
-
-
-// initialize factory
-MQ(factory_opts, (err, factory) => {
-  if (err) {
-    return console.error(err);
-  }
-
-  var consumed = 0;
-  var produced = 0;
-
-  function run_consumer(q) {
-    q.pop('c1', {}, (err, res) => {
-//      console.log('consumer[%s]: got res %j', q.name(), res, {});
-      consumed++;
-
-      if (consumed > 100) {
-        factory.close();
-      } else {
-        if (!(consumed % 10)) console.log('< %d', consumed);
-
-        setTimeout(function () {
-          run_consumer(q);
-        }, 222);
-      }
-    });
-  }
-
-  function run_producer(q) {
-    q.push({
-      a: 1,
-      b: '666',
-      t: new Date(),
-      n: produced
-    }, (err, res) => {
-      produced++;
-
-      if (produced > 3) {
-
-      } else {
-        if (!(produced % 10)) console.log('> %d', produced);
-
-        setTimeout(function () {
-          run_producer(q);
-        }, 33);
-      }
-    });
-  }
-
-  var opts = {};
-
-  var q = factory.queue('bench_test_queue_0', opts);
-//  run_consumer(q);
-//  run_producer(q);
-
-  async.series ([
-    cb => q.push ({q:0, a: 'ryetyeryre 0'}, cb),
-    cb => q.push ({q:1, a: 'ryetyeryre 1'}, cb),
-    cb => q.pop ('me', (err, res) => {console.log ('>>>>>>> POP c0:', res); cb ();}),
-    cb => q.pop ('me', (err, res) => {console.log ('>>>>>>> POP c1:', res); cb ();}),
-
-    cb => {
-      q.pop ('me', (err, res) => console.log ('>>>>>>> POP c2:', res));
-      q.pop ('me', (err, res) => console.log ('>>>>>>> POP c3:', res));
-      q.pop ('me', (err, res) => console.log ('>>>>>>> POP c4:', res));
-      q.pop ('me', (err, res) => console.log ('>>>>>>> POP c5:', res));
-      q.pop ('me', (err, res) => console.log ('>>>>>>> POP c6:', res));
-      cb ();
-    },
-    cb => setTimeout (cb, 2000),
-    cb => {q.pause(true); cb ();},
-    cb => q.push ({q:2, a: 'ryetyeryre 2'}, cb),
-    cb => q.push ({q:3, a: 'ryetyeryre 3'}, cb),
-    cb => q.push ({q:4, a: 'ryetyeryre 4'}, cb),
-    cb => q.push ({q:5, a: 'ryetyeryre 5'}, cb),
-    cb => setTimeout (cb, 2000),
-    cb => {q.pause(false); cb ();},
-    cb => setTimeout (cb, 2000),
-  ], () => {
-    q.cancel();
-    factory.close();
-  });
-});

package/bench/multi-q.js

@@ -1,85 +0,0 @@
-var async =  require ('async');
-var should = require ('should');
-var Chance = require ('chance');
-
-var produced = 0;
-var consumed = 0;
-
-var chance = new Chance();
-
-
-//var MQ = require ('../backends/redis-oq');
-//var MQ = require ('../backends/redis-list');
-//var MQ = require ('../backends/mongo');
-var MQ = require ('../backends/pl-mongo');
-
-var redis_signaller = require ('../signal/redis-pubsub');
-var redis_stats =     require ('../stats/redis');
-
-var opts = {
-  signaller: {
-    provider: new redis_signaller ()
-  },
-  stats: {
-    provider: new redis_stats ()
-  }
-};
-    
-MQ (opts, function (err, factory) {
-  if (err) {
-    return console.error (err);
-  }
-
-  
-function run_consumer (q) {
-  q.pop ('c1', {}, function (err, res) {
-//    console.log ('consumer[%s]: got res %j', q.name(), res, {});
-    consumed++;
-
-    if (consumed > 1000000) {
-      factory.close ();
-    }
-    else {
-      if (!(consumed % 10000)) console.log ('< %d', consumed)
-//    setTimeout (function () {
-      run_consumer (q);
-//    }, chance.integer({ min: 200, max: 2000 }));
-    }
-  });
-}
-
-function run_producer (q) {
-  q.push ({a:1, b:'666'}, function (err, res) {
-    produced++;
-
-    if (produced > 1000000) {
-      
-    }
-    else {
-      if (!(produced % 10000)) console.log ('> %d', produced)
-//    setTimeout (function () {
-      run_producer (q);
-//    }, chance.integer({ min: 200, max: 2000 }));
-    }
-  });
-}
-
-
-
-  var q0 = factory.queue('bench_test_queue_0', opts);
-  run_consumer (q0);
-  run_producer (q0);
-/*
-  var q1 = factory.queue('bench_test_queue_1', opts);
-  run_consumer (q1);
-  run_producer (q1);
-
-  var q2 = factory.queue('bench_test_queue_2', opts);
-  run_consumer (q2);
-  run_producer (q2);
-
-  var q3 = factory.queue('bench_test_queue_3', opts);
-  run_consumer (q3);
-  run_producer (q3);
-*/
-});

package/bench/redis-oq-consumer-producer.js

@@ -1,52 +0,0 @@
-var async =  require ('async');
-var should = require ('should');
-var Chance = require ('chance');
-
-var chance = new Chance();
-
-function run_consumer (q) {
-  q.pop ('c1', {}, function (err, res) {
-    console.log ('consumer: got err %j', err, {});
-    console.log ('consumer: got res %j', res, {});
-
-//    setTimeout (function () {
-      run_consumer (q);
-//    }, chance.integer({ min: 200, max: 2000 }));
-  });
-}
-
-function run_producer (q) {
-  q.push ({a:1, b:'666'}, function (err, res) {
-    console.log ('producer: got err %j', err, {});
-    console.log ('producer: got res %j', res, {});
-
-    setTimeout (function () {
-      run_producer (q);
-    }, (random.from0to (10) + 1) * 1000);
-  });
-}
-
-
-var MQ = require ('../backends/redis-oq');
-var redis_signaller = require ('../signal/redis-pubsub');
-var redis_stats = require ('../stats/redis');
-
-var opts = {
-  signaller: {
-    provider: new redis_signaller ()
-  },
-  stats: {
-    provider: new redis_stats ()
-  }
-};
-    
-MQ (opts, function (err, factory) {
-  if (err) {
-    return logger.error (err);
-  }
-
-  var q = factory.queue('bench_test_queue', opts);
-
-  run_consumer (q);
-  run_producer (q);
-});

package/bench/redis-oq-consumer.js

@@ -1,40 +0,0 @@
-var async =  require ('async');
-var should = require ('should');
-
-var counter = 0;
-
-function run_consumer (q) {
-  q.pop ('c1', {}, function (err, res) {
-    console.log ('consumer: got err %j', err, {});
-    console.log ('consumer: got res %j', res, {});
-
-    counter++;
-    logger.info ('consumer: got %d', counter);
-    run_consumer (q);
-  });
-}
-
-var MQ = require ('../backends/redis-oq');
-
-var opts = {
-};
-    
-MQ (opts, function (err, factory) {
-  if (err) {
-    return logger.error (err);
-  }
-
-  var q_opts = {
-    logger: logger,
-    signaller: {
-      provider: require ('../signal/redis-pubsub')
-    },
-    stats: {
-      provider: require ('../stats/redis')
-    }
-  };
-
-  var q = factory.queue ('test_queue', q_opts);
-
-  run_consumer (q);
-});

package/bench/redis-oq-producer.js

@@ -1,43 +0,0 @@
-var async =  require ('async');
-var should = require ('should');
-
-var counter = 0;
-
-function run_producer (q) {
-  q.push ({a:1, b:'666'}, function (err, res) {
-    console.log ('producer: got err %j', err, {});
-    console.log('producer: got res %j', res, {});
-
-    counter++;
-    console.log('producer: got %d', counter);
-    setTimeout (function () {
-      run_producer (q);
-    }, 333);
-  });
-}
-
-
-var MQ = require ('../backends/redis-oq');
-
-var opts = {
-};
-    
-MQ (opts, function (err, factory) {
-  if (err) {
-    return logger.error (err);
-  }
-
-  var q_opts = {
-    logger: logger,
-    signaller: {
-      provider: require ('../signal/redis-pubsub')
-    },
-    stats: {
-      provider: require ('../stats/redis')
-    }
-  };
-
-  var q = factory.queue ('test_queue', q_opts);
-
-  run_producer (q);
-});

package/docker-compose/docker-compose.yaml

@@ -1,18 +0,0 @@
-version: '3'
-
-services:
-  mongo:
-    image: mongo:4.4.2-bionic
-    command: --logpath /dev/null
-    restart: on-failure
-    ports:
-      - 27017:27017
-
-  redis:
-    image: redis:alpine
-    ports:
-     - 6379:6379
-
-networks:
-  default:
-

package/docker-compose.yaml

@@ -1,18 +0,0 @@
-version: '3'
-
-services:
-  mongo:
-    image: mongo:4.4.2-bionic
-    command: --logpath /dev/null
-    restart: on-failure
-    ports:
-      - 27017:27017
-
-  redis:
-    image: redis:alpine
-    ports:
-     - 6379:6379
-
-networks:
-  default:
-