keuss 1.7.4 → 2.0.1

package/backends/postgres.js

@@ -0,0 +1,380 @@
+const _ =    require ('lodash');
+const uuid = require ('uuid');
+const pg =   require ('pg');
+
+const Queue =    require ('../Queue');
+const QFactory = require ('../QFactory');
+
+const debug = require('debug')('keuss:Queue:postgres');
+
+class PGQueue extends Queue {
+
+
+	// TODO escape table name properly , as identifier, and check for reserved names using pg_get_keywords(), if the name is in select * from pg_get_keywords() where catdesc = 'R', then it should not be used
+  //////////////////////////////////////////////
+  constructor (name, factory, opts, orig_opts) {
+    super (name, factory, opts, orig_opts);
+    this._pool = factory._pool;
+    this._tbl_name = this._name; 
+  }
+
+
+  /////////////////////////////////////////
+  static Type () {
+    return 'postgres:simple';
+  }
+
+
+  /////////////////////////////////////////
+  type () {
+    return 'postgres:simple';
+  }
+
+
+  //////////////////////////////////////////////
+  // ensure table and indexes exists
+  _init (cb) {
+    this._pool.query (`
+    CREATE TABLE IF NOT EXISTS ${this._tbl_name} (
+      _id      VARCHAR(64) PRIMARY KEY,
+      _pl      JSONB,
+      mature   TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP NOT NULL,
+      tries    INTEGER DEFAULT 0 NOT NULL,
+      reserved TIMESTAMPTZ
+    );
+    CREATE INDEX IF NOT EXISTS idx_${this._tbl_name}_mature ON ${this._tbl_name} (mature);
+    `, err => cb (err));
+  }
+
+
+  /////////////////////////////////////////
+  // add element to queue
+  insert (entry, cb) {
+    const _id =     entry.id || uuid.v4();
+    const tries =   entry.tries || 0;
+    const mature =  new Date (entry.mature);
+
+    const pl = {
+      payload: entry.payload,
+      hdrs:    entry.hdrs || {}
+    }
+
+    if (Buffer.isBuffer (pl.payload)) {
+      pl.payload = pl.payload.toString ('base64');
+      pl.type = 'buffer';
+    }
+
+    this._pool.query (`INSERT INTO ${this._tbl_name} VALUES($1, $2, $3, $4)`, [_id, pl, mature, tries], (err, res) => {
+      if (err) return cb (err);
+      cb (null, _id)
+    })
+  }
+
+
+  /////////////////////////////////////////
+  // get element from queue
+  get (cb) {    
+    this._pool.query (`
+      DELETE FROM ${this._tbl_name}
+      WHERE _id = (
+        SELECT _id
+        FROM ${this._tbl_name}
+        WHERE mature < now()
+        ORDER BY mature
+        FOR UPDATE SKIP LOCKED
+        LIMIT 1
+      )
+      RETURNING *;
+    `, (err, res) => {
+      if (err) return cb (err);
+
+      if (_.size (res.rows) == 0) return cb (null, null); // not found
+      
+      const pl = res.rows[0];
+
+      _.merge (pl, pl._pl);
+      delete (pl._pl);
+
+      if (pl.type == 'buffer') {
+        try {
+          pl.payload = Buffer.from (pl.payload, 'base64');
+        } catch (e) {
+          // TODO maybe log some error/Statistic?
+        }
+      }
+
+      cb (null, pl);
+    })
+  }
+
+
+  //////////////////////////////////
+  // reserve element: call cb (err, pl) where pl has an id
+  reserve (cb) {
+    const delay = this._opts.reserve_delay || 120;
+
+    this._pool.query (`
+      UPDATE ${this._tbl_name}
+      SET
+        tries = tries + 1,
+        mature = now() + make_interval(secs => ${delay}),
+        reserved = now()
+      WHERE _id = (
+        SELECT _id
+        FROM ${this._tbl_name}
+        WHERE mature < now()
+        ORDER BY mature
+        FOR UPDATE SKIP LOCKED
+        LIMIT 1
+      )
+      RETURNING *;
+    `, (err, res) => {
+      if (err) return cb (err);
+      
+      if (_.size (res.rows) == 0) return cb (null, null); // not found
+      
+      const pl = res.rows[0];
+
+      _.merge (pl, pl._pl);
+      delete (pl._pl);
+
+      // adjust tries to pre-update
+      pl.tries--;
+
+      if (pl.type == 'buffer') {
+        try {
+          pl.payload = Buffer.from (pl.payload, 'base64');
+        } catch (e) {
+          // TODO maybe log some error/Statistic?
+        }
+      }
+
+      cb (null, pl);
+    })
+  }
+
+
+  //////////////////////////////////
+  // commit previous reserve, by p.id
+  commit (id, cb) {
+    if (!uuid.validate (id)) return cb ('id [' + id + '] can not be used as commit id: not a valid UUID');
+
+    this._pool.query (`
+      DELETE FROM ${this._tbl_name}
+      WHERE _id = $1
+      AND reserved IS NOT NULL
+    `, 
+    [id],
+    (err, res) => {
+      if (err) return cb (err);
+      cb (null, res && (res.rowCount == 1));
+    })
+  }
+
+
+  //////////////////////////////////
+  // rollback previous reserve, by p.id
+  rollback (id, next_t, cb) {
+    if (_.isFunction (next_t)) {
+      cb = next_t;
+      next_t = null;
+    }
+
+    if (!uuid.validate (id)) return cb ('id [' + id + '] can not be used as rollback id: not a valid UUID');
+
+    const nxt = (next_t ? new Date (next_t) : Queue.now ());
+
+    this._pool.query (`
+      UPDATE ${this._tbl_name}
+      SET
+        reserved = NULL,
+        mature = $1
+      WHERE _id = $2
+      AND reserved IS NOT NULL
+    `, 
+    [nxt, id],
+    (err, res) => {
+      if (err) return cb (err);
+      cb (null, res && (res.rowCount == 1));
+    })
+  }
+
+
+  //////////////////////////////////
+  // queue size including non-mature elements
+  totalSize (cb) {
+    this._pool.query (`
+      SELECT COUNT(*)
+      FROM ${this._tbl_name}
+    `, 
+    (err, res) => {
+      if (err) return cb (err);
+      cb (null, parseInt (res.rows[0].count));
+    })
+  }
+
+
+  //////////////////////////////////
+  // queue size NOT including non-mature elements
+  size (cb) {
+    this._pool.query (`
+      SELECT COUNT(*)
+      FROM ${this._tbl_name}
+      WHERE mature < now()
+    `, 
+    (err, res) => {
+      if (err) return cb (err);
+      cb (null, parseInt (res.rows[0].count));
+    })
+  }
+
+
+  //////////////////////////////////
+  // queue size of non-mature elements only
+  schedSize (cb) {
+    this._pool.query (`
+      SELECT COUNT(*)
+      FROM ${this._tbl_name}
+      WHERE mature >= now()
+      AND reserved IS NULL
+    `, 
+    (err, res) => {
+      if (err) return cb (err);
+      cb (null, parseInt (res.rows[0].count));
+    })
+  }
+
+
+  //////////////////////////////////
+  // queue size of reserved elements only
+  resvSize (cb) {
+    this._pool.query (`
+      SELECT COUNT(*)
+      FROM ${this._tbl_name}
+      WHERE mature >= now()
+      AND reserved IS NOT NULL
+    `, 
+    (err, res) => {
+      if (err) return cb (err);
+      cb (null, parseInt (res.rows[0].count));
+    })
+  }
+
+
+  /////////////////////////////////////////
+  // get element from queue
+  next_t (cb) {  
+    this._pool.query (`
+      SELECT mature
+      FROM ${this._tbl_name}
+      ORDER BY mature
+      LIMIT 1
+    `, (err, res) => {
+      if (err)                    return cb (err);
+      if (_.size (res.rows) == 0) return cb (null, null); // not found
+      
+      cb (null, res.rows[0].mature);
+    });
+  }
+
+
+  //////////////////////////////////////////////
+  // remove by id
+  remove (id, cb) {
+    if (!uuid.validate (id)) return cb ('id [' + id + '] can not be used as remove id: not a valid UUID');
+
+    this._pool.query (`
+      DELETE FROM ${this._tbl_name}
+      WHERE _id = $1
+      AND reserved IS NULL
+    `, 
+    [id],
+    (err, res) => {
+      if (err) return cb (err);
+      cb (null, res && (res.rowCount == 1));
+    })
+  }
+
+
+  ///////////////////////////////////////////////////////////////////////////////
+  // private parts
+
+};
+
+
+
+///////////////////////////////////////////////////////////////////////////////
+class Factory extends QFactory {
+  constructor (opts, pg_pool) {
+    super (opts);
+    this._pool = pg_pool;
+    debug ('crated Factory with options %j', opts);
+  }
+
+  queue (name, opts, cb) {
+    if (!cb) {
+      cb = opts;
+      opts = {};
+    }
+    
+    const full_opts = {};
+    _.merge(full_opts, this._opts, opts);
+    debug ('creating queue with full_opts %j', full_opts);
+
+    const q = new PGQueue (name, this, full_opts, opts);
+    q._init (err => {
+      if (err) return cb (err);
+      cb (null, q);
+    });
+  }
+
+  close (cb) {
+    super.close (() => {
+      debug ('closing pool...');
+      this._pool.end (cb);
+    });
+  }
+
+  type () {
+    return PGQueue.Type ();
+  }
+
+  capabilities () {
+    return {
+      sched:    true,
+      reserve:  true,
+      pipeline: false,
+      tape:     false,
+      remove:   true
+    };
+  }
+}
+
+
+///////////////////////////////////////////////////////////////////////////////
+function creator (opts, cb) {
+  const _opts = opts || {};
+
+  debug ('Creator: creating pool with %j', _opts);
+
+  const dflt = {
+    user:     'pg', 
+    password: 'pg',
+    host:     'localhost',
+    port:     5432,
+    database: 'pg'
+  }
+
+  const pool = new pg.Pool(_.merge ({}, dflt, _opts.postgres));
+
+  // ensure the pool can connect
+  pool.query ('select 1', err => {
+    if (err) return cb (err);
+
+    debug ('pool created, can connect. Creating Factory...');
+    const F = new Factory (_opts, pool);
+    F.async_init (err => cb (err, F));
+  });
+}
+
+module.exports = creator;

package/backends/ps-mongo.js

@@ -16,9 +16,7 @@ class PersistentMongoQueue extends Queue {
 
     if (!this._opts.ttl) this._opts.ttl = 3600;
 
-    this._factory = factory;
     this._col = factory._db.collection (name);
-    this.ensureIndexes (function (err) {});
   }
 
 
@@ -37,7 +35,6 @@ class PersistentMongoQueue extends Queue {
   insert (entry, callback) {
     this._col.insertOne (entry, {}, (err, result) => {
       if (err) return callback (err);
-      // TODO result.insertedCount must be 1
       callback (null, result.insertedId);
     });
   }
@@ -273,10 +270,10 @@ class PersistentMongoQueue extends Queue {
 
   //////////////////////////////////////////////////////////////////
   // create needed indexes for O(1) functioning
-  ensureIndexes (cb) {
+  _ensureIndexes (cb) {
     this._col.createIndex ({mature : 1}, err => {
       if (err) return cb (err);
-      this._col.createIndex({processed: 1}, {expireAfterSeconds: this._opts.ttl}, err => cb (err));
+      this._col.createIndex({processed: 1}, {expireAfterSeconds: this._opts.ttl}, err => cb (err, this));
     });
   }
 }
@@ -289,10 +286,16 @@ class Factory extends QFactory_MongoDB_defaults {
     this._db = mongo_conn.db();
   }
 
-  queue (name, opts) {
-    var full_opts = {};
+  queue (name, opts, cb) {
+    if (!cb) {
+      cb = opts;
+      opts = {};
+    }
+    
+    const full_opts = {};
     _.merge(full_opts, this._opts, opts);
-    return new PersistentMongoQueue (name, this, full_opts, opts);
+    const q = new PersistentMongoQueue (name, this, full_opts, opts);
+    q._ensureIndexes (cb);
   }
 
   close (cb) {

package/backends/redis-list.js

@@ -106,10 +106,15 @@ class Factory extends QFactory {
     this._rediscl = rediscl;
   }
 
-  queue (name, opts) {
-    var full_opts = {};
+  queue (name, opts, cb) {
+    if (!cb) {
+      cb = opts;
+      opts = {};
+    }
+    
+    const full_opts = {};
     _.merge(full_opts, this._opts, opts);
-    return new RedisListQueue (name, this, full_opts, opts);
+    return setImmediate(() => cb (null, new RedisListQueue (name, this, full_opts, opts)));
   }
 
   close (cb) {
@@ -143,8 +148,3 @@ function creator (opts, cb) {
 }
 
 module.exports = creator;
-
-
-
-
-

package/backends/redis-oq.js

@@ -128,10 +128,15 @@ class Factory extends QFactory {
     this._roq_factory = roq_factory;
   }
 
-  queue (name, opts) {
-    var full_opts = {};
+  queue (name, opts, cb) {
+    if (!cb) {
+      cb = opts;
+      opts = {};
+    }
+    
+    const full_opts = {};
     _.merge(full_opts, this._opts, opts);
-    return new RedisOQ (name, this, full_opts, opts);
+    return setImmediate (() => cb (null, new RedisOQ (name, this, full_opts, opts)));
   }
 
   close (cb) {

package/backends/stream-mongo.js

@@ -17,21 +17,11 @@ class StreamMongoQueue extends Queue {
 
     if (!this._opts.ttl) this._opts.ttl = 3600;
 
-    this._factory = factory;
     this._col = factory._db.collection (name);
     this._groups_str = this._opts.groups || 'A,B:C';
     this._groups_vector = this._groups_str.split (/[:,;.-]/).map (i => i.trim());
     this._gid = this._opts.group || this._groups_vector[0];
 
-    this.ensureIndexes (err => {
-      if (err) {
-        console.error ('keuss:Queue:StreamMongo: index creation failed, queues performance will be severely impacted:', err);
-      }
-      else {
-        debug ('indexes created');
-      }
-    });
-
     debug ('created with groups %j and gid %s (used for pop/reserve only)', this._groups_vector, this._gid);
   }
 
@@ -70,7 +60,6 @@ class StreamMongoQueue extends Queue {
 
     this._col.insertOne (entry, {}, (err, result) => {
       if (err) return callback (err);
-      // TODO result.insertedCount must be 1
       callback (null, result.insertedId);
       this._groups_vector.forEach (i => this._stats.incr (`stream.${i}.put`));
     });
@@ -403,7 +392,7 @@ class StreamMongoQueue extends Queue {
 
   //////////////////////////////////////////////////////////////////
   // create needed indexes for O(1) functioning
-  ensureIndexes (cb) {
+  _ensureIndexes (cb) {
     const tasks = [];
 
     this._groups_vector.forEach (i => {
@@ -412,7 +401,7 @@ class StreamMongoQueue extends Queue {
       tasks.push (cb => this._col.createIndex (idx, cb));
     });
     tasks.push (cb => this._col.createIndex ({t: 1}, {expireAfterSeconds: this._opts.ttl}, cb));
-    async.series (tasks, cb);
+    async.series (tasks, err => cb (err, this));
   }
 }
 
@@ -424,10 +413,16 @@ class Factory extends QFactory_MongoDB_defaults {
     this._db = mongo_conn.db();
   }
 
-  queue (name, opts) {
+  queue (name, opts, cb) {
+    if (!cb) {
+      cb = opts;
+      opts = {};
+    }
+    
     const full_opts = {};
     _.merge(full_opts, this._opts, opts);
-    return new StreamMongoQueue (name, this, full_opts, opts);
+    const q = new StreamMongoQueue (name, this, full_opts, opts);
+    q._ensureIndexes (cb);
   }
 
   close (cb) {

package/package.json

@@ -1,11 +1,13 @@
 {
   "name": "keuss",
-  "version": "1.7.4",
+  "version": "2.0.1",
   "keywords": [
     "queue",
     "persistent",
     "job",
     "mongodb",
+    "postgres",
+    "postgresql",
     "redis",
     "HA",
     "pipeline",
@@ -27,19 +29,21 @@
   "license": "MIT",
   "dependencies": {
     "@nodebb/mubsub": "~1.8.0",
-    "async": "~3.2.4",
-    "async-lock": "~1.3.1",
-    "debug": "~4.3.4",
-    "ioredis": "~5.3.2",
+    "async": "~3.2.5",
+    "async-lock": "~1.4.1",
+    "debug": "~4.3.5",
+    "ioredis": "~5.4.1",
     "lodash": "~4.17.21",
-    "mitt": "~3.0.0",
+    "mitt": "~3.0.1",
     "mongodb": "~4.17.0",
-    "uuid": "~8.3.2"
+    "uuid": "~8.3.2",
+    "pg": "~8.12.0"
   },
   "devDependencies": {
     "chance": "~1.1.11",
-    "mocha": "~10.2.0",
+    "mocha": "~10.4.0",
     "should": "~13.2.3",
+    "nyc": "~15.1.0",
     "why-is-node-running": "^2.2.2"
   },
   "scripts": {

package/stats/mem.js

@@ -68,7 +68,6 @@ class MemStats extends Stats {
     this._s.paused = false;
 
     // TODO remove from factory
-
     if (cb) cb();
   }
 

package/.github/workflows/codeql-analysis.yml

@@ -1,72 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL"
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ master ]
-  schedule:
-    - cron: '26 13 * * 1'
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'javascript' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        
-        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-        # queries: security-extended,security-and-quality
-
-        
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-
-    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
-    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
-
-    # - run: |
-    #   echo "Run, Build Application using script"
-    #   ./location_of_script_within_repo/buildscript.sh
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2