keuss 1.6.14 → 1.7.0

package/.github/workflows/codeql-analysis.yml

@@ -0,0 +1,72 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '26 13 * * 1'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'javascript' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+        
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #   echo "Run, Build Application using script"
+    #   ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2

package/Queue.js

@@ -98,6 +98,9 @@ class Queue {
   // remove (and return if possible) by id
   remove (id, callback) {callback(null, null);}
 
+  // extra information & status
+  extra_info (cb) {cb (null, {});}
+
   // end of expected redefinitions on subclasses
   ////////////////////////////////////////////////////////////////////////////
 
@@ -113,6 +116,29 @@ class Queue {
   capabilities () {
     return this._factory.capabilities ();
   }
+  
+  
+  info (cb) {
+    async.parallel ({
+      size:      cb => this.size (cb),
+      totalSize: cb => this.totalSize (cb),
+      schedSize: cb => this.schedSize (cb),
+      resvSize:  cb => this.resvSize (cb),
+      next_t:    cb => this.next_t (cb),
+      stats:     cb => this.stats (cb),
+      paused:    cb => this.paused (cb),
+      extra:     cb => this.extra_info (cb),
+    }, (err, res) => {
+      if (err) return cb (err);
+
+      res.name =         this.name();
+      res.ns =           this.ns();
+      res.type =         this.type();
+      res.capabilities = this.capabilities();
+
+      cb (null, res);
+    });
+  }
 
   // T of next mature
   nextMatureDate () {return this._next_mature_t;}

package/Signal.js

@@ -73,6 +73,12 @@ class Signal {
   }
 
 
+  // to be extended: generic pubsub service
+  subscribe_extra (topic, on_cb) {return false}
+  unsubscribe_extra (subscr) {}
+  emit_extra (topic, ev, cb) {if (cb) cb ();}
+
+
   static _hrtimeAsMSecs (hrtime) {
     return (hrtime[0] * 1000) + (hrtime[1] / 1e6);
   }

package/backends/ps-mongo.js

@@ -14,6 +14,8 @@ class PersistentMongoQueue extends Queue {
   constructor (name, factory, opts, orig_opts) {
     super (name, factory, opts, orig_opts);
 
+    if (!this._opts.ttl) this._opts.ttl = 3600;
+
     this._factory = factory;
     this._col = factory._db.collection (name);
     this.ensureIndexes (function (err) {});
@@ -50,7 +52,10 @@ class PersistentMongoQueue extends Queue {
     };
 
     var updt = {
-      $set: {processed: new Date ()}
+      $set: {
+        processed: new Date (),
+        mature: Queue.nowPlusSecs (100 * this._opts.ttl)
+      }
     };
 
     var opts = {
@@ -78,7 +83,10 @@ class PersistentMongoQueue extends Queue {
     };
 
     var update = {
-      $set: {mature: Queue.nowPlusSecs (delay), reserved: new Date ()},
+      $set: {
+        mature: Queue.nowPlusSecs (delay), 
+        reserved: new Date ()
+      },
       $inc: {tries: 1}
     };
 
@@ -113,7 +121,10 @@ class PersistentMongoQueue extends Queue {
     }
 
     var updt = {
-      $set:   {processed: new Date ()},
+      $set:   {
+        processed: new Date (),
+        mature: Queue.nowPlusSecs (100 * this._opts.ttl)
+      },
       $unset: {reserved: ''}
     };
 
@@ -226,7 +237,11 @@ class PersistentMongoQueue extends Queue {
     }
 
     var updt = {
-      $set:   {processed: new Date (), removed: true},
+      $set:   {
+        processed: new Date (),
+        mature: Queue.nowPlusSecs (100 * this._opts.ttl),
+        removed: true
+      },
     };
 
     var opts = {};
@@ -261,7 +276,7 @@ class PersistentMongoQueue extends Queue {
   ensureIndexes (cb) {
     this._col.createIndex ({mature : 1}, err => {
       if (err) return cb (err);
-      this._col.createIndex({processed: 1}, {expireAfterSeconds: this._opts.ttl || 3600}, err => cb (err));
+      this._col.createIndex({processed: 1}, {expireAfterSeconds: this._opts.ttl}, err => cb (err));
     });
   }
 }

package/backends/stream-mongo.js

@@ -0,0 +1,476 @@
+const _ =     require ('lodash');
+const async = require ('async');
+
+const MongoClient = require ('mongodb').MongoClient;
+const mongo =       require ('mongodb');
+
+const Queue =                     require ('../Queue');
+const QFactory_MongoDB_defaults = require ('../QFactory-MongoDB-defaults');
+
+var debug = require('debug')('keuss:Queue:StreamMongo');
+
+class StreamMongoQueue extends Queue {
+
+  //////////////////////////////////////////////
+  constructor (name, factory, opts, orig_opts) {
+    super (name, factory, opts, orig_opts);
+
+    if (!this._opts.ttl) this._opts.ttl = 3600;
+
+    this._factory = factory;
+    this._col = factory._db.collection (name);
+    this._groups_str = this._opts.groups || 'A,B:C';
+    this._groups_vector = this._groups_str.split (/[:,;.-]/).map (i => i.trim());
+    this._gid = this._opts.group || this._groups_vector[0];
+
+    this.ensureIndexes (err => {
+      if (err) {
+        console.error ('keuss:Queue:StreamMongo: index creation failed, queues performance will be severely impacted:', err);
+      }
+      else {
+        debug ('indexes created');
+      }
+    });
+
+    debug ('created with groups %j and gid %s (used for pop/reserve only)', this._groups_vector, this._gid);
+  }
+
+
+  /////////////////////////////////////////
+  static Type () {
+    return 'mongo:stream';
+  }
+
+
+  /////////////////////////////////////////
+  type () {
+    return 'mongo:stream';
+  }
+
+
+  /////////////////////////////////////////
+  _vector (item) {
+    const r = {};
+    this._groups_vector.forEach (i => r[i] = item);
+    return r;
+  }
+
+
+  /////////////////////////////////////////
+  // add element to queue
+  insert (entry, callback) {
+    const mtr = entry.mature;
+    const tr =  entry.tries;
+
+    entry.tries =     this._vector (tr);
+    entry.mature =    this._vector (mtr);
+    entry.processed = this._vector (false);
+    
+    entry.t = new Date();
+
+    this._col.insertOne (entry, {}, (err, result) => {
+      if (err) return callback (err);
+      // TODO result.insertedCount must be 1
+      callback (null, result.insertedId);
+      this._groups_vector.forEach (i => this._stats.incr (`stream.${i}.put`));
+    });
+  }
+
+
+  /////////////////////////////////////////
+  // get element from queue
+  get (callback) {
+    const gid = this._gid;
+    const q = {};
+
+    q[`mature.${gid}`] = {$lte: Queue.nowPlusSecs (0)};
+    q[`processed.${gid}`] = false;
+
+    const updt = {
+      $set: {}
+    };
+    updt.$set[`processed.${gid}`] = new Date ();
+    updt.$set[`mature.${gid}`] = Queue.nowPlusSecs (100 * this._opts.ttl);
+
+    const opts = {
+      sort: {}
+    };
+    opts.sort[`mature.${gid}`] = 1;
+
+    debug ('get() with q %O, upd %O, opts %o', q, updt, opts);
+
+    this._col.findOneAndUpdate (q, updt, opts, (err, result) => {
+      if (err) return callback (err);
+      const v = result && result.value;
+      if (!v) return callback ();
+      if (v.payload._bsontype == 'Binary') v.payload = v.payload.buffer;
+      v.mature = v.mature[gid];
+      v.tries = v.tries[gid];
+      delete v.processed;
+      delete v.t;
+      callback (null, v);
+      this._stats.incr (`stream.${gid}.get`);
+    });
+  }
+
+
+  //////////////////////////////////
+  // reserve element: call cb (err, pl) where pl has an id
+  reserve (callback) {
+    const gid = this._gid;
+    const delay = this._opts.reserve_delay || 120;
+    
+    const q = {};
+
+    q[`mature.${gid}`] = {$lte: Queue.nowPlusSecs (0)};
+    q[`processed.${gid}`] = false;
+
+    const updt = {
+      $set: {},
+      $inc: {}
+    };
+    updt.$set[`reserved.${gid}`] = new Date ();
+    updt.$set[`mature.${gid}`] = Queue.nowPlusSecs (delay);
+    updt.$inc[`tries.${gid}`] = 1;
+
+    const opts = {
+      sort: {},
+      returnDocument: 'before'
+    };
+    opts.sort[`mature.${gid}`] = 1;
+
+    debug ('reserve() with q %O, upd %O, opts %o', q, updt, opts);
+
+    this._col.findOneAndUpdate (q, updt, opts, (err, result) => {
+      if (err) return callback (err);
+      const v = result && result.value;
+      if (!v) return callback ();
+      if (v.payload._bsontype == 'Binary') v.payload = v.payload.buffer;
+      v.mature = v.mature[gid];
+      v.tries = v.tries[gid];
+      delete v.processed;
+      delete v.t;
+      callback (null, v);
+      this._stats.incr (`stream.${gid}.reserve`);
+    });
+  }
+
+
+  //////////////////////////////////
+  // commit previous reserve, by p.id
+  commit (id, callback) {
+    const gid = this._gid;
+    let q;
+
+    try {
+      q =  {
+        _id: (_.isString(id) ? new mongo.ObjectID (id) : id),
+      };
+      q[`reserved.${gid}`] = {$exists: true};
+    }
+    catch (e) {
+      return callback ('id [' + id + '] can not be used as rollback id: ' + e);
+    }
+
+    const updt = {
+      $set:   {},
+      $unset: {}
+    };
+    updt.$set[`processed.${gid}`] = new Date ();
+    updt.$set[`mature.${gid}`] = Queue.nowPlusSecs (100 * this._opts.ttl);
+    updt.$unset[`reserved.${gid}`] = '';
+
+    const opts = {};
+
+    debug ('commit() with q %O, upd %O, opts %o', q, updt, opts);
+
+    this._col.updateOne (q, updt, opts, (err, result) => {
+      if (err) return callback (err);
+      callback (null, result && (result.modifiedCount == 1));
+      this._stats.incr (`stream.${gid}.commit`);
+    });
+  }
+
+
+  //////////////////////////////////
+  // rollback previous reserve, by p.id
+  rollback (id, next_t, callback) {
+    const gid = this._gid;
+    let q;
+
+    if (_.isFunction (next_t)) {
+      callback = next_t;
+      next_t = null;
+    }
+
+    try {
+      q =  {
+        _id: (_.isString(id) ? new mongo.ObjectID (id) : id),
+      };
+      q[`reserved.${gid}`] = {$exists: true};
+    }
+    catch (e) {
+      return callback ('id [' + id + '] can not be used as rollback id: ' + e);
+    }
+
+    const updt = {
+      $set:   {},
+      $unset: {}
+    };
+    updt.$set[`mature.${gid}`] = (next_t ? new Date (next_t) : Queue.now ());
+    updt.$unset[`reserved.${gid}`] = '';
+
+    const opts = {};
+
+    debug ('rollback() with q %O, upd %O, opts %o', q, updt, opts);
+
+    this._col.updateOne (q, updt, opts, (err, result) => {
+      if (err) return callback (err);
+      callback (null, result && (result.modifiedCount == 1));
+      this._stats.incr (`stream.${gid}.rollback`);
+    });
+  }
+
+
+  //////////////////////////////////
+  // queue size including non-mature elements
+  totalSize (callback, gid) {
+    const gr = gid || this._gid;
+
+    const q = {};
+    q[`processed.${gr}`] = false;
+
+    const opts = {};
+    this._col.countDocuments (q, opts, callback);
+  }
+
+
+  //////////////////////////////////
+  // queue size NOT including non-mature elements
+  size (callback, gid) {
+    const gr = gid || this._gid;
+
+    const q = {};
+    q[`processed.${gr}`] = false;
+    q[`mature.${gr}`] = {$lte: Queue.now()};
+
+    const opts = {};
+    this._col.countDocuments (q, opts, callback);
+  }
+
+
+  //////////////////////////////////
+  // queue size of non-mature elements only
+  schedSize (callback, gid) {
+    const gr = gid || this._gid;
+
+    const q = {};
+    q[`processed.${gr}`] = false;
+    q[`reserved.${gr}`] = {$exists: false};
+    q[`mature.${gr}`] = {$gt: Queue.now()};
+
+    const opts = {};
+    this._col.countDocuments (q, opts, callback);
+  }
+
+
+  //////////////////////////////////
+  // queue size of reserved elements only
+  resvSize (callback, gid) {
+    const gr = gid || this._gid;
+
+    const q = {};
+    q[`processed.${gr}`] = false;
+    q[`reserved.${gr}`] = {$exists: true};
+    q[`mature.${gr}`] = {$gt: Queue.now()};
+
+    const opts = {};
+    this._col.countDocuments (q, opts, callback);
+  }
+
+
+  /////////////////////////////////////////
+  // get element from queue
+  next_t (callback, gid) {
+    const gr = gid || this._gid;
+    
+    const q = {};
+    q[`processed.${gr}`] = false;
+
+    const sort = {};
+    sort[`mature.${gr}`] = 1;
+
+    this._col
+    .find (q)
+    .limit(1)
+    .sort (sort)
+    .project ({mature:1})
+    .next ((err, result) => {
+      if (err) return callback (err);
+      debug ('next_t with git %s: got %o', gr, result);
+      callback (null, result && result.mature && result.mature[gr]);
+    });
+  }
+
+
+  //////////////////////////////////
+  // queue size of non-mature elements only.
+  // COMMENTED OUT: takes 1 sec per each 100K elements in collection
+  /*
+  extra_info (callback) {
+    const cursor = this._col.aggregate ([
+      {
+        $group : {
+          _id : "v",
+          r: {
+            $accumulator: {
+              init: `function() { 
+                return { size: {}, totalSize: {}, resvSize: {}, schedSize: {}, processed: {} }
+              }`,
+              accumulate: `function(state, mature, reserved, processed) { 
+                for (const gr in mature) {
+                  if (!state.size[gr])      state.size[gr] = 0;
+                  if (!state.totalSize[gr]) state.totalSize[gr] = 0;
+                  if (!state.resvSize[gr])  state.resvSize[gr] = 0;
+                  if (!state.schedSize[gr]) state.schedSize[gr] = 0;
+                  if (!state.processed[gr]) state.processed[gr] = 0;
+
+                  const mtr = (mature[gr].getTime() < new Date().getTime());
+                  const rsv = (reserved && ((reserved[gr] != null) && (reserved[gr] != undefined)));
+                  const prc = processed[gr];
+
+                  if (prc) {
+                    state.processed[gr]++;
+                  }
+                  else {
+                    state.totalSize[gr]++;
+
+                    if (mtr) {
+                      state.size[gr]++;
+                    }
+                    else {
+                      if (rsv) state.resvSize[gr]++;
+                      else     state.schedSize[gr]++;
+                    }
+                  }
+                }
+
+                return state;
+              }`,
+              accumulateArgs: ['$mature', '$reserved', '$processed'],  
+              merge: `function(state1, state2) { 
+                const res = { size: {}, totalSize: {}, resvSize: {}, schedSize: {} }
+                for (const gr in state1) {
+                  res.size[gr] =      state1.size[gr] +      (state2.size[gr] || 0)
+                  res.totalSize[gr] = state1.totalSize[gr] + (state2.totalSize[gr] || 0)
+                  res.resvSize[gr] =  state1.resvSize[gr] +  (state2.resvSize[gr] || 0)
+                  res.schedSize[gr] = state1.schedSize[gr] + (state2.schedSize[gr] || 0)
+                  res.processed[gr] = state1.processed[gr] + (state2.processed[gr] || 0)
+                }
+                for (const gr in state2) {
+                  if (!state1.size[gr])      res.size[gr] =      state2.size[gr]
+                  if (!state1.totalSize[gr]) res.totalSize[gr] = state2.totalSize[gr]
+                  if (!state1.schedSize[gr]) res.schedSize[gr] = state2.schedSize[gr]
+                  if (!state1.resvSize[gr])  res.resvSize[gr] =  state2.resvSize[gr]
+                  if (!state1.processed[gr]) res.processed[gr] = state2.processed[gr]
+                }
+                return res
+              }`,
+              lang: "js"
+            }
+          }
+        }
+      }
+
+    ]);
+
+
+
+
+    cursor.toArray ((err, res) => {
+      debug ('calculating resvSize: aggregation pipeline returns %o', err);
+      debug ('calculating resvSize: aggregation pipeline returns %o', res);
+      if (err) return callback (err);
+      if (res.length == 0) return callback (null, 0);
+      callback (null, res[0].r);
+    });
+  }
+*/
+
+
+  ///////////////////////////////////////////////////////////////////////////////
+  // private parts
+
+  //////////////////////////////////////////////////////////////////
+  // create needed indexes for O(1) functioning
+  ensureIndexes (cb) {
+    const tasks = [];
+
+    this._groups_vector.forEach (i => {
+      const idx = {};
+      idx[`mature.${i}`] = 1;
+      tasks.push (cb => this._col.createIndex (idx, cb));
+    });
+    tasks.push (cb => this._col.createIndex ({t: 1}, {expireAfterSeconds: this._opts.ttl}, cb));
+    async.series (tasks, cb);
+  }
+}
+
+
+class Factory extends QFactory_MongoDB_defaults {
+  constructor (opts, mongo_conn) {
+    super (opts);
+    this._mongo_conn = mongo_conn;
+    this._db = mongo_conn.db();
+  }
+
+  queue (name, opts) {
+    const full_opts = {};
+    _.merge(full_opts, this._opts, opts);
+    return new StreamMongoQueue (name, this, full_opts, opts);
+  }
+
+  close (cb) {
+    super.close (() => {
+      if (this._mongo_conn) {
+        this._mongo_conn.close ();
+        this._mongo_conn = null;
+      }
+
+      if (cb) return cb ();
+    });
+  }
+
+  type () {
+    return StreamMongoQueue.Type ();
+  }
+
+  capabilities () {
+    return {
+      sched:    true,
+      reserve:  true,
+      pipeline: false,
+      tape:     true,
+      remove:   false,
+      stream:   true
+    };
+  }
+}
+
+function creator (opts, cb) {
+  const _opts = opts || {};
+  const m_url = _opts.url || 'mongodb://localhost:27017/keuss';
+
+  MongoClient.connect (m_url, { useNewUrlParser: true }, (err, cl) => {
+    if (err) return cb (err);
+    const F = new Factory (_opts, cl);
+    F.async_init (err => cb (null, F));
+  });
+}
+
+module.exports = creator;
+
+
+
+
+