@itentialopensource/adapter-utils 4.44.9

package/lib/throttle.js

@@ -0,0 +1,1289 @@
+/* @copyright Itential, LLC 2018 */
+
+// Set globals
+/* global pronghornProps log */
+/* eslint class-methods-use-this:warn */
+/* eslint comma-dangle: ["error", "never"] */
+/* eslint consistent-return:warn */
+/* eslint no-loop-func:warn */
+/* eslint no-param-reassign:warn */
+/* eslint no-underscore-dangle: [2, { "allow": ["_id"] }] */
+
+/* NodeJS internal API utilities */
+const uuid = require('uuid');
+const os = require('os');
+const fs = require('fs');
+
+/* Set up event emitter and other required references  */
+const AsyncLockCl = require('async-lock');
+
+let transUtilInst = null;
+let dbUtilInst = null;
+
+// Global Variables - From Properties
+let props = {};
+let numberPhs = 1;
+let syncAsync = 'sync';
+let MaxInQueue = 1000;
+let concurrentMax = 1;
+let expireTimeout = 0;
+let avgRuntime = 200;
+const priorityTable = [];
+
+// Other global variables
+let id = null;
+let phInstance = null;
+let queueColl = '_queue';
+const memQueue = [];
+const memQlock = 0;
+let avgQueue = [];
+const avgQlock = 0;
+let avgPtr = 1;
+const avgSize = 25;
+let avgTotal = 0;
+let qlock = null;
+// let MONGOQ;
+
+/* THROTTLE ENGINE INTERNAL FUNCTIONS         */
+/*
+ * INTERNAL FUNCTION: getQueueItems is used to retrieve queue items from
+ *    the database or memory based on the filter provided
+ */
+function getQueueItems(dbUI, collectionName, filter, callback) {
+  const origin = `${id}-throttle-getQueueItems`;
+  log.spam(origin);
+
+  try {
+    let myFilter = filter;
+
+    if (myFilter === null || myFilter === undefined) {
+      myFilter = {};
+    }
+
+    // If the number of Pronghorns is greater than 1, the queue is in the database
+    if (numberPhs > 1) {
+      // actual call to retrieve the queue items from the database
+      // return MONGOQ.find(myFilter).toArray((qerror, queueItems) => {
+      return dbUI.find(collectionName, filter, null, false, (err, res) => {
+        if (err) {
+          const errorObj = transUtilInst.formatErrorObject(origin, 'Find Error', [myFilter, err], null, null, null);
+          log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+          return callback(null, errorObj);
+        }
+        return res.toArray((qerror, queueItems) => {
+          if (qerror) {
+            const errorObj = transUtilInst.formatErrorObject(origin, 'No Queue Item', [myFilter, qerror], null, null, null);
+            log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+            return callback(null, errorObj);
+          }
+          return callback(queueItems);
+        });
+      });
+    }
+
+    if (myFilter.transNum !== undefined && myFilter.transNum.$lte !== undefined) {
+      // If the number of Pronghorns is 1, the queue is in memory
+      // need to add capabilities to searches as needed
+      // currently - before me, active and all
+      // Lock the memQueue while getting the ones before me
+      return qlock.acquire(memQlock, (done) => {
+        const tempQueue = [];
+
+        // push all of the transactions before me into the return queue
+        for (let i = 0; i < memQueue.length; i += 1) {
+          if (memQueue[i].transNum <= myFilter.transNum.$lte) {
+            tempQueue.push(memQueue[i]);
+          } else {
+            break;
+          }
+        }
+
+        // return the temp queue
+        done(tempQueue);
+      }, (ret, error) => {
+        if (error) {
+          const errorObj = transUtilInst.formatErrorObject(origin, 'No Queue Item', [myFilter, error], null, null, null);
+          log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+          return callback(null, errorObj);
+        }
+
+        return callback(ret);
+      });
+    }
+
+    if (myFilter.active !== undefined && myFilter.active === true) {
+      // currently - active and all
+      // Lock the memQueue while finding the active items
+      return qlock.acquire(memQlock, (done) => {
+        const activeOnes = [];
+
+        for (let i = 0; i < memQueue.length; i += 1) {
+          if (memQueue[i].active === true) {
+            activeOnes.push(memQueue[i]);
+          }
+        }
+
+        done(activeOnes);
+      }, (ret, error) => {
+        if (error) {
+          const errorObj = transUtilInst.formatErrorObject(origin, 'No Queue Item', [myFilter, error], null, null, null);
+          log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+          return callback(null, errorObj);
+        }
+
+        return callback(ret);
+      });
+    }
+
+    // unsupported filter - just return entire queue
+    // Lock the memQueue while cloning it
+    return qlock.acquire(memQlock, (done) => {
+      // return a clone so it is current state and not modified as we are looking
+      const tempQueue = memQueue.slice();
+      done(tempQueue);
+    }, (ret, error) => {
+      if (error) {
+        const errorObj = transUtilInst.formatErrorObject(origin, 'No Queue Item', [myFilter, error], null, null, null);
+        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+        return callback(null, errorObj);
+      }
+
+      return callback(ret);
+    });
+  } catch (e) {
+    // handle any exception
+    const errorObj = transUtilInst.checkAndReturn(e, origin, 'Issue getting queue items');
+    return callback(null, errorObj);
+  }
+}
+
+/*
+ * INTERNAL FUNCTION: claimTurn is used to start the request and
+ *    claim the turn. This means setting the start time and changing
+ *    the running and active flags to true.
+ */
+function claimTurn(dbUI, collectionName, queueItem, callback) {
+  const origin = `${id}-throttle-claimTurn`;
+  log.spam(origin);
+
+  try {
+    const cur = new Date();
+
+    // set up the update object to mark the queue item that the execution is starting
+    // set start, change active to true, change running to true
+    const data = {
+      _id: queueItem._id,
+      ph_instance: queueItem.ph_instance,
+      request_id: queueItem.request_id,
+      transNum: queueItem.transNum,
+      priority: queueItem.priority,
+      start: cur.getTime(),
+      end: queueItem.end,
+      active: true,
+      running: true,
+      event: queueItem.event
+    };
+
+    // If the number of Pronghorns is greater than 1, the queue is in the database
+    if (numberPhs > 1) {
+      // actual call to claim the turn from the queue in the database
+      // return MONGOQ.replaceOne({ _id: data._id }, data, (error, result) => {
+      return dbUI.replaceOne(collectionName, { _id: data._id }, data, {}, null, false, (error, result) => {
+        if (error) {
+          const errorObj = transUtilInst.formatErrorObject(origin, 'Unable To Claim Turn', [queueItem.request_id, queueItem.transNum, error], null, null, null);
+          log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+          return callback(null, errorObj);
+        }
+        log.debug(`${origin}: Mongo replace one returned ${result}`);
+        return callback(data);
+      });
+    }
+
+    // If the number of Pronghorns is 1, the queue is in memory
+    // Lock the memQueue while finding and updating the item
+    return qlock.acquire(memQlock, (done) => {
+      const index = memQueue.indexOf(queueItem);
+
+      if (index >= 0) {
+        memQueue[index] = data;
+        done(memQueue[index]);
+      } else {
+        done(null, 'Queue Item Not Found');
+      }
+    }, (ret, error) => {
+      if (error) {
+        const errorObj = transUtilInst.formatErrorObject(origin, 'Unable To Claim Turn', [queueItem.request_id, queueItem.transNum, error], null, null, null);
+        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+        return callback(null, errorObj);
+      }
+
+      return callback(ret);
+    });
+  } catch (e) {
+    // handle any exception
+    const errorObj = transUtilInst.checkAndReturn(e, origin, 'Issue claiming turn');
+    return callback(null, errorObj);
+  }
+}
+
+/*
+ * INTERNAL FUNCTION: freeQueueItem is used to free the turn from the given request.
+ *    This is done by deleting the item from the database or removing it from the
+ *    queue.
+ */
+function freeQueueItem(dbUI, collectionName, queueItem, callback) {
+  const origin = `${id}-throttle-freeQueueItem`;
+  log.spam(origin);
+
+  try {
+    // If the number of Pronghorns is greater than 1, the queue is in the database
+    if (numberPhs > 1) {
+      // actual call to remove the request from the queue in the database
+      // return MONGOQ.deleteOne({ _id: queueItem._id }, (error, result) => {
+      return dbUI.delete(collectionName, { _id: queueItem._id }, {}, false, null, false, (error, result) => {
+        if (error) {
+          const errorObj = transUtilInst.formatErrorObject(origin, 'Unable To Free Turn', [queueItem.request_id, queueItem.transNum, error], null, null, null);
+          log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+          return callback(null, errorObj);
+        }
+        return callback(result);
+      });
+    }
+
+    // If the number of Pronghorns is 1, the queue is in memory
+    // Lock the memQueue while finding and removing the item
+    return qlock.acquire(memQlock, (done) => {
+      const index = memQueue.indexOf(queueItem);
+
+      if (index >= 0) {
+        memQueue.splice(index, 1);
+        done('successfully removed from queue');
+      } else {
+        done(null, 'Queue Item Not Found');
+      }
+    }, (ret, error) => {
+      if (error) {
+        const errorObj = transUtilInst.formatErrorObject(origin, 'Unable To Free Turn', [queueItem.request_id, queueItem.transNum, error], null, null, null);
+        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+        return callback(null, errorObj);
+      }
+
+      return callback(ret);
+    });
+  } catch (e) {
+    // handle any exception
+    const errorObj = transUtilInst.checkAndReturn(e, origin, 'Issue freeing queue item');
+    return callback(null, errorObj);
+  }
+}
+
+/*
+ * INTERNAL FUNCTION: freeExpiredQueueItems goes through the current active usage to
+ *    determine if any have finished running long enough ago that the turn has expired.
+ *    If it has, this method calls freeQueueItem to have the turn freed in Pronghorn so
+ *    another request can get the turn. This is only used when there is some graceful
+ *    expiration to a concurrent run. If runs expire immediately, expireTimeout will
+ *    be 0 and this does nothing.
+ */
+function freeExpiredQueueItems(callback) {
+  const origin = `${id}-throttle-freeExpiredQueueItems`;
+  log.spam(origin);
+
+  try {
+    const status = 'success';
+
+    // if timeout is 0, this is handled in finish queue item as that is more efficient
+    if (expireTimeout === 0) {
+      return callback(status);
+    }
+
+    const cur = new Date();
+
+    // actual call to retrieve the in use turn from the database
+    return getQueueItems(dbUtilInst, queueColl, { active: true }, (currentUse, gerror) => {
+      if (gerror) {
+        return callback(null, gerror);
+      }
+
+      // if there is nothing to free just return
+      if (currentUse.length === 0) {
+        return callback(status);
+      }
+
+      let handled = 0;
+
+      // go through the queue items to see if any have expired
+      for (let i = 0; i < currentUse.length; i += 1) {
+        // if the transaction finished and the end + timeout is less than the current time
+        if (currentUse[i].end !== null
+            && (Number(currentUse[i].end) + Number(expireTimeout) < cur.getTime())) {
+          freeQueueItem(dbUtilInst, queueColl, currentUse[i], (flic, ferror) => {
+            if (ferror) {
+              return callback(null, ferror);
+            }
+
+            handled += 1;
+            // when we have handled everything - return
+            if (handled === currentUse.length) {
+              log.spam(`${origin}: ${flic}`);
+              return callback(status);
+            }
+          });
+        } else {
+          handled += 1;
+        }
+        // when we have handled everything - return
+        if (handled === currentUse.length) {
+          return callback(status);
+        }
+      }
+
+      return callback(status);
+    });
+  } catch (e) {
+    // handle any exception
+    const errorObj = transUtilInst.checkAndReturn(e, origin, 'Issue freeing queue items');
+    return callback(null, errorObj);
+  }
+}
+
+/*
+ * INTERNAL FUNCTION: checkTurnAvailable is used to check what is available
+ *    and what is ahead of me to determine if there is availability for my
+ *    use.
+ */
+function checkTurnAvailable(myRequest, myTransNum, callback) {
+  const origin = `${id}-throttle-checkTurnAvailable`;
+  log.spam(origin);
+
+  try {
+    return freeExpiredQueueItems((freeRes, ferror) => {
+      if (ferror) {
+        return callback(null, ferror);
+      }
+
+      log.spam(`${origin}: ${freeRes}`);
+
+      // only want to set filter if using DB - if memory want the entire queue so we get all priorities
+      let useFilter = null;
+      if (numberPhs > 1) {
+        useFilter = { transNum: { $lte: myTransNum } };
+      }
+
+      // actual call to retrieve the queue items from the database
+      return getQueueItems(dbUtilInst, queueColl, useFilter, (queueItems, gerror) => {
+        if (gerror) {
+          return callback(null, gerror);
+        }
+
+        let numActive = 0;
+        let beforeMe = 0;
+
+        // go through the queue items - need to determine if available and my place in queue
+        // Can not assume that I am last in the returned items or that they are in order as data
+        // from Mongo is not sorted - YET!
+        for (let i = 0; i < queueItems.length; i += 1) {
+          if (JSON.stringify(queueItems[i]) !== '{}') {
+            // if this item is active
+            if (queueItems[i].active) {
+              numActive += 1;
+            } else if (numberPhs > 1) {
+              // if database, need to be able to reorder since db doesn't maintain order
+              // if the item is not active but before me in the queue
+              if (Number(queueItems[i].transNum) < Number(myTransNum)) {
+                beforeMe += 1;
+              } else if (Number(queueItems[i].transNum) === Number(myTransNum)
+                  && queueItems[i].ph_instance < phInstance) {
+                // if the item is not active but before me in the queue (same trans - < pronghorn)
+                beforeMe += 1;
+              } else if (Number(queueItems[i].transNum) === Number(myTransNum)
+                  && queueItems[i].ph_instance === phInstance
+                  && Number(queueItems[i].request_id) < Number(myRequest)) {
+                // if the item is not active but before me in the queue (same trans but request id)
+                beforeMe += 1;
+              }
+            } else if (queueItems[i].transNum === Number(myTransNum)) {
+              // if this is me --- I know my index and can finish searching
+              beforeMe = i - numActive;
+              break;
+            }
+          }
+        }
+
+        // clear the memory from the queue items so it can be reclaimed
+        queueItems = undefined;
+
+        // number of spaces available (max - active)
+        const spaceAvail = Number(concurrentMax) - numActive;
+
+        // can I run? if turn available are greater then before me - yes
+        if (beforeMe < spaceAvail) {
+          return callback(0);
+        }
+
+        // otherwise the ones before can be reduced by the ones that are about to start running
+        return callback((beforeMe - spaceAvail) + 1);
+      });
+    });
+  } catch (e) {
+    // handle any exception
+    const errorObj = transUtilInst.checkAndReturn(e, origin, 'Issue checking turn');
+    return callback(null, errorObj);
+  }
+}
+
+/*
+ * INTERNAL FUNCTION: gettingCloseInterval checks if it is my turn under faster since
+ *    my turn is getting near.
+ */
+function gettingCloseInterval(myRequest, transNum, callback) {
+  const origin = `${id}-throttle-gettingCloseInterval`;
+  log.spam(origin);
+
+  try {
+    let intRun = false;
+    const fastInt = (avgTotal / avgSize) * 0.5;
+
+    // rapid inner interval - should be done when it is almost my time to run.
+    const intervalObject = setInterval(() => {
+      // Prevents running the interval ontop of itself in case interval time is less than time
+      // it takes to run
+      if (!intRun) {
+        intRun = true;
+
+        // check if there is an actual turn available
+        checkTurnAvailable(myRequest, transNum, (toRun, cerror) => {
+          if (cerror) {
+            return callback(null, cerror);
+          }
+
+          // is it my turn to run
+          if (toRun === 0) {
+            clearInterval(intervalObject);
+            return callback(true);
+          }
+
+          intRun = false;
+          log.debug(`${origin}: Request ${myRequest} Transaction ${transNum} waiting for turn to become free`);
+        });
+      }
+    }, fastInt);
+  } catch (e) {
+    // handle any exception
+    const errorObj = transUtilInst.checkAndReturn(e, origin, 'Issue during getting close');
+    return callback(null, errorObj);
+  }
+}
+
+class SystemXThrottle {
+  /**
+   * Throttle
+   * @constructor
+   */
+  constructor(prongid, properties, transUtilCl, dbUtilCl) {
+    this.myid = prongid;
+    id = prongid;
+    this.transUtil = transUtilCl;
+    this.dbUtil = dbUtilCl;
+
+    // set globals (available to private functions)
+    transUtilInst = this.transUtil;
+    dbUtilInst = this.dbUtil;
+
+    // this uniquely identifies this adapter on this pronghorn system
+    phInstance = `${id} - ${os.hostname()}`;
+    queueColl = id + queueColl;
+    this.qlockInst = new AsyncLockCl();
+    this.alock = new AsyncLockCl();
+    qlock = this.qlockInst;
+
+    // set up the properties I care about
+    this.refreshProperties(properties);
+  }
+
+  /**
+   * @callback verifyCallback
+   * @param {Boolean} result - the result of the verify ready
+   * @param {String} error - any error that occured
+   */
+  /**
+   * @callback queueCallback
+   * @param {Object} queueItem - the updated queue item
+   * @param {String} error - any error that occured
+   */
+
+  /* THROTTLE ENGINE EXTERNAL FUNCTIONS         */
+  /* refreshProperties - take in new properties without having to restart   */
+  /* verifyReady - verify that we are ready       */
+  /* requestQueueItem - put self in queue       */
+  /* waitingMyTurn - interval waiting turn to run   */
+  /* finishTurn - done using my turn to run       */
+  /**
+   * refreshProperties is used to set up all of the properties for the throttle engine.
+   * It allows properties to be changed later by simply calling refreshProperties rather
+   * than having to restart the throttle engine.
+   *
+   * @function refreshProperties
+   * @param {Object} properties - an object containing all of the properties
+   */
+  refreshProperties(properties) {
+    const origin = `${this.myid}-throttle-refreshProperties`;
+    log.trace(origin);
+    props = properties;
+
+    if (!props) {
+      log.error(`${origin}: Throttle received no properties!`);
+      return;
+    }
+
+    if (props.throttle) {
+      // set the throttle number of pronghorns (optional - default is 1)
+      if (props.throttle.number_pronghorns && Number(props.throttle.number_pronghorns) >= 1) {
+        numberPhs = Number(props.throttle.number_pronghorns);
+      }
+
+      // set the throttle synchronous or asynchronous (optional - default is synchronous)
+      if (props.throttle.sync_async && (props.throttle.sync_async === 'async'
+          || props.throttle.sync_async === 'asynchronous')) {
+        syncAsync = 'async';
+        log.error(`${origin}: Throttle Engine does not currently support async ${syncAsync}`);
+        syncAsync = 'sync';
+      }
+
+      // set the throttle maximum queue size (optional - default is 1000)
+      if (props.throttle.max_in_queue && Number(props.throttle.max_in_queue) > 0) {
+        MaxInQueue = Number(props.throttle.max_in_queue);
+      }
+
+      // set the throttle max (optional - default is 1)
+      if (props.throttle.concurrent_max !== null && Number(props.throttle.concurrent_max) > -1) {
+        if (props.throttle.concurrent_max === 0) {
+          log.warn(`${origin}: concurrent_max is set to 0. All requests will be blocked until concurrent_max is not 0`);
+        }
+        concurrentMax = Number(props.throttle.concurrent_max);
+      }
+
+      // set the expire timeout (optional - default is 0 milliseconds)
+      if (props.throttle.expire_timeout && Number(props.throttle.expire_timeout) >= 0) {
+        expireTimeout = Number(props.throttle.expire_timeout);
+      }
+
+      // set the queue interval (optional - default is 200) can not be less than 50ms
+      if (props.throttle.avg_runtime && Number(props.throttle.avg_runtime) >= 50) {
+        avgRuntime = Number(props.throttle.avg_runtime);
+      }
+
+      // set the priority table (default is empty)
+      if (props.throttle.priorities && props.throttle.priorities.length > 0) {
+        for (let p = 0; p < props.throttle.priorities.length; p += 1) {
+          if (props.throttle.priorities.value !== undefined && props.throttle.priorities.value !== null
+              && props.throttle.priorities.percent !== undefined && props.throttle.priorities.percent !== null) {
+            const prior = {
+              value: props.throttle.priorities.value,
+              percent: props.throttle.priorities.percent
+            };
+            priorityTable.push(prior);
+          }
+        }
+      }
+
+      // reset the average runtime queue
+      this.alock.acquire(avgQlock, (done) => {
+        avgQueue = [];
+        avgTotal = 0;
+
+        // load up the tools for calculating the average run time
+        for (let i = 0; i < avgSize; i += 1) {
+          avgQueue.push(avgRuntime);
+          avgTotal += avgRuntime;
+        }
+
+        done(avgTotal / avgSize);
+      }, (ret, error) => {
+        if (error) {
+          log.error(`${origin}: Error from updating average queue: ${error}`);
+        }
+
+        log.debug(`${origin}: Average run time now reset to: ${ret}`);
+      });
+    }
+  }
+
+  /**
+   * verifyReady is used to verify everything needed for throttling is set up. This
+   * generally means that it there is more than one Pronghorn, the database collection
+   * exists.
+   *
+   * @function verifyReady
+   * @param {Function} callback - a callback function to return whether throttle engine is ready
+   */
+  verifyReady(callback) {
+    const origin = `${this.myid}-throttle-verifyReady`;
+    log.trace(origin);
+
+    try {
+      // if we are using Mongo - make sure it is set up properly
+      if (numberPhs > 1) {
+        const adapterProps = pronghornProps.adapterProps.adapters;
+        let prongo = null;
+
+        // Find the 'pronghorn' db
+        for (let i = 0; i < adapterProps.length; i += 1) {
+          if (adapterProps[i].type === 'MongoDriver') {
+            prongo = adapterProps[i];
+            break;
+          }
+        }
+
+        if (prongo === null) {
+          const errorObj = this.transUtil.formatErrorObject(origin, 'Missing Data', ['Database Properties'], null, null, null);
+          log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+          return callback(null, errorObj);
+        }
+
+        // Connection URL
+        let murl = prongo.properties.url;
+        log.spam(`${origin}: ${murl}`);
+        const dbName = prongo.properties.db;
+
+        // define some local variables to help in validating the properties.json file
+        let sslEnabled = false;
+        let sslValidate = false;
+        let sslCheckServerIdentity = false;
+        let sslCA = null;
+        let replSetEnabled = false;
+        let dbAuthEnabled = false;
+        let dbUsername = 'pronghorn';
+        let dbPassword = 'pronghorn';
+
+        /*
+        * this first section is configuration mapping
+        * it can be replaced with the config object when available
+        */
+        if (prongo.properties.ssl) {
+          // enable ssl encryption?
+          if (prongo.properties.ssl.enabled === true) {
+            log.info(`${origin}: Connecting to MongoDB with SSL.`);
+            sslEnabled = true;
+            // validate the server's certificate against a known certificate authority?
+            if (prongo.properties.ssl.acceptInvalidCerts === false) {
+              sslValidate = true;
+              log.info(`${origin}: Certificate based SSL MongoDB connections will be used.`);
+              // if validation is enabled, we need to read the CA file
+              if (prongo.properties.ssl.sslCA) {
+                try {
+                  sslCA = [fs.readFileSync(prongo.properties.ssl.sslCA)];
+                } catch (err) {
+                  const errorObj = this.transUtil.formatErrorObject(origin, 'Missing File', ['CA FIle'], null, null, null);
+                  log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+                  return callback(null, errorObj);
+                }
+              } else {
+                const errorObj = this.transUtil.formatErrorObject(origin, 'Missing File', ['CA FIle'], null, null, null);
+                log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+                return callback(null, errorObj);
+              }
+            } else {
+              log.info(`${origin}: SSL MongoDB connection without CA certificate validation.`);
+            }
+            // validate the server certificate against the configured url?
+            if (prongo.properties.ssl.checkServerIdentity === true) {
+              sslCheckServerIdentity = true;
+            } else {
+              log.warn(`${origin}: WARNING: Skipping server identity validation`);
+            }
+          } else {
+            log.warn(`${origin}: WARNING: Connecting to MongoDB without SSL.`);
+          }
+        } else {
+          log.warn(`${origin}: WARNING: Connecting to MongoDB without SSL.`);
+        }
+
+        // are we using a replication set?
+        if (prongo.properties.replSet && prongo.properties.replSet.enabled === true) {
+          replSetEnabled = true;
+          murl = prongo.properties.url;
+        }
+        // are we using a username and password to authenticate?
+        if (prongo.properties.credentials) {
+          if (prongo.properties.credentials.dbAuth === true) {
+            dbAuthEnabled = true;
+          } else {
+            log.warn(`${origin}: WARNING: Connecting to MongoDB without user authentication.`);
+          }
+          if (prongo.properties.credentials.user) {
+            dbUsername = prongo.properties.credentials.user;
+          } else {
+            log.info(`${origin}: Using default mongo username`);
+          }
+          if (prongo.properties.credentials.passwd) {
+            dbPassword = prongo.properties.credentials.passwd;
+          } else {
+            log.info(`${origin}: Using default mongo password`);
+          }
+          if (dbAuthEnabled && (dbUsername === null || dbPassword === null)) {
+            const errorObj = this.transUtil.formatErrorObject(origin, 'Database Credentials', [], null, null, null);
+            log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+            return callback(null, errorObj);
+          }
+        } else {
+          log.warn(`${origin}: WARNING: Connecting to MongoDB without user authentication.`);
+        }
+
+        /*
+        * This second section is to construct the mongo options object
+        */
+        const opts = {
+          reconnectTries: 2000,
+          reconnectInterval: 1000,
+          ssl: sslEnabled,
+          sslValidate,
+          checkServerIdentity: sslCheckServerIdentity
+        };
+
+        const options = (replSetEnabled === true) ? { replSet: opts } : { server: opts };
+        log.debug(`${origin}: Connecting to MongoDB with options ${JSON.stringify(options)}`);
+
+        if (sslValidate === true) {
+          opts.sslCA = sslCA;
+        }
+
+        // Connect to the DB
+        log.info(`${origin}: Workflow Engine: Establishing connection to Pronghorn DB...`);
+
+        this.dbUtil.connect((alive, client) => {
+          if (!alive) {
+            const errorObj = this.transUtil.formatErrorObject(origin, 'Database Nonexistent', [], null, null, null);
+            log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+            return callback(null, errorObj);
+          }
+          log.info(`${origin}: Workflow Engine: Connection to Pronghorn DB Established`);
+          // Use db specified by properties file
+          const db = client.db(dbName);
+          /*
+          * once we are connected to mongo, we need to authenticate the connection
+          */
+          if (dbAuthEnabled) {
+            db.authenticate(dbUsername, dbPassword, (autherr, result) => {
+              if (autherr) {
+                const errorObj = this.transUtil.formatErrorObject(origin, 'Database Credentials', [], null, null, null);
+                log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+                return callback(null, errorObj);
+              }
+              if (result === false) {
+                const errorObj = this.transUtil.formatErrorObject(origin, 'Database Credentials', [], null, null, null);
+                log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+                return callback(null, errorObj);
+              }
+
+              /*
+              * finally, we need to perform a health check to ensure we can read from the database
+              */
+              log.info(`${origin}: Successfully authenticated with the Mongo DB server as user : ${dbUsername}`);
+            });
+          }
+
+          // See if the queue collection exists in the database
+          db.listCollections({ name: queueColl }).toArray((dberr, dbres) => {
+            // if it does not exist, create it and index it
+            if (dberr || dbres === null || dbres === undefined || dbres.length <= 0) {
+              log.info(`${origin}: Queue collection does not exist, creating new collection`);
+
+              // add the queue to the database
+              this.dbUtil.createCollection(queueColl, null, false, (error, result) => {
+                if (error) {
+                  const errorObj = this.transUtil.formatErrorObject(origin, 'Unable To Create Queue', [error], null, null, null);
+                  log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+                  return callback(null, errorObj);
+                }
+                log.info(`${origin}: ${result}`);
+                // Get the queue collection
+                this.dbUtil.createIndex(queueColl, { transNum: 1 }, {}, null, (inerr, inres) => {
+                  if (inerr) {
+                    log.error(`${origin}: Received an error on indexing ${inerr}`);
+                  } else {
+                    log.debug(`${origin}: Mongo ensure index returned ${inres}`);
+                  }
+                });
+                return callback(true);
+                // ensureIndex is deprecated, using Node.js MongoDB's createIndex method now.
+              });
+              // db.createCollection(queueColl, null, false, (data, error) => {
+              //   if (error) {
+              //     const errorObj = this.transUtil.formatErrorObject(origin, 'Unable To Create Queue', [error], null, null, null);
+              //     log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+              //     return callback(null, errorObj);
+              //   }
+
+              //   // Get the queue collection
+              //   MONGOQ = db.collection(queueColl);
+              //   MONGOQ.ensureIndex({ transNum: 1 }, (inerr, inres) => {
+              //     if (inerr) {
+              //       log.error(`${origin}: Received an error on indexing ${inerr}`);
+              //     } else {
+              //       log.debug(`${origin}: Mongo ensure index returned ${inres}`);
+              //     }
+              //   });
+
+              //   return callback(true);
+              // });
+            } else {
+              // if the collection already exists in the database
+              log.info(`${origin}: Queue collection check passed`);
+
+              // Get the queue collection
+              // MONGOQ = db.collection(queueColl);
+              this.dbUtil.createCollection(queueColl, null, false, (error, res) => {
+                if (error) {
+                  log.error(`${origin}: ${error}`);
+                  return callback(null, error);
+                }
+                log.info(`${origin}: ${res}`);
+                this.dbUtil.createIndex(queueColl, { transNum: 1 }, {}, null, (inerr, inres) => {
+                  if (inerr) {
+                    log.error(`${origin}: Received an error on indexing ${inerr}`);
+                  } else {
+                    log.debug(`${origin}: Mongo ensure index returned ${inres}`);
+                  }
+                });
+                // ensureIndex is deprecated, using Node.js MongoDB's createIndex method now.
+              });
+              // MONGOQ.ensureIndex({ transNum: 1 }, (inerr, inres) => {
+              //   if (inerr) {
+              //     log.error(`Received an error on indexing ${inerr}`);
+              //   }
+
+              //   log.debug(`Mongo ensure index returned ${inres}`);
+              // });
+
+              // Delete any queue belonging to this adapter
+              // These requests are no longer waiting/processing since the adapter went down
+              const deleteFilter = {
+                ph_instance: phInstance
+              };
+
+              this.dbUtil.delete(queueColl, deleteFilter, {}, true, null, false, (delerr, res) => {
+                if (delerr) {
+                  const errorObj = this.transUtil.formatErrorObject(origin, 'Unable To Clear Queue', [delerr], null, null, null);
+                  log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+                  return callback(null, errorObj);
+                }
+                log.debug(`${origin}: Mongo delete many returned ${res}`);
+                return callback(true);
+              });
+              // MONGOQ.deleteMany(deleteFilter, (delerr, res) => {
+              //   if (delerr) {
+              //     const errorObj = this.transUtil.formatErrorObject(origin, 'Unable To Clear Queue', [delerr], null, null, null);
+              //     log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+              //     return callback(null, errorObj);
+              //   }
+
+              //   log.debug(`Mongo delete many returned ${res}`);
+              //   return callback(true);
+              // });
+            }
+          });
+        });
+      } else {
+        // if in memory queue nothing to verify
+        return callback(true);
+      }
+    } catch (e) {
+      // handle any exception
+      const errorObj = this.transUtil.checkAndReturn(e, origin, 'Issue during verify ready');
+      return callback(null, errorObj);
+    }
+  }
+
+  /**
+   * requestQueueItem is used to request a queue item putting the request on the queue
+   * so that it can acquire the turn when it is available.
+   *
+   * @function requestQueueItem
+   * @param {String} myRequest - identifies the request. unique to this Pronghorn (required)
+   * @param {String} transNum - something to denote approximate order
+   *                            (e.g. time of the request) (required)
+   * @param {Number} priority - identifies the priorityfor the request (optional)
+   * @param {Function} callback - a callback function to return the resulting Queue Object
+   */
+  requestQueueItem(myRequest, transNum, priority, event, callback) {
+    const origin = `${this.myid}-throttle-requestQueueItem`;
+    log.trace(origin);
+
+    try {
+      if (myRequest === null || myRequest === undefined) {
+        const errorObj = this.transUtil.formatErrorObject(origin, 'Missing Data', ['request id'], null, null, null);
+        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+        return callback(null, errorObj);
+      }
+      if (transNum === null || transNum === undefined) {
+        const errorObj = this.transUtil.formatErrorObject(origin, 'Missing Data', ['transaction number'], null, null, null);
+        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+        return callback(null, errorObj);
+      }
+
+      // if priority
+      let myPercent = 100;
+      if (priority !== undefined && priority !== null && priority !== -1) {
+        // find the priority
+        for (let p = 0; p < priorityTable.length; p += 1) {
+          if (priority === priorityTable[p].value) {
+            try {
+              myPercent = Number(priorityTable[p].percent);
+            } catch (exp) {
+              myPercent = 100;
+            }
+            break;
+          }
+        }
+      }
+      // must make percent legitimate
+      if (myPercent < 0 || myPercent > 100) {
+        myPercent = 100;
+      }
+
+      // set up the queue item request - set instance, request, transNum,
+      // and set active and running flags to false
+      const data = {
+        _id: uuid.v4(),
+        ph_instance: phInstance,
+        request_id: myRequest,
+        transNum: Number(transNum),
+        priority: myPercent,
+        start: null,
+        end: null,
+        active: false,
+        running: false,
+        event
+      };
+
+      // If the number of Pronghorns is greater than 1, the queue is in the database
+      if (numberPhs > 1) {
+        // determine if there is space in the queue
+
+        // count is now deprecated, use countDocuments instead.
+        // operator replacements (count => countDocuments): $where => $expr, $near => $geoWithin with $center, $nearSphere => $geoWithin with $centerSphere
+        // return MONGOQ.count({}, (gerror, qsize) => {
+        return this.dbUtil.countDocuments(queueColl, {}, {}, null, false, (gerror, qsize) => {
+          if (gerror) {
+            const errorObj = this.transUtil.formatErrorObject(origin, 'Database Error', [gerror], null, null, null);
+            log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+            return callback(null, errorObj);
+          }
+
+          // check to see if queue maxed out
+          if (qsize >= Number(MaxInQueue)) {
+            const errorObj = this.transUtil.formatErrorObject(origin, 'Queue Full', [myRequest, transNum, qsize], null, null, null);
+            log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+            return callback(null, errorObj);
+          }
+
+          // write my queue item into the database queue
+          // return MONGOQ.insertOne(data, (error, result) => {
+          return this.dbUtil.create(queueColl, data, null, false, (error, result) => {
+            if (error) {
+              const errorObj = this.transUtil.formatErrorObject(origin, 'Database Error', [error], null, null, null);
+              log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+              return callback(null, errorObj);
+            }
+
+            log.debug(`${origin}: Mongo insert one returned ${result}`);
+            return callback(data);
+          });
+        });
+      }
+
+      // If the number of Pronghorns is 1, the queue is in memory
+      // Lock the memQueue while adding the item
+      return qlock.acquire(memQlock, (done) => {
+        // if the queue is less than concurrentMax length -
+        // just add it to queue at end as it will run right away
+        if (memQueue.length < concurrentMax) {
+          memQueue.push(data);
+          done(memQueue[memQueue.length - 1]);
+        } else if (myPercent >= 0 && myPercent < 100) {
+          // if prioritized - put it at the right place in the queue
+          let myIndex = Math.round(memQueue.length * (myPercent / 100));
+
+          // if myIndex is less than what is running, insert after running ones
+          if (myIndex < concurrentMax) {
+            myIndex = concurrentMax + 1;
+          }
+
+          // if at the end push
+          if (myIndex >= memQueue.length) {
+            myIndex = memQueue.length;
+            memQueue.push(data);
+          } else {
+            // need to put behind any other equal or higher priority requests already in the queue
+            for (let pos = myIndex; pos < memQueue.length; pos += 1) {
+              // when we find a lower priority request, we will put this one in front of it so break the loop
+              if (memQueue[pos].priority === undefined || memQueue[pos].priority === null || memQueue[pos].priority > myPercent) {
+                myIndex = pos;
+                break;
+              }
+            }
+
+            // if not at the end, insert after current index
+            memQueue.splice(myIndex, 0, data);
+          }
+
+          // return my item
+          done(memQueue[myIndex]);
+        } else if (memQueue.length >= Number(MaxInQueue)) {
+          // check to see if queue maxed out
+          done(null, 'Queue Full');
+        } else {
+          // put at the end of the queue
+          memQueue.push(data);
+
+          // return my item
+          done(memQueue[memQueue.length - 1]);
+        }
+      }, (ret, error) => {
+        if (error) {
+          const errorObj = this.transUtil.formatErrorObject(origin, 'Queue Full', [myRequest, transNum, memQueue.length], null, null, null);
+          log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+          return callback(null, errorObj);
+        }
+        return callback(ret);
+      });
+    } catch (e) {
+      // handle any exception
+      const errorObj = this.transUtil.checkAndReturn(e, origin, 'Issue while requesting a queue item');
+      return callback(null, errorObj);
+    }
+  }
+
+  /**
+   * waitingMyTurn is used to determine when is it safe for the item to run. It attempts to see
+   * if there is availability first, if it can not (because all spaces are in use) it will put the
+   * attempts into an interval that re attempts. The interval is based on where in the queue and
+   * approximate time before there will be availability.
+   *
+   * @function waitingMyTurn
+   * @param {Object} queueItem - the queue request which would have been returned from
+   *                             requestQueueItem (required)
+   * @param {Function} callback - a callback function to return the resulting Queue Object
+   */
+  waitingMyTurn(queueItem, callback) {
+    const origin = `${this.myid}-throttle-waitingMyTurn`;
+    log.trace(origin);
+
+    try {
+      if (queueItem === null || queueItem === undefined) {
+        const errorObj = this.transUtil.formatErrorObject(origin, 'Missing Data', ['queue item'], null, null, null);
+        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+        return callback(null, errorObj);
+      }
+      if (queueItem.request_id === undefined || Number(queueItem.request_id) < 0) {
+        const errorObj = this.transUtil.formatErrorObject(origin, 'Missing Data', ['queue item -> request id'], null, null, null);
+        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+        return callback(null, errorObj);
+      }
+      if (queueItem.transNum === undefined || Number(queueItem.transNum) < 0) {
+        const errorObj = this.transUtil.formatErrorObject(origin, 'Missing Data', ['queue item -> transaction number'], null, null, null);
+        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+        return callback(null, errorObj);
+      }
+      if ((queueItem.active !== undefined && queueItem.active)
+          || (queueItem.end !== undefined && queueItem.end !== null)) {
+        const errorObj = this.transUtil.formatErrorObject(origin, 'Item In Wrong State', ['already started'], null, null, null);
+        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+        return callback(null, errorObj);
+      }
+
+      // check if there is an actual turn available
+      return checkTurnAvailable(queueItem.request_id, queueItem.transNum, (toRun, cerror) => {
+        if (cerror) {
+          return callback(null, cerror);
+        }
+
+        // is it my turn to run
+        if (toRun === 0) {
+          return claimTurn(this.dbUtil, queueColl, queueItem, callback);
+        }
+
+        if (toRun <= (3 * concurrentMax)) {
+          // if I have less than 3 wait cycles, OK to start running checks faster
+          return gettingCloseInterval(queueItem.request_id, queueItem.transNum, (waitEnd, gerror) => {
+            if (gerror) {
+              return callback(null, gerror);
+            }
+
+            log.spam(`${origin}: ${waitEnd}`);
+            return claimTurn(this.dbUtil, queueColl, queueItem, callback);
+          });
+        }
+        // set divisor to be 1 if concurrentMax is 0
+        const concurrentMaxDivisor = concurrentMax !== 0 ? concurrentMax : 1;
+        // calculate the wait for the outer timeout (this one needs to get us close to our run time)
+        let outerInterval = (toRun / concurrentMaxDivisor) * (expireTimeout + (avgTotal / avgSize));
+
+        // Use the 90% of the outer intverval or the outer Interval -2 seconds which ever is greater
+        if ((outerInterval * 0.1) > 2000) {
+          outerInterval -= 2000;
+        } else {
+          outerInterval *= 0.95;
+        }
+
+        log.debug(`${origin}: Request ${queueItem.request_id} Transaction ${queueItem.transNum} Outer Interval set to: ${outerInterval}`);
+
+        // outer interval to get a turn request
+        // The outer interval is really used to get us close to when we should run
+        const intervalObject = setTimeout(() => {
+          clearTimeout(intervalObject);
+
+          // recurrsive call in case re-estimate is needed
+          // (abort, things taking longer than approximation, etc)
+          return this.waitingMyTurn(queueItem, callback);
+        }, outerInterval);
+      });
+    } catch (e) {
+      // handle any exception
+      const errorObj = this.transUtil.checkAndReturn(e, origin, 'Issue while waiting turn');
+      return callback(null, errorObj);
+    }
+  }
+
+  /**
+   * finishTurn is used to mark the request completed. This means setting the end time
+   * and changing the running flag to false. If there is no graceful expiration of the
+   * concurrent request, then this makes the call to free the turn.
+   *
+   * @function finishTurn
+   * @param {Object} queueItem - the queue request which would have been returned from
+   *                             waitingMyTurn (required)
+   * @param {Number} reqEnd - the time it took to execute the call (optional)
+   * @param {Function} callback - a callback function to return the finished Queue Object
+   */
+  finishTurn(queueItem, reqEnd, callback) {
+    const origin = `${this.myid}-throttle-finishTurn`;
+    log.trace(origin);
+
+    try {
+      if (queueItem === null || queueItem === undefined) {
+        const errorObj = this.transUtil.formatErrorObject(origin, 'Missing Data', ['queue item'], null, null, null);
+        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+        return callback(null, errorObj);
+      }
+      if (queueItem.start === undefined || queueItem.start === null
+          || queueItem.active === undefined || !queueItem.active) {
+        const errorObj = this.transUtil.formatErrorObject(origin, 'Item In Wrong State', ['not started'], null, null, null);
+        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+        return callback(null, errorObj);
+      }
+      if (queueItem.end !== undefined && queueItem.end !== null) {
+        const errorObj = this.transUtil.formatErrorObject(origin, 'Item In Wrong State', ['already ended'], null, null, null);
+        log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+        return callback(null, errorObj);
+      }
+
+      const cur = new Date();
+
+      // set up the update object to mark the queue item that the execution finished
+      // set end and change running to false
+      const data = {
+        _id: queueItem._id,
+        ph_instance: queueItem.ph_instance,
+        request_id: queueItem.request_id,
+        transNum: queueItem.transNum,
+        priority: queueItem.priority,
+        start: queueItem.start,
+        end: cur.getTime(),
+        active: queueItem.active,
+        running: false,
+        event: queueItem.event
+      };
+
+      // Lock the avgQlock while update the average time
+      if (reqEnd !== null && reqEnd !== '') {
+        this.alock.acquire(avgQlock, (done) => {
+          const reqTimeMS = reqEnd / 100000;
+          avgTotal = (avgTotal - avgQueue[avgPtr]) + reqTimeMS;
+          avgQueue[avgPtr] = reqTimeMS;
+          avgPtr += 1;
+
+          if (avgPtr === avgSize) {
+            avgPtr = 0;
+          }
+
+          done(avgTotal / avgSize);
+        }, (ret, error) => {
+          if (error) {
+            log.error(`${origin}: Error from updating average queue: ${error}`);
+          }
+
+          log.debug(`${origin}: Average run time now set to: ${ret}`);
+        });
+      }
+
+      // if there is no timeout, delete the request and free the turn
+      if (expireTimeout === 0) {
+        return freeQueueItem(this.dbUtil, queueColl, queueItem, callback);
+      }
+
+      if (numberPhs > 1) {
+        // otherwise mark the request done, can not free it yet
+        // If the number of Pronghorns is greater than 1, the queue is in the database
+        // return MONGOQ.replaceOne({ _id: data._id }, data, (error, result) => {
+        return this.dbUtil.replaceOne(queueColl, { _id: data._id }, data, {}, null, false, (error, result) => {
+          if (error) {
+            const errorObj = this.transUtil.formatErrorObject(origin, 'Database Error', ['could not update item'], null, null, null);
+            log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+            return callback(null, errorObj);
+          }
+
+          log.debug(`${origin}: Mongo replace one returned ${result}`);
+          return callback(data);
+        });
+      }
+
+      // If the number of Pronghorns is 1, the queue is in memory
+      // Lock the memQueue while finding and updating the item
+      return qlock.acquire(memQlock, (done) => {
+        const index = memQueue.indexOf(queueItem);
+
+        if (index >= 0) {
+          memQueue[index] = data;
+          done(memQueue[index]);
+        } else {
+          done(null, 'item not found in queue');
+        }
+      }, (ret, error) => {
+        if (error) {
+          const errorObj = this.transUtil.formatErrorObject(origin, 'No Queue Item', ['data'], null, null, null);
+          log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+          return callback(null, errorObj);
+        }
+
+        return callback(ret);
+      });
+    } catch (e) {
+      // handle any exception
+      const errorObj = this.transUtil.checkAndReturn(e, origin, 'Issue during finish turn');
+      return callback(null, errorObj);
+    }
+  }
+
+  /**
+   * getQueue is used to get information for all of the requests currently in the queue.
+   *
+   * @function getQueue
+   * @param {Function} callback - a callback function to return the Queue
+   */
+  getQueue(callback) {
+    const origin = `${this.myid}-throttle-getQueue`;
+    log.trace(origin);
+
+    try {
+      return getQueueItems(this.dbUtil, queueColl, null, callback);
+    } catch (e) {
+      // handle any exception
+      const errorObj = this.transUtil.checkAndReturn(e, origin, 'Issue during get queue');
+      return callback(null, errorObj);
+    }
+  }
+}
+
+module.exports = SystemXThrottle;