@itentialopensource/adapter-utils 4.44.9

package/lib/dbUtil.js

@@ -0,0 +1,1300 @@
+/* Required libraries.  */
+/* global g_redis log */
+/* eslint consistent-return:warn */
+/* eslint no-underscore-dangle: [2, { "allow": ["_id"] }] */
+/* eslint no-unused-vars:warn */
+
+const fs = require('fs-extra');
+const uuid = require('uuid');
+
+/* Fetch in the other needed components for the this Adaptor */
+const { MongoClient } = require('mongodb');
+
+// Other global variables
+let adapterDir = '';
+let saveFS = false;
+let storDir = `${adapterDir}/storage`;
+let id = null;
+
+const Storage = {
+  UNDEFINED: 0,
+  DBINFO: 1,
+  ADAPTERDB: 2,
+  FILESYSTEM: 3,
+  IAPDB: 4
+};
+Object.freeze(Storage);
+
+/* DB UTILS INTERNAL FUNCTIONS         */
+/** getFromJson
+ * @summary returns information from a json file on the file system
+ */
+function getFromJson(fileName, filter) {
+  const origin = `${id}-dbUtil-getFromJson`;
+  log.trace(origin);
+
+  // verify the required data has been provided
+  if (fileName === undefined || fileName === null || fileName === '') {
+    log.warn(`${origin}: Must provide a file name`);
+    return null;
+  }
+
+  // get the entity and the action if this is a metric
+  let ent = null;
+  let act = null;
+  let metric = false;
+  const useFilter = filter;
+  if (useFilter && useFilter.metric) {
+    metric = true;
+    if (useFilter.metric.entity) {
+      ent = useFilter.metric.entity;
+    }
+    if (useFilter.metric.action) {
+      act = useFilter.metric.action;
+    }
+    if (!ent || !act) {
+      log.warn(`${origin}: metric provided with one of entity/action, require both`);
+      return null;
+    }
+    delete useFilter.metric;
+  }
+
+  try {
+    // make sure what we need exists and has been provided
+    if (!fs.existsSync(`${storDir}`)) {
+      log.warn(`${origin}: Could not find adapter storage directory - nothing to retrieve`);
+      return null;
+    }
+
+    // determine if the file exists so we retrieve the data
+    if (fs.existsSync(`${storDir}/${fileName}.json`)) {
+      const content = JSON.parse(fs.readFileSync(`${storDir}/${fileName}.json`, 'utf-8'));
+      const toReturn = [];
+
+      if (metric) {
+        // if metric need to match the entity and action
+        content.table.forEach((item) => {
+          if (ent === item.entity && act === item.action) {
+            toReturn.push(item);
+          }
+        });
+      } else {
+        // if not metric must match the items in the filter
+        const key = Object.keys(useFilter);
+        content.table.forEach((item) => {
+          let push = true;
+          key.forEach((fil) => {
+            if (useFilter[fil] !== item[fil]) push = false;
+          });
+          if (push) toReturn.push(item);
+        });
+      }
+      const filtered = content.table.filter((el) => {
+        Object.keys(useFilter).forEach((obj) => {
+          if (el[obj] !== useFilter[obj]) {
+            return false;
+          }
+        });
+        return true;
+      });
+      return toReturn;
+    }
+
+    // if no file, nothing to return
+    return null;
+  } catch (ex) {
+    log.warn(`${origin}: Caught Exception ${ex}`);
+    return null;
+  }
+}
+
+/** countJSON
+ * @summary returns a count from a json storage file
+ */
+function countJSON(fileName, filter) {
+  const origin = `${id}-dbUtil-countJSON`;
+  log.trace(origin);
+
+  // verify the required data has been provided
+  if (fileName === undefined || fileName === null || fileName === '') {
+    log.warn(`${origin}: Must provide a file name`);
+    return null;
+  }
+
+  try {
+    // make sure what we need exists and has been provided
+    if (!fs.existsSync(`${storDir}`)) {
+      log.warn(`${origin}: Could not find adapter storage directory - nothing to count`);
+      return null;
+    }
+
+    // determine if the file exists so we can count data from it
+    if (fs.existsSync(`${storDir}/${fileName}.json`)) {
+      const data = getFromJson(fileName, filter);
+      if (data) {
+        return data.length;
+      }
+      return -1;
+    }
+  } catch (ex) {
+    log.warn(`${origin}: Caught Exception ${ex}`);
+    return null;
+  }
+}
+
+/** saveAsJson
+ * @summary saves information into a json storage file
+ */
+function saveAsJson(fileName, data) {
+  const origin = `${id}-dbUtil-saveAsJson`;
+  log.trace(origin);
+
+  // verify the required data has been provided
+  if (fileName === undefined || fileName === null || fileName === '') {
+    log.warn(`${origin}: Must provide a file name`);
+    return null;
+  }
+
+  // get the entity and the action if this is a metric
+  let ent = null;
+  let act = null;
+  let metric = false;
+  const useData = data;
+  if (useData && useData.metric) {
+    metric = true;
+    if (useData.metric.entity) {
+      ent = useData.metric.entity;
+    }
+    if (useData.metric.action) {
+      act = useData.metric.action;
+    }
+    if (!ent || !act) {
+      log.warn(`${origin}: metric provided with one of entity/action, require both`);
+      return null;
+    }
+    delete useData.metric;
+  }
+
+  try {
+    // make sure what we need exists and has been provided
+    if (!fs.existsSync(`${storDir}`)) {
+      // need to make the storage directory if it does not exist
+      fs.mkdirSync(`${storDir}`);
+    }
+
+    // determine if the file exists so we add data to it
+    if (fs.existsSync(`${storDir}/${fileName}.json`)) {
+      // have to read, append & save
+      const content = JSON.parse(fs.readFileSync(`${storDir}/${fileName}.json`, 'utf-8'));
+      let exists = false;
+
+      content.table.forEach((item) => {
+        const toPush = item;
+        if (metric) {
+          // if it is a metric and we already have entity and action need to edit the data
+          if (ent === item.entity && act === item.action) {
+            exists = true;
+            Object.keys(useData).forEach((key) => {
+              if (key === '$inc') {
+                Object.keys(useData[key]).forEach((inc) => {
+                  if (!toPush[inc]) {
+                    toPush[inc] = useData[key][inc];
+                  }
+                  toPush[inc] += useData[key][inc];
+                });
+              } else if (key === '$set') {
+                Object.keys(useData[key]).forEach((set) => {
+                  toPush[set] = useData[key][set];
+                });
+              }
+            });
+          }
+        }
+      });
+      if (!exists) {
+        // push new thing to table.
+        const toPush = {};
+        const keysArray = Object.keys(useData);
+        keysArray.forEach((i) => {
+          const newKeys = Object.keys(useData[i]);
+          newKeys.forEach((j) => {
+            toPush[j] = useData[i][j];
+          });
+        });
+        content.table.push(toPush);
+      }
+
+      // now that is updated, write the file back out
+      fs.writeFileSync(`${storDir}/${fileName}.json`, JSON.stringify(content, null, 2));
+      return useData;
+    }
+
+    // if the file has not been created yet
+    const obj = { table: [] };
+    const toPush = {};
+    const keysArray = Object.keys(data);
+    keysArray.forEach((outer) => {
+      const newKeys = Object.keys(data[outer]);
+      newKeys.forEach((inner) => {
+        toPush[inner] = data[outer][inner];
+      });
+    });
+    obj.table.push(toPush);
+
+    // write the file out to the file system
+    fs.writeFileSync(`${storDir}/${fileName}.json`, JSON.stringify(obj, null, 2));
+    return data;
+  } catch (ex) {
+    log.warn(`${origin}: Caught Exception ${ex}`);
+    return null;
+  }
+}
+
+/** removeFromJSON
+ * @summary removes information from a json storage file
+ */
+function removeFromJSON(fileName, filter, multiple) {
+  const origin = `${id}-dbUtil-removeFromJSON`;
+  log.trace(origin);
+
+  // verify the required data has been provided
+  if (fileName === undefined || fileName === null || fileName === '') {
+    log.warn(`${origin}: Must provide a file name`);
+    return null;
+  }
+
+  // get the entity and the action if this is a metric
+  let ent = null;
+  let act = null;
+  let metric = false;
+  const useFilter = filter;
+  if (useFilter && useFilter.metric) {
+    metric = true;
+    if (useFilter.metric.entity) {
+      ent = useFilter.metric.entity;
+    }
+    if (useFilter.metric.action) {
+      act = useFilter.metric.action;
+    }
+    if (!ent && !act) {
+      log.warn(`${origin}: metric provided with one of entity/action, require both`);
+      return null;
+    }
+    delete useFilter.metric;
+  }
+
+  try {
+    // make sure what we need exists and has been provided
+    if (!fs.existsSync(`${storDir}`)) {
+      log.warn(`${origin}: Could not find adapter storage directory - nothing to remove`);
+      return null;
+    }
+
+    // determine if the file exists so we remove data from it
+    if (fs.existsSync(`${storDir}/${fileName}.json`)) {
+      const content = JSON.parse(fs.readFileSync(`${storDir}/${fileName}.json`, 'utf-8'));
+      const toReturn = [];
+
+      // if this is a metric make sure the entity and action match
+      if (metric) {
+        content.table = content.table.filter((item) => {
+          if (ent === item.entity && act === item.action) {
+            toReturn.push(item);
+            return false;
+          }
+          return true;
+        });
+      } else {
+        // go through content to determine if item matches the filter
+        let ctr = 0;
+        // create the contents that are being removed
+        const removed = content.table.filter((el) => {
+          Object.keys(useFilter).forEach((obj) => {
+            if (el[obj] !== useFilter[obj]) {
+              return false;
+            }
+          });
+          ctr += 1;
+          if (!multiple && ctr > 1) {
+            return false;
+          }
+          return true;
+        });
+        let ctr1 = 0;
+        // remove the items from the contents
+        content.table = content.table.filter((el, i) => {
+          Object.keys(useFilter).forEach((obj) => {
+            if (el[obj] !== useFilter[obj]) {
+              return true;
+            }
+          });
+          ctr1 += 1;
+          if (!multiple && ctr1 > 1) {
+            return true;
+          }
+          return false;
+        });
+
+        // write the contents back to the file
+        fs.writeFileSync(`${storDir}/${fileName}.json`, JSON.stringify(content, null, 2));
+        return removed;
+      }
+
+      // write the contents back to the file
+      fs.writeFileSync(`${storDir}/${fileName}.json`, JSON.stringify(content, null, 2));
+      return toReturn;
+    }
+
+    log.error(`${origin}: Collection ${fileName} does not exist`);
+    return null;
+  } catch (ex) {
+    log.warn(`${origin}: Caught Exception ${ex}`);
+    return null;
+  }
+}
+
+/** deleteJSON
+ * @summary deletes a json storage file
+ */
+function deleteJSON(fileName) {
+  const origin = `${id}-dbUtil-deleteJSON`;
+  log.trace(origin);
+
+  // verify the required data has been provided
+  if (fileName === undefined || fileName === null || fileName === '') {
+    log.warn(`${origin}: Must provide a file name`);
+    return null;
+  }
+
+  try {
+    // make sure what we need exists and has been provided
+    if (!fs.existsSync(`${storDir}`)) {
+      log.warn(`${origin}: Could not find adapter storage directory - nothing to delete`);
+      return null;
+    }
+
+    // determine if the file exists so we remove - also assume we add a .json suffix
+    if (fs.existsSync(`${storDir}/${fileName}.json`)) {
+      fs.remove(`${storDir}/${fileName}.json`).catch((some) => {
+        log.info(`${origin}: ${some}`);
+        fs.rmdirSync(`${storDir}/${fileName}.json`);
+      });
+    }
+
+    // successful -- return the fileName
+    return fileName;
+  } catch (ex) {
+    log.warn(`${origin}: Caught Exception ${ex}`);
+    return null;
+  }
+}
+
+class DBUtil {
+  /**
+   * These are database utilities that can be used by the adapter to interact with a
+   * mongo database
+   * @constructor
+   */
+  constructor(prongId, properties, directory) {
+    this.myid = prongId;
+    id = prongId;
+    this.baseDir = directory;
+    adapterDir = this.baseDir;
+    storDir = `${adapterDir}/storage`;
+    this.props = properties;
+    this.adapterMongoClient = null;
+
+    // set up the properties I care about
+    this.refreshProperties(properties);
+  }
+
+  /**
+   * refreshProperties is used to set up all of the properties for the db utils.
+   * It allows properties to be changed later by simply calling refreshProperties rather
+   * than having to restart the db utils.
+   *
+   * @function refreshProperties
+   * @param {Object} properties - an object containing all of the properties
+   */
+  refreshProperties(properties) {
+    const origin = `${this.myid}-dbUtil-refreshProperties`;
+    log.trace(origin);
+    this.dburl = null;
+    this.dboptions = {};
+    this.database = this.myid;
+
+    // verify the necessary information was received
+    if (!properties) {
+      log.error(`${origin}: DB Utils received no properties!`);
+      return;
+    }
+    if (!properties.mongo || !properties.mongo.host) {
+      log.info(`${origin}: No default adapter database configured!`);
+      return;
+    }
+
+    // set the database port
+    let port = 27017;
+    if (properties.mongo.port) {
+      port = properties.mongo.port;
+    }
+    // set the database
+    if (properties.mongo.database) {
+      this.database = properties.mongo.database;
+    }
+    // set the user
+    let username = null;
+    if (properties.mongo.username) {
+      username = properties.mongo.username;
+    }
+    // set the password
+    let password = null;
+    if (properties.mongo.password) {
+      password = properties.mongo.password;
+    }
+
+    // format the database url
+    this.dburl = 'mongodb://';
+    log.info(`${origin}: Default adapter database at: ${properties.mongo.host}`);
+
+    if (username) {
+      this.dburl += `${encodeURIComponent(username)}:${encodeURIComponent(password)}@`;
+      log.info(`${origin}: Default adapter database will use authentication.`);
+    }
+    this.dburl += `${encodeURIComponent(properties.mongo.host)}:${encodeURIComponent(port)}/${encodeURIComponent(this.database)}`;
+
+    // are we using a replication set need to add it to the url
+    if (properties.mongo.replSet) {
+      this.dburl += `?${properties.mongo.replSet}`;
+      log.info(`${origin}: Default adapter database will use replica set.`);
+    }
+
+    // Do we need SSL to connect to the database
+    if (properties.mongo.db_ssl && properties.mongo.db_ssl.enabled === true) {
+      log.info(`${origin}: Default adapter database will use SSL.`);
+      this.dboptions.ssl = true;
+
+      // validate the server's certificate against a known certificate authority?
+      if (properties.mongo.db_ssl.accept_invalid_cert === false) {
+        this.dboptions.sslValidate = true;
+        log.info(`${origin}: Default adapter database will use Certificate based SSL.`);
+        // if validation is enabled, we need to read the CA file
+        if (properties.mongo.db_ssl.ca_file) {
+          try {
+            this.dboptions.sslCA = [fs.readFileSync(properties.mongo.db_ssl.ca_file)];
+          } catch (err) {
+            log.error(`${origin}: Error: unable to load Mongo CA file: ${err}`);
+          }
+        } else {
+          log.error(`${origin}: Error: Certificate validation is enabled but a CA is not specified.`);
+        }
+        if (properties.mongo.db_ssl.key_file) {
+          try {
+            this.dboptions.sslKey = [fs.readFileSync(properties.mongo.db_ssl.key_file)];
+          } catch (err) {
+            log.error(`${origin}: Error: Unable to load Mongo Key file: ${err}`);
+          }
+        }
+        if (properties.mongo.db_ssl.cert_file) {
+          try {
+            this.dboptions.sslCert = [fs.readFileSync(properties.mongo.db_ssl.cert_file)];
+          } catch (err) {
+            log.error(`${origin}: Error: Unable to load Mongo Certificate file: ${err}`);
+          }
+        }
+      } else {
+        this.dboptions.sslValidate = false;
+        log.info(`${origin}: Default adapter database not using Certificate based SSL.`);
+      }
+    } else {
+      log.info(`${origin}: Default adapter database not using SSL.`);
+    }
+
+    // if we were provided with a path to save metrics - instead of just a boolean
+    // will assume this is a place where all adapter stuff can go!
+    saveFS = this.props.save_metric || false;
+    if (saveFS && typeof saveFS === 'string' && saveFS !== '') {
+      storDir = saveFS;
+    }
+  }
+
+  /**
+   * Call to determine the storage target. If the storage is a Mongo database, then
+   * the client connection and database is returned.
+   *
+   * @function determineStorage
+   * @param {object} dbInfo - the url for the database to connect to (dburl, dboptions, database)
+   * @param {callback} callback - the error, the storage, the Mongo client connection, the Mongo database
+   */
+  determineStorage(dbInfo, callback) {
+    const origin = `${this.myid}-dbUtil-determineStorage`;
+
+    if (dbInfo) {
+      // priority 1 - use the dbInfo passed in
+      if (dbInfo.dburl && dbInfo.database) {
+        MongoClient.connect(dbInfo.dburl, dbInfo.dboptions, (err, mongoClient) => {
+          if (err) {
+            log.error(`${origin}: Error! Failed to connect to database: ${err}`);
+            return callback(err, null, null, null);
+          }
+          log.debug('using dbinfo');
+          return callback(null, Storage.DBINFO, mongoClient, dbInfo.database);
+        });
+      } else {
+        const err = 'Error! Marlformed dbInfo';
+        return callback(err, null, null, Storage.DBINFO);
+      }
+    } else if (this.adapterMongoClient) {
+      // priority 2 - use the adapter database
+      log.debug('using adapter db');
+      return callback(null, Storage.ADAPTERDB, this.adapterMongoClient, this.database);
+    } else if (this.dburl === null && dbInfo === null) {
+      // priority 3 - use the filesystem
+      log.debug('using filesystem');
+      return callback(null, Storage.FILESYSTEM, null, null);
+    }
+  }
+
+  /**
+   * Call to connect and authenticate to the adapter database
+   *
+   * @function connect
+   */
+  connect(callback) {
+    // callback format: (this.alive, mongoClient); generally mongoClient should only be used if this.alive is true.
+    const origin = `${this.myid}-dbUtil-connect`;
+    log.trace(origin);
+
+    // const options = (replSetEnabled === true) ? { replSet: opts } : { server: opts };
+    log.debug(`${origin}: Connecting to MongoDB with options ${JSON.stringify(this.dboptions)}`);
+
+    // Now we will start the process of connecting to mongo db
+    return MongoClient.connect(this.dburl, this.dboptions, (err, mongoClient) => {
+      if (!mongoClient) {
+        log.error(`${origin}: Error! Exiting... Must start MongoDB first ${err}`);
+        this.alive = false;
+        return callback(this.alive);
+      }
+      mongoClient.on('close', () => {
+        this.alive = false;
+        log.error(`${origin}: MONGO CONNECTION LOST...`);
+      });
+
+      mongoClient.on('reconnect', () => {
+        // we still need to check if we are properly authenticated
+        // so we just list collections to test it.
+        this.clientDB.listCollections().toArray((error) => {
+          if (error) {
+            log.error(`${origin}: ${error}`);
+            this.alive = false;
+          } else {
+            log.info(`${origin}: MONGO CONNECTION BACK...`);
+            this.alive = true;
+            this.adapterMongoClient = mongoClient;
+          }
+        });
+      });
+
+      mongoClient.db(this.database).serverConfig.on('left', (type) => {
+        if (type === 'primary') {
+          this.alive = false;
+          log.info(`${origin}: MONGO PRIMARY CONNECTION LOST...`);
+        } else if (type === 'secondary') {
+          log.info(`${origin}: MONGO SECONDARY CONNECTION LOST...`);
+        }
+      });
+
+      mongoClient.db(this.database).serverConfig.on('joined', (type) => {
+        if (type === 'primary') {
+          log.info(`${origin}: MONGO PRIMARY CONNECTION BACK...`);
+        } else if (type === 'secondary') {
+          log.info(`${origin}: MONGO SECONDARY CONNECTION BACK...`);
+        }
+      });
+
+      log.info(`${origin}: mongo running @${this.dburl}/${this.database}`);
+      this.clientDB = mongoClient.db(this.database);
+      this.adapterMongoClient = mongoClient;
+
+      // we don't have authentication defined but we still need to check if Mongo does not
+      // require one, so we just list collections to test if it's doable.
+      return this.clientDB.listCollections().toArray((error) => {
+        if (error) {
+          log.error(`${origin}: ${error}`);
+          this.alive = false;
+        } else {
+          log.info(`${origin}: MongoDB connection has been established`);
+          this.alive = true;
+          this.adapterMongoClient = mongoClient;
+        }
+        return callback(this.alive, mongoClient);
+      });
+    });
+  }
+
+  /**
+   * Call to disconnect from the adapter database
+   *
+   * @function disconnect
+   */
+  disconnect() {
+    if (this.adapterMongoClient) {
+      this.adapterMongoClient.close();
+    }
+  }
+
+  /**
+   * createCollection creates the provided collection in the file system or database.
+   *
+   * @function createCollection
+   * @param {string} collectionName - the name of the collection to create
+   * @param {object} dbInfo - the url for the database to connect to (dburl, dboptions, database)
+   * @param {boolean} fsWrite - turn on write to the file system
+   */
+  createCollection(collectionName, dbInfo, fsWrite, callback) {
+    const origin = `${this.myid}-dbUtil-createCollection`;
+    log.trace(origin);
+
+    try {
+      // verify the required data has been provided
+      if (!collectionName || (typeof collectionName !== 'string')) {
+        log.warn(`${origin}: Missing Collection Name or not string`);
+        return callback(`${origin}: Missing Collection Name or not string`, null);
+      }
+
+      return this.determineStorage(dbInfo, (storageError, storage, mongoClient, database) => {
+        if (storageError) {
+          return callback(`${origin}: ${storageError}`, null);
+        }
+
+        // if using file storage
+        if (storage === Storage.FILESYSTEM) {
+          // if there is no adapter directory - can not do anything so error
+          if (!fs.existsSync(`${this.baseDir}`)) {
+            log.warn(`${origin}: Not able to create storage - missing base directory!`);
+            return callback(`${origin}: Not able to create storage - missing base directory!`, null);
+          }
+          // if there is no storage directory - create it
+          if (!fs.existsSync(`${this.baseDir}/storage`)) {
+            fs.mkdirSync(`${this.baseDir}/storage`);
+          }
+          // if the collection already exists - no need to create it
+          if (fs.existsSync(`${this.baseDir}/storage/${collectionName}`)) {
+            log.debug(`${origin}: storage file collection already exists`);
+            return callback(null, collectionName);
+          }
+          // create the new collection on the file system
+          fs.mkdirSync(`${this.baseDir}/storage/${collectionName}`);
+          log.debug(`${origin}: storage file collection ${collectionName} created`);
+          return callback(null, collectionName);
+        }
+
+        // if using MongoDB storage
+        if (storage === Storage.DBINFO || storage === Storage.ADAPTERDB) {
+          return mongoClient.db(database).createCollection(collectionName, (err, res) => {
+            if (storage === Storage.DBINFO && mongoClient) mongoClient.close();
+            if (err) {
+              // error we get back if the collection already existed - not a true error
+              if (err.codeName === 'NamespaceExists') {
+                log.debug(`${origin}: database collection already exists`);
+                return callback(null, collectionName);
+              }
+              log.warn(`${origin}: Error creating collection ${err}`);
+              return callback(`${origin}: Error creating collection ${err}`, null);
+            }
+            log.spam(`${origin}: db response ${res}`);
+            log.debug(`${origin}: database collection ${collectionName} created`);
+            return callback(null, collectionName);
+          });
+        }
+      });
+    } catch (ex) {
+      log.warn(`${origin}: Caught Exception - ${ex}`);
+      return callback(`${origin}: Caught Exception - ${ex}`, null);
+    }
+  }
+
+  /**
+   * removeCollection removes the provided collection from the file system or database.
+   *
+   * @function removeCollection
+   * @param {string} collectionName - the name of the collection to remove
+   * @param {object} dbInfo - the url for the database to connect to (dburl, dboptions, database)
+   * @param {boolean} fsWrite - turn on write to the file system
+   */
+  removeCollection(collectionName, dbInfo, fsWrite, callback) {
+    const origin = `${this.myid}-dbUtil-removeCollection`;
+    log.trace(origin);
+
+    try {
+      // verify the required data has been provided
+      if (!collectionName || (typeof collectionName !== 'string')) {
+        log.warn(`${origin}: Missing Collection Name or not string`);
+        return callback(`${origin}: Missing Collection Name or not string`, null);
+      }
+
+      return this.determineStorage(dbInfo, (storageError, storage, mongoClient, database) => {
+        if (storageError) {
+          return callback(`${origin}: ${storageError}`, null);
+        }
+
+        // if using file storage
+        if (storage === Storage.FILESYSTEM) {
+          const deld = deleteJSON(collectionName);
+          if (deld) {
+            log.debug(`${origin}: storage file collection ${collectionName} removed`);
+            return callback(null, deld);
+          }
+          log.debug(`${origin}: could not remove storage file collection`);
+          return callback(`${origin}: could not remove storage file collection`, null);
+        }
+
+        // if using MongoDB storage
+        if (storage === Storage.DBINFO || storage === Storage.ADAPTERDB) {
+          // get the list of collections from the database - can we just drop?
+          return mongoClient.db(database).listCollections().toArray((err, result) => {
+            if (err) {
+              log.warn(`${origin}: Failed to get collections ${err}`);
+              return callback(`${origin}: Failed to get collections ${err}`, null);
+            }
+            // go through the collections to get the correct one for removal
+            for (let e = 0; e < result.length; e += 1) {
+              if (result[e].name === collectionName) {
+                // now that we found it, remove it
+                return mongoClient.db(database).collection(collectionName).drop({}, (err1, res) => {
+                  if (storage === Storage.DBINFO && mongoClient) mongoClient.close();
+                  if (err1) {
+                    log.warn(`${origin}: Failed to remove collection ${err1}`);
+                    return callback(`${origin}: Failed to remove collection ${err1}`, null);
+                  }
+                  log.spam(`${origin}: db response ${res}`);
+                  log.debug(`${origin}: database collection ${collectionName} removed`);
+                  return callback(null, collectionName);
+                });
+              }
+            }
+            return callback(null, collectionName);
+          });
+        }
+      });
+    } catch (ex) {
+      log.warn(`${origin}: Caught Exception - ${ex}`);
+      return callback(`${origin}: Caught Exception - ${ex}`, null);
+    }
+  }
+
+  /**
+   * Call to create an item in the database
+   *
+   * @function create
+   * @param {string} collectionName - the collection to save the item in. (required)
+   * @param {string} data - the modification to make. (required)
+   * @param {object} dbInfo - the url for the database to connect to (dburl, dboptions, database)
+   * @param {boolean} fsWrite - turn on write to the file system
+   */
+  create(collectionName, data, dbInfo, fsWrite, callback) {
+    const origin = `${this.myid}-dbUtil-create`;
+    log.trace(origin);
+
+    try {
+      // verify the required data has been provided
+      if (!collectionName) {
+        log.warn(`${origin}: Missing Collection Name`);
+        return callback(`${origin}: Missing Collection Name`, null);
+      }
+      if (!data) {
+        log.warn(`${origin}: Missing data to add`);
+        return callback(`${origin}: Missing data to add`, null);
+      }
+      if ((data.entity && !data.action) || (!data.entity && data.action)) {
+        log.warn(`${origin}: Inconsistent entity/action set`);
+        return callback(`${origin}: Inconsistent entity/action set`, null);
+      }
+
+      // create the unique identifier (should we let mongo do this?)
+      const dataInfo = data;
+      if (!{}.hasOwnProperty.call(dataInfo, '_id')) {
+        dataInfo._id = uuid.v4();
+      }
+
+      return this.determineStorage(dbInfo, (storageError, storage, mongoClient, database) => {
+        if (storageError) {
+          return callback(`${origin}: ${storageError}`, null);
+        }
+
+        // if using file storage
+        if (storage === Storage.FILESYSTEM) {
+          // save it to file in the adapter storage directory
+          const saved = saveAsJson(collectionName, data);
+          if (!saved) {
+            log.warn(`${origin}: Data has not been saved to file storage`);
+            return callback(`${origin}: Data has not been saved to file storage`, null);
+          }
+          log.debug(`${origin}: Data saved in file storage`);
+          return callback(null, saved);
+        }
+
+        // if using MongoDB storage
+        if (storage === Storage.DBINFO || storage === Storage.ADAPTERDB) {
+          // Add the data to the database
+          // insertOne has only 2 parameters: the data to be added & callback. Not an identifier.
+          return mongoClient.db(database).collection(collectionName).insertOne(dataInfo, (err, result) => {
+            if (storage === Storage.DBINFO && mongoClient) mongoClient.close();
+            if (err) {
+              log.warn(`${origin}: Failed to insert data in collection ${err}`);
+              return callback(`${origin}: Failed to insert data in collection ${err}`, null);
+            }
+            log.spam(`${origin}: db response ${result}`);
+            log.debug(`${origin}: Data saved in database`);
+            return callback(null, data);
+          });
+        }
+      });
+    } catch (ex) {
+      log.warn(`${origin}: Caught Exception - ${ex}`);
+      return callback(`${origin}: Caught Exception - ${ex}`, null);
+    }
+  }
+
+  /**
+   * Call to create an index in the database
+   *
+   * @function createIndex
+   * @param {string} collectionName - the collection to index. (required)
+   * @param {string} fieldOrSpec - what to index. (required)
+   * @param {object} dbInfo - the url for the database to connect to (dburl, dboptions, database)
+   */
+  createIndex(collectionName, fieldOrSpec, options, dbInfo, callback) {
+    const origin = `${this.myid}-dbUtil-createIndex`;
+    log.trace(origin);
+
+    try {
+      // verify the required data has been provided
+      if (!collectionName) {
+        log.warn(`${origin}: Missing Collection Name`);
+        return callback(`${origin}: Missing Collection Name`, null);
+      }
+      if (!fieldOrSpec) {
+        log.warn(`${origin}: Missing Specs`);
+        return callback(`${origin}: Missing Specs`, null);
+      }
+
+      return this.determineStorage(dbInfo, (storageError, storage, mongoClient, database) => {
+        if (storageError) {
+          return callback(`${origin}: ${storageError}`, null);
+        }
+
+        // if using file storage
+        if (storage === Storage.FILESYSTEM) {
+          // no database - no index
+          log.warn(`${origin}: No database - no index`);
+          return callback(`${origin}: No database - no index`, null);
+        }
+
+        // if using MongoDB storage
+        if (storage === Storage.DBINFO || storage === Storage.ADAPTERDB) {
+          // create the index on the collection
+          return mongoClient.db(database).collection(collectionName).createIndex(fieldOrSpec, options || {}, (err, res) => {
+            if (storage === Storage.DBINFO && mongoClient) mongoClient.close();
+            if (err) {
+              log.warn(`${origin}: Failed to index data in collection ${err}`);
+              return callback(`${origin}: Failed to index data in collection ${err}`, null);
+            }
+            log.debug(`${origin}: Data in collection indexed`);
+            return callback(null, res);
+          });
+        }
+      });
+    } catch (ex) {
+      log.warn(`${origin}: Caught Exception - ${ex}`);
+      return callback(`${origin}: Caught Exception - ${ex}`, null);
+    }
+  }
+
+  /**
+   * Call to count the documents in a collection
+   *
+   * @function countDocuments
+   * @param {string} collectionName - the collection to count documents in. (required)
+   * @param {object} query - the query to minimize documents you count. (required)
+   * @param {object} dbInfo - the url for the database to connect to (dburl, dboptions, database)
+   * @param {boolean} fsWrite - turn on write to the file system
+   */
+  countDocuments(collectionName, query, options, dbInfo, fsWrite, callback) {
+    const origin = `${this.myid}-dbUtil-countDocuments`;
+    log.trace(origin);
+
+    try {
+      // verify the required data has been provided
+      if (!collectionName) {
+        log.warn(`${origin}: Missing Collection Name`);
+        return callback(`${origin}: Missing Collection Name`, null);
+      }
+
+      return this.determineStorage(dbInfo, (storageError, storage, mongoClient, database) => {
+        if (storageError) {
+          return callback(`${origin}: ${storageError}`, null);
+        }
+
+        // if using file storage
+        if (storage === Storage.FILESYSTEM) {
+          // get a count from the JSON
+          const data = countJSON(collectionName, query);
+          if (!data || data === -1) {
+            log.warn(`${origin}: Could not count data from file storage`);
+            return callback(`${origin}: Could not count data from file storage`, null);
+          }
+          log.debug(`${origin}: Count from file storage ${data}`);
+          return callback(null, data);
+        }
+
+        // if using MongoDB storage
+        if (storage === Storage.DBINFO || storage === Storage.ADAPTERDB) {
+          // get the count from mongo
+          return mongoClient.db(database).collection(collectionName).count(query, options, (err, res) => {
+            if (storage === Storage.DBINFO && mongoClient) mongoClient.close();
+            if (err) {
+              log.warn(`${origin}: Failed to count collection ${err}`);
+              return callback(`${origin}: Failed to count collection ${err}`, null);
+            }
+            log.debug(`${origin}: Count from database ${res}`);
+            return callback(null, res);
+          });
+        }
+      });
+    } catch (ex) {
+      log.warn(`${origin}: Caught Exception - ${ex}`);
+      return callback(`${origin}: Caught Exception - ${ex}`, null);
+    }
+  }
+
+  /**
+   * Delete items from a collection
+   *
+   * @function delete
+   * @param {string} collectionName - the collection to remove document from. (required)
+   * @param {object} filter - the filter for the document(s) to remove. (required)
+   * @param {object} dbInfo - the url for the database to connect to (dburl, dboptions, database)
+   * @param {boolean} fsWrite - turn on write to the file system
+   */
+  delete(collectionName, filter, options, multiple, dbInfo, fsWrite, callback) {
+    const origin = `${this.myid}-dbUtil-delete`;
+    log.trace(origin);
+
+    try {
+      // verify the required data has been provided
+      if (!collectionName) {
+        log.warn(`${origin}: Missing Collection Name`);
+        return callback(`${origin}: Missing Collection Name`, null);
+      }
+      if (!filter) {
+        log.warn(`${origin}: Missing Filter`);
+        return callback(`${origin}: Missing Filter`, null);
+      }
+      if (multiple === undefined || multiple === null) {
+        log.warn(`${origin}: Missing Multiple flag`);
+        return callback(`${origin}: Missing Multiple flag`, null);
+      }
+
+      return this.determineStorage(dbInfo, (storageError, storage, mongoClient, database) => {
+        if (storageError) {
+          return callback(`${origin}: ${storageError}`, null);
+        }
+
+        // if using file storage
+        if (storage === Storage.FILESYSTEM) {
+          // verify the collection exists
+          if (!fs.existsSync(`${adapterDir}/storage/${collectionName}.json`)) {
+            log.warn(`${origin}: Collection ${collectionName} does not exist`);
+            return callback(null, `${origin}: Collection ${collectionName} does not exist`);
+          }
+          // remove the item from the collection
+          const deld = removeFromJSON(collectionName, filter, multiple);
+          if (!deld) {
+            log.warn(`${origin}: Data has not been deleted from file storage`);
+            return callback(`${origin}: Data has not been deleted from file storage`, null);
+          }
+          log.debug(`${origin}: Data has been deleted from file storage`);
+          return callback(null, deld);
+        }
+
+        // if using MongoDB storage
+        if (storage === Storage.DBINFO || storage === Storage.ADAPTERDB) {
+          if (!multiple) {
+            // delete the single item from mongo
+            return mongoClient.db(database).collection(collectionName).deleteOne(filter, options, (err, res) => {
+              if (storage === Storage.DBINFO && mongoClient) mongoClient.close();
+              if (err) {
+                log.warn(`${origin}: Failed to delete data from database ${err}`);
+                return callback(`${origin}: Failed delete data from database ${err}`, null);
+              }
+              log.debug(`${origin}: Data has been deleted from database`);
+              return callback(null, res);
+            });
+          }
+
+          // delete the multiple items from mongo
+          return mongoClient.db(database).collection(collectionName).deleteMany(filter, options, (err, res) => {
+            if (storage === Storage.DBINFO && mongoClient) mongoClient.close();
+            if (err) {
+              log.warn(`${origin}: Failed to delete data from database ${err}`);
+              return callback(`${origin}: Failed delete data from database ${err}`, null);
+            }
+            log.debug(`${origin}: Data has been deleted from database`);
+            return callback(null, res);
+          });
+        }
+      });
+    } catch (ex) {
+      log.warn(`${origin}: Caught Exception - ${ex}`);
+      return callback(`${origin}: Caught Exception - ${ex}`, null);
+    }
+  }
+
+  /**
+   * Replace an item in a collection
+   *
+   * @function replaceOne
+   * @param {string} collectionName - the collection to replace document in. (required)
+   * @param {object} filter - the filter for the document(s) to replace. (required)
+   * @param {object} doc - the filter for the document(s) to replace. (required)
+   * @param {object} dbInfo - the url for the database to connect to (dburl, dboptions, database)
+   * @param {boolean} fsWrite - turn on write to the file system
+   */
+  replaceOne(collectionName, filter, doc, options, dbInfo, fsWrite, callback) {
+    const origin = `${this.myid}-dbUtil-replaceOne`;
+    log.trace(origin);
+
+    try {
+      // verify the required data has been provided
+      if (!collectionName) {
+        log.warn(`${origin}: Missing Collection Name`);
+        return callback(`${origin}: Missing Collection Name`, null);
+      }
+      if (!filter) {
+        log.warn(`${origin}: Missing Filter`);
+        return callback(`${origin}: Missing Filter`, null);
+      }
+      if (!doc) {
+        log.warn(`${origin}: Missing Document`);
+        return callback(`${origin}: Missing Document`, null);
+      }
+
+      return this.determineStorage(dbInfo, (storageError, storage, mongoClient, database) => {
+        if (storageError) {
+          return callback(`${origin}: ${storageError}`, null);
+        }
+
+        // if using file storage
+        if (storage === Storage.FILESYSTEM) {
+          // remove the data from the collection
+          const rem = removeFromJSON(collectionName, filter, false);
+          if (rem) {
+            // add the data into the collection
+            const sav = saveAsJson(collectionName, doc);
+            if (sav) {
+              log.debug(`${origin}: Data replaced in file storage`);
+              return callback(null, sav);
+            }
+            log.warn(`${origin}: Could not save doc into file storage`);
+            return callback(`${origin}: Could not save doc into file storage`, null);
+          }
+          log.warn(`${origin}: Could not delete from file storage`);
+          return callback(`${origin}: Could not delete from file storage`, null);
+        }
+
+        // if using MongoDB storage
+        if (storage === Storage.DBINFO || storage === Storage.ADAPTERDB) {
+          // replace an items in mongo
+          return mongoClient.db(database).collection(collectionName).replaceOne(filter, doc, options, (err, res) => {
+            if (storage === Storage.DBINFO && mongoClient) mongoClient.close();
+            if (err) {
+              log.warn(`${origin}: Failed to replace data in database ${err}`);
+              return callback(`${origin}: Failed replace data in database ${err}`, null);
+            }
+            log.debug(`${origin}: Data replaced in file storage`);
+            return callback(null, res);
+          });
+        }
+      });
+    } catch (ex) {
+      log.warn(`${origin}: Caught Exception - ${ex}`);
+      return callback(`${origin}: Caught Exception - ${ex}`, null);
+    }
+  }
+
+  /**
+   * Call to find items in the database
+   *
+   * @function find
+   * @param {string} collectionName - the collection name to search in. (required)
+   * @param {object} options - the options to use to find data.
+   * options = {
+   *  filter : <filter Obj>,
+   *  sort : <sort Obj>,
+   *  start : <start position>,
+   *  limit : <limit of results>,
+   * }
+   * @param {object} dbInfo - the url for the database to connect to (dburl, dboptions, database)
+   * @param {boolean} fsWrite - turn on write to the file system
+   */
+  find(collectionName, options, dbInfo, fsWrite, callback) {
+    const origin = `${this.myid}-dbUtil-find`;
+    log.trace(origin);
+
+    try {
+      // verify the required data has been provided
+      if (!collectionName) {
+        log.warn(`${origin}: Missing Collection Name`);
+        return callback(`${origin}: Missing Collection Name`, null);
+      }
+
+      // get the collection so we can run the remove on the collection
+      let filter = {};
+      let start = 0;
+      let sort = {};
+      let limit = 10;
+      if (options) {
+        filter = options.filter || {};
+        start = options.start || 0;
+        sort = options.sort || {};
+        if (Object.hasOwnProperty.call(options, 'limit')) {
+          ({ limit } = options);
+        }
+      }
+
+      // If limit is not specified, default to 10.
+      // Note: limit may be 0, which is equivalent to setting no limit.
+
+      // Replace filter with regex to allow for substring lookup
+      // TODO: Need to create a new filter object instead of mutating the exsisting one
+      const filterKeys = Object.keys(filter).filter((key) => (key[0] !== '$' && typeof filter[key] === 'string'));
+      filterKeys.map((key) => {
+        try {
+          const escapedFilter = filter[key].replace(/([.?*+^$[\]\\(){}|-])/g, '\\$1');
+          const regexedFilter = new RegExp(`.*${escapedFilter}.*`, 'i');
+          filter[key] = {
+            $regex: regexedFilter
+          };
+        } catch (e) {
+          delete filter[key];
+        }
+        return key;
+      });
+
+      return this.determineStorage(dbInfo, (storageError, storage, mongoClient, database) => {
+        if (storageError) {
+          return callback(`${origin}: ${storageError}`, null);
+        }
+
+        // if using file storage
+        if (storage === Storage.FILESYSTEM) {
+          // Find it from file in the adapter
+          let toReturn = getFromJson(collectionName, filter);
+          if (toReturn && toReturn.length > limit) {
+            let curEnd = start + limit;
+            if (curEnd < toReturn.length) {
+              curEnd = toReturn.length;
+            }
+            toReturn = toReturn.slice(start, curEnd);
+          }
+          log.trace(`${origin}: Data retrieved from file storage`);
+          return callback(null, toReturn);
+        }
+
+        // if using MongoDB storage
+        if (storage === Storage.DBINFO || storage === Storage.ADAPTERDB) {
+          // Find the data in the database
+          return mongoClient.db(database).collection(collectionName).find(filter).sort(sort)
+            .skip(start)
+            .limit(limit)
+            .toArray()
+            .then((value) => {
+              log.debug(`${origin}: Data retrieved from database`);
+              return callback(null, value);
+            })
+            .finally(() => {
+              if (storage === Storage.DBINFO && mongoClient) mongoClient.close();
+            });
+        }
+      });
+    } catch (ex) {
+      log.warn(`${origin}: Caught Exception - ${ex}`);
+      return callback(`${origin}: Caught Exception - ${ex}`, null);
+    }
+  }
+
+  /**
+   * Call to find an item in a collection and modify it.
+   *
+   * @function findAndModify
+   * @param {string} collectionName - the collection to find things from. (required)
+   * @param {object} filter - the filter used to find objects. (optional)
+   * @param {array} sort - how to sort the items (first one in order will be modified). (optional)
+   * @param {object} data - the modification to make. (required)
+   * @param {boolean} upsert - option for the whether to insert new objects. (optional)
+   * @param {object} dbInfo - the url for the database to connect to (dburl, dboptions, database) (optional)
+   * @param {boolean} fsWrite - turn on write to the file system (optional)
+   * @param {updateCallback} callback - a callback function to return a result
+   *                                    (the new object) or the error
+   */
+  findAndModify(collectionName, filter, sort, data, upsert, dbInfo, fsWrite, callback) {
+    const origin = `${this.myid}-dbUtil-findAndModify`;
+    log.trace(origin);
+
+    try {
+      // verify the required data has been provided
+      if (collectionName === undefined || collectionName === null || collectionName === '') {
+        log.warn(`${origin}: Missing Collection Name`);
+        return callback(`${origin}: Missing Collection Name`, null);
+      }
+      if (data === undefined || data === null || typeof data !== 'object' || Object.keys(data).length === 0) {
+        log.warn(`${origin}: Missing data for modification`);
+        return callback(`${origin}: Missing data for modification`, null);
+      }
+
+      const options = {
+        sort,
+        upsert,
+        returnOriginal: false
+      };
+
+      return this.determineStorage(dbInfo, (storageError, storage, mongoClient, database) => {
+        if (storageError) {
+          return callback(`${origin}: ${storageError}`, null);
+        }
+
+        // if using file storage
+        if (storage === Storage.FILESYSTEM) {
+          // save it to file in the adapter storage directory
+          const saved = saveAsJson(collectionName, data);
+          if (!saved) {
+            log.error(`${origin}: Data has not been saved`);
+            return callback(`${origin}: Data has not been saved`, null);
+          }
+          log.debug(`${origin}: Data modified in file storage`);
+          return callback(null, saved);
+        }
+
+        // if using MongoDB storage
+        if (storage === Storage.DBINFO || storage === Storage.ADAPTERDB) {
+          // find and modify the data in the database
+          return mongoClient.db(database).collection(collectionName).findOneAndUpdate((filter || {}), data, options, (err, result) => {
+            if (storage === Storage.DBINFO && mongoClient) mongoClient.close();
+            if (err) {
+              log.warn(`${origin}: Failed to modified data in database ${err}`);
+              return callback(`${origin}: Failed modified data in database ${err}`, null);
+            }
+            log.debug(`${origin}: Data modified in database`);
+            return callback(null, result);
+          });
+        }
+      });
+    } catch (ex) {
+      log.warn(`${origin}: Caught Exception - ${ex}`);
+      return callback(`${origin}: Caught Exception - ${ex}`, null);
+    }
+  }
+}
+
+module.exports = DBUtil;