@itentialopensource/adapter-kafkav2 0.19.0 → 0.21.0

package/utils/dbUtil.js

@@ -0,0 +1,1603 @@
+/* Required libraries.  */
+/* global g_redis log */
+/* eslint consistent-return:warn */
+/* eslint no-underscore-dangle: [2, { "allow": ["_id"] }] */
+/* eslint no-unused-vars:warn */
+
+const fs = require('fs-extra');
+const uuid = require('uuid');
+
+/* Fetch in the other needed components for the this Adaptor */
+const {
+  MongoClient, MongoClientOptions, CreateIndexesOptions, CountDocumentsOptions,
+  DeleteOptions, DeleteResult, ReplaceOptions, Document, UpdateResult, WithId, Filter,
+  UpdateFilter, FindOneAndUpdateOptions, Sort
+} = require('mongodb');
+
+// Other global variables
+let adapterDir = '';
+let saveFS = false;
+let storDir = `${adapterDir}/storage`;
+let entityDir = `${adapterDir}/entities`;
+let id = null;
+
+const Storage = {
+  UNDEFINED: 0,
+  DBINFO: 1,
+  ADAPTERDB: 2,
+  FILESYSTEM: 3,
+  IAPDB: 4
+};
+Object.freeze(Storage);
+
+/* DB UTILS INTERNAL FUNCTIONS         */
+/** getFromJson
+ * @summary returns information from a json file on the file system
+ */
+function getFromJson(fileName, filter) {
+  const origin = `${id}-dbUtil-getFromJson`;
+  log.trace(origin);
+
+  // verify the required data has been provided
+  if (fileName === undefined || fileName === null || fileName === '') {
+    log.warn(`${origin}: Must provide a file name`);
+    return null;
+  }
+
+  // get the entity and the action if this is a metric
+  let ent = null;
+  let act = null;
+  let metric = false;
+  const useFilter = filter;
+  if (useFilter && useFilter.metric) {
+    metric = true;
+    if (useFilter.metric.entity) {
+      ent = useFilter.metric.entity;
+    }
+    if (useFilter.metric.action) {
+      act = useFilter.metric.action;
+    }
+    if (!ent || !act) {
+      log.warn(`${origin}: metric provided with one of entity/action, require both`);
+      return null;
+    }
+    delete useFilter.metric;
+  }
+
+  try {
+    if (fileName === 'adapter_configs') {
+      const retEntity = filter.entity;
+      if (!retEntity || !fs.existsSync(`${entityDir}`) || !fs.existsSync(`${entityDir}/${retEntity}`)) {
+        log.warn(`${origin}: Could not find adapter entity directory ${retEntity} - nothing to retrieve`);
+        return null;
+      }
+      if (!fs.existsSync(`${entityDir}/${retEntity}/action.json`)) {
+        log.warn(`${origin}: Could not find action.json for entity ${retEntity} - nothing to retrieve`);
+        return null;
+      }
+
+      // load the mockdatafiles
+      let files = fs.readdirSync(`${entityDir}/${retEntity}`);
+      const mockdatafiles = {};
+      if (files.includes('mockdatafiles') && fs.lstatSync(`${entityDir}/${retEntity}/mockdatafiles`).isDirectory()) {
+        fs.readdirSync(`${entityDir}/${retEntity}/mockdatafiles`).forEach((file) => {
+          if (file.split('.').pop() === 'json') {
+            const mockpath = `${entityDir}/${retEntity}/mockdatafiles/${file}`;
+            let data = {};
+            try {
+              data = JSON.parse(fs.readFileSync(mockpath));
+            } catch (ex) {
+              log.warn(`could not parse mockdata file ${file}, using empty object!`);
+            }
+            mockdatafiles[mockpath.split('/').pop()] = data;
+          } else if (file.split('.').pop() === 'xml') {
+            const mockpath = `${entityDir}/${retEntity}/mockdatafiles/${file}`;
+            mockdatafiles[mockpath.split('/').pop()] = fs.readFileSync(mockpath);
+          }
+        });
+      }
+
+      // load the action data
+      let actions;
+      if (files.includes('action.json')) {
+        actions = JSON.parse(fs.readFileSync(`${entityDir}/${retEntity}/action.json`));
+      }
+
+      // Load schema.json and other schemas in remaining json files
+      files = files.filter((f) => (f !== 'action.json') && f.endsWith('.json'));
+      const schema = [];
+      files.forEach((file) => {
+        const data = JSON.parse(fs.readFileSync(`${entityDir}/${retEntity}/${file}`));
+        schema.push({
+          name: file,
+          schema: data
+        });
+      });
+
+      // return the data
+      return {
+        actions: actions.actions,
+        schema,
+        mockdatafiles
+      };
+    }
+
+    // make sure what we need exists and has been provided
+    if (!fs.existsSync(`${storDir}`)) {
+      log.warn(`${origin}: Could not find adapter storage directory - nothing to retrieve`);
+      return null;
+    }
+
+    // determine if the file exists so we retrieve the data
+    if (fs.existsSync(`${storDir}/${fileName}.json`)) {
+      const content = JSON.parse(fs.readFileSync(`${storDir}/${fileName}.json`, 'utf-8'));
+      const toReturn = [];
+
+      if (metric) {
+        // if metric need to match the entity and action
+        content.table.forEach((item) => {
+          if (ent === item.entity && act === item.action) {
+            toReturn.push(item);
+          }
+        });
+      } else {
+        // if not metric must match the items in the filter
+        // if no filter, match everything
+        let key = [];
+        if (useFilter && useFilter.length > 0) {
+          key = Object.keys(useFilter);
+        }
+        content.table.forEach((item) => {
+          let push = true;
+          // check each key - if it does not match a key do not add to return data
+          key.forEach((fil) => {
+            if (useFilter[fil] !== item[fil]) push = false;
+          });
+          if (push) toReturn.push(item);
+        });
+      }
+      // not sure why we do this - a little different than above.
+      const filtered = content.table.filter((el) => {
+        if (useFilter) {
+          Object.keys(useFilter).forEach((obj) => {
+            if (el[obj] !== useFilter[obj]) {
+              return false;
+            }
+          });
+        }
+        return true;
+      });
+      return toReturn;
+    }
+
+    // if no file, nothing to return
+    return null;
+  } catch (ex) {
+    log.warn(`${origin}: Caught Exception ${ex}`);
+    return null;
+  }
+}
+
+/** countJSON
+ * @summary returns a count from a json storage file
+ */
+function countJSON(fileName, filter) {
+  const origin = `${id}-dbUtil-countJSON`;
+  log.trace(origin);
+
+  // verify the required data has been provided
+  if (fileName === undefined || fileName === null || fileName === '') {
+    log.warn(`${origin}: Must provide a file name`);
+    return null;
+  }
+
+  try {
+    // make sure what we need exists and has been provided
+    if (!fs.existsSync(`${storDir}`)) {
+      log.warn(`${origin}: Could not find adapter storage directory - nothing to count`);
+      return null;
+    }
+
+    // determine if the file exists so we can count data from it
+    if (fs.existsSync(`${storDir}/${fileName}.json`)) {
+      const data = getFromJson(fileName, filter);
+      if (data) {
+        return data.length;
+      }
+      return -1;
+    }
+  } catch (ex) {
+    log.warn(`${origin}: Caught Exception ${ex}`);
+    return null;
+  }
+}
+
+/** saveAsJson
+ * @summary saves information into a json storage file
+ */
+function saveAsJson(fileName, data) {
+  const origin = `${id}-dbUtil-saveAsJson`;
+  log.trace(origin);
+
+  // verify the required data has been provided
+  if (fileName === undefined || fileName === null || fileName === '') {
+    log.warn(`${origin}: Must provide a file name`);
+    return null;
+  }
+
+  // get the entity and the action if this is a metric
+  let ent = null;
+  let act = null;
+  let metric = false;
+  const useData = data;
+  if (useData && useData.metric) {
+    metric = true;
+    if (useData.metric.entity) {
+      ent = useData.metric.entity;
+    }
+    if (useData.metric.action) {
+      act = useData.metric.action;
+    }
+    if (!ent || !act) {
+      log.warn(`${origin}: metric provided with one of entity/action, require both`);
+      return null;
+    }
+    delete useData.metric;
+  }
+
+  try {
+    // make sure what we need exists and has been provided
+    if (!fs.existsSync(`${storDir}`)) {
+      // need to make the storage directory if it does not exist
+      fs.mkdirSync(`${storDir}`);
+    }
+
+    // determine if the file exists so we add data to it
+    if (fs.existsSync(`${storDir}/${fileName}.json`)) {
+      // have to read, append & save
+      const content = JSON.parse(fs.readFileSync(`${storDir}/${fileName}.json`, 'utf-8'));
+      let exists = false;
+
+      content.table.forEach((item) => {
+        const toPush = item;
+        if (metric) {
+          // if it is a metric and we already have entity and action need to edit the data
+          if (ent === item.entity && act === item.action) {
+            exists = true;
+            Object.keys(useData).forEach((key) => {
+              if (key === '$inc') {
+                Object.keys(useData[key]).forEach((inc) => {
+                  if (!toPush[inc]) {
+                    toPush[inc] = useData[key][inc];
+                  }
+                  toPush[inc] += useData[key][inc];
+                });
+              } else if (key === '$set') {
+                Object.keys(useData[key]).forEach((set) => {
+                  toPush[set] = useData[key][set];
+                });
+              }
+            });
+          }
+        }
+      });
+      if (!exists) {
+        // push new thing to table.
+        const toPush = {};
+        const keysArray = Object.keys(useData);
+        keysArray.forEach((i) => {
+          const newKeys = Object.keys(useData[i]);
+          newKeys.forEach((j) => {
+            toPush[j] = useData[i][j];
+          });
+        });
+        content.table.push(toPush);
+      }
+
+      // now that is updated, write the file back out
+      fs.writeFileSync(`${storDir}/${fileName}.json`, JSON.stringify(content, null, 2));
+      return useData;
+    }
+
+    // if the file has not been created yet
+    const obj = { table: [] };
+    const toPush = {};
+    const keysArray = Object.keys(data);
+    keysArray.forEach((outer) => {
+      const newKeys = Object.keys(data[outer]);
+      newKeys.forEach((inner) => {
+        toPush[inner] = data[outer][inner];
+      });
+    });
+    obj.table.push(toPush);
+
+    // write the file out to the file system
+    fs.writeFileSync(`${storDir}/${fileName}.json`, JSON.stringify(obj, null, 2));
+    return data;
+  } catch (ex) {
+    log.warn(`${origin}: Caught Exception ${ex}`);
+    return null;
+  }
+}
+
+/** removeFromJSON
+ * @summary removes information from a json storage file
+ */
+function removeFromJSON(fileName, filter, multiple) {
+  const origin = `${id}-dbUtil-removeFromJSON`;
+  log.trace(origin);
+
+  // verify the required data has been provided
+  if (fileName === undefined || fileName === null || fileName === '') {
+    log.warn(`${origin}: Must provide a file name`);
+    return null;
+  }
+
+  // get the entity and the action if this is a metric
+  let ent = null;
+  let act = null;
+  let metric = false;
+  const useFilter = filter;
+  if (useFilter && useFilter.metric) {
+    metric = true;
+    if (useFilter.metric.entity) {
+      ent = useFilter.metric.entity;
+    }
+    if (useFilter.metric.action) {
+      act = useFilter.metric.action;
+    }
+    if (!ent && !act) {
+      log.warn(`${origin}: metric provided with one of entity/action, require both`);
+      return null;
+    }
+    delete useFilter.metric;
+  }
+
+  try {
+    // make sure what we need exists and has been provided
+    if (!fs.existsSync(`${storDir}`)) {
+      log.warn(`${origin}: Could not find adapter storage directory - nothing to remove`);
+      return null;
+    }
+
+    // determine if the file exists so we remove data from it
+    if (fs.existsSync(`${storDir}/${fileName}.json`)) {
+      const content = JSON.parse(fs.readFileSync(`${storDir}/${fileName}.json`, 'utf-8'));
+      const toReturn = [];
+
+      // if this is a metric make sure the entity and action match
+      if (metric) {
+        content.table = content.table.filter((item) => {
+          if (ent === item.entity && act === item.action) {
+            toReturn.push(item);
+            return false;
+          }
+          return true;
+        });
+      } else {
+        // go through content to determine if item matches the filter
+        let ctr = 0;
+        // create the contents that are being removed
+        const removed = content.table.filter((el) => {
+          Object.keys(useFilter).forEach((obj) => {
+            if (el[obj] !== useFilter[obj]) {
+              return false;
+            }
+          });
+          ctr += 1;
+          if (!multiple && ctr > 1) {
+            return false;
+          }
+          return true;
+        });
+        let ctr1 = 0;
+        // remove the items from the contents
+        content.table = content.table.filter((el, i) => {
+          Object.keys(useFilter).forEach((obj) => {
+            if (el[obj] !== useFilter[obj]) {
+              return true;
+            }
+          });
+          ctr1 += 1;
+          if (!multiple && ctr1 > 1) {
+            return true;
+          }
+          return false;
+        });
+
+        // write the contents back to the file
+        fs.writeFileSync(`${storDir}/${fileName}.json`, JSON.stringify(content, null, 2));
+        return removed;
+      }
+
+      // write the contents back to the file
+      fs.writeFileSync(`${storDir}/${fileName}.json`, JSON.stringify(content, null, 2));
+      return toReturn;
+    }
+
+    log.error(`${origin}: Collection ${fileName} does not exist`);
+    return null;
+  } catch (ex) {
+    log.warn(`${origin}: Caught Exception ${ex}`);
+    return null;
+  }
+}
+
+/** deleteJSON
+ * @summary deletes a json storage file
+ */
+function deleteJSON(fileName) {
+  const origin = `${id}-dbUtil-deleteJSON`;
+  log.trace(origin);
+
+  // verify the required data has been provided
+  if (fileName === undefined || fileName === null || fileName === '') {
+    log.warn(`${origin}: Must provide a file name`);
+    return null;
+  }
+
+  try {
+    // make sure what we need exists and has been provided
+    if (!fs.existsSync(`${storDir}`)) {
+      log.warn(`${origin}: Could not find adapter storage directory - nothing to delete`);
+      return null;
+    }
+
+    // determine if the file exists so we remove - also assume we add a .json suffix
+    if (fs.existsSync(`${storDir}/${fileName}.json`)) {
+      fs.remove(`${storDir}/${fileName}.json`).catch((some) => {
+        log.info(`${origin}: ${some}`);
+        fs.rmdirSync(`${storDir}/${fileName}.json`);
+      });
+    }
+
+    // successful -- return the fileName
+    return fileName;
+  } catch (ex) {
+    log.warn(`${origin}: Caught Exception ${ex}`);
+    return null;
+  }
+}
+
+class DBUtil {
+  /**
+   * These are database utilities that can be used by the adapter to interact with a
+   * mongo database
+   * @constructor
+   */
+  constructor(prongId, properties, directory) {
+    this.myid = prongId;
+    id = prongId;
+    this.baseDir = directory;
+    adapterDir = this.baseDir;
+    storDir = `${adapterDir}/storage`;
+    entityDir = `${adapterDir}/entities`;
+    this.props = properties;
+    this.adapterMongoClient = null;
+
+    // set up the properties I care about
+    this.refreshProperties(properties);
+  }
+
+  /**
+   * refreshProperties is used to set up all of the properties for the db utils.
+   * It allows properties to be changed later by simply calling refreshProperties rather
+   * than having to restart the db utils.
+   *
+   * @function refreshProperties
+   * @param {Object} properties - an object containing all of the properties
+   */
+  refreshProperties(properties) {
+    const origin = `${this.myid}-dbUtil-refreshProperties`;
+    log.trace(origin);
+    this.dburl = null;
+    this.dboptions = {};
+    this.database = this.myid;
+
+    // verify the necessary information was received
+    if (!properties) {
+      log.error(`${origin}: DB Utils received no properties!`);
+      return;
+    }
+    if (!properties.mongo || !properties.mongo.host) {
+      log.info(`${origin}: No default adapter database configured!`);
+      return;
+    }
+    this.dboptions = properties.mongo.dboptions || {};
+
+    // set the database port
+    let port = 27017;
+    if (properties.mongo.port) {
+      port = properties.mongo.port;
+    }
+    // set the database
+    if (properties.mongo.database) {
+      this.database = properties.mongo.database;
+    }
+    // set the user
+    let username = null;
+    if (properties.mongo.username) {
+      username = properties.mongo.username;
+    }
+    // set the password
+    let password = null;
+    if (properties.mongo.password) {
+      password = properties.mongo.password;
+    }
+
+    // format the database url
+    this.dburl = 'mongodb://';
+    log.info(`${origin}: Default adapter database at: ${properties.mongo.host}`);
+
+    if (username) {
+      this.dburl += `${encodeURIComponent(username)}:${encodeURIComponent(password)}@`;
+      log.info(`${origin}: Default adapter database will use authentication.`);
+    }
+    this.dburl += `${encodeURIComponent(properties.mongo.host)}:${encodeURIComponent(port)}/${encodeURIComponent(this.database)}`;
+
+    // are we using a replication set need to add it to the url
+    if (properties.mongo.replSet) {
+      this.dburl += `?${properties.mongo.replSet}`;
+      log.info(`${origin}: Default adapter database will use replica set.`);
+    }
+
+    // Do we need SSL to connect to the database
+    if (properties.mongo.db_ssl && properties.mongo.db_ssl.enabled === true) {
+      log.info(`${origin}: Default adapter database will use SSL.`);
+      this.dboptions.ssl = true;
+
+      // validate the server's certificate against a known certificate authority?
+      if (properties.mongo.db_ssl.accept_invalid_cert === false) {
+        this.dboptions.sslValidate = true;
+        log.info(`${origin}: Default adapter database will use Certificate based SSL.`);
+        // if validation is enabled, we need to read the CA file
+        if (properties.mongo.db_ssl.ca_file) {
+          try {
+            this.dboptions.sslCA = [fs.readFileSync(properties.mongo.db_ssl.ca_file)];
+          } catch (err) {
+            log.error(`${origin}: Error: unable to load Mongo CA file: ${err}`);
+          }
+        } else {
+          log.error(`${origin}: Error: Certificate validation is enabled but a CA is not specified.`);
+        }
+        if (properties.mongo.db_ssl.key_file) {
+          try {
+            this.dboptions.sslKey = [fs.readFileSync(properties.mongo.db_ssl.key_file)];
+          } catch (err) {
+            log.error(`${origin}: Error: Unable to load Mongo Key file: ${err}`);
+          }
+        }
+        if (properties.mongo.db_ssl.cert_file) {
+          try {
+            this.dboptions.sslCert = [fs.readFileSync(properties.mongo.db_ssl.cert_file)];
+          } catch (err) {
+            log.error(`${origin}: Error: Unable to load Mongo Certificate file: ${err}`);
+          }
+        }
+      } else {
+        this.dboptions.sslValidate = false;
+        log.info(`${origin}: Default adapter database not using Certificate based SSL.`);
+      }
+    } else {
+      log.info(`${origin}: Default adapter database not using SSL.`);
+    }
+
+    // if we were provided with a path to save metrics - instead of just a boolean
+    // will assume this is a place where all adapter stuff can go!
+    saveFS = this.props.save_metric || false;
+    if (saveFS && typeof saveFS === 'string' && saveFS !== '') {
+      storDir = saveFS;
+    }
+  }
+
+  /**
+   * @callback determineStorageCallback
+   * @param {string} err - the error message
+   * @param {int} storageType - the storage type
+   * @param {MongoClient} mongoClient - the mongo client
+   * @param {string} database - the database
+   */
+
+  /**
+   * @typedef DBInfo
+   * @property {string} dburl
+   * @property {MongoClientOptions} dboptions
+   * @property {string} database
+   */
+
+  /**
+   * @typedef StorageInfo
+   * @property {int} storageType
+   * @property {MongoClient} mongoClient
+   * @property {string} database
+   */
+
+  /**
+   * Call to determine the storage target. If the storage is a Mongo database, then
+   * the client connection and database is returned.
+   *
+   * @function determineStorage
+   * @param {DBInfo | null} dbInfo - the url for the database to connect to (dburl, dboptions, database). If null, tries to connect to adapter database, then the filesystem
+   * @param {determineStorageCallback | undefined} callback - the callback to invoke. If undefined, the function acts like a promise. If defined, the function returns void.
+   *
+   * @returns {Promise<StorageInfo>} - {storageType, mongoClient, database}. storageType is an enum
+   * @throws {string} - the error message
+   */
+  async determineStorage(dbInfo, callback = undefined) {
+    const origin = `${this.myid}-dbUtil-determineStorage`;
+    const useCallback = typeof callback === 'function';
+
+    const createReturnObj = (storageType, mongoClient, database) => ({
+      storageType,
+      mongoClient,
+      database
+    });
+
+    let mongoClient = null;
+    let storageType = null;
+    let database = null;
+
+    if (dbInfo) {
+      // priority 1 - use the dbInfo passed in
+      if (dbInfo.dburl && dbInfo.database) {
+        try {
+          mongoClient = await MongoClient.connect(dbInfo.dburl, dbInfo.dboptions);
+          log.debug('Using dbInfo');
+          storageType = Storage.DBINFO;
+          database = dbInfo.database;
+          if (useCallback) return callback(null, storageType, mongoClient, database);
+          return createReturnObj(storageType, mongoClient, database);
+        } catch (err) {
+          const msg = `${origin}: Error! Failed to connect to database: ${err}`;
+          log.error(msg);
+          if (useCallback) return callback(msg, null, null, null);
+          throw msg;
+        }
+      } else {
+        const msg = 'Error! Malformed dbInfo';
+        if (useCallback) return callback(msg, null, null, Storage.DBINFO); // Unknown why callback ends with Storage.DBINFO here, kept for legacy purposes
+        throw msg;
+      }
+    } else if (this.adapterMongoClient) {
+      // priority 2 - use the adapter database
+      log.debug('Using adapter db');
+      storageType = Storage.ADAPTERDB;
+      mongoClient = this.adapterMongoClient;
+      database = this.database;
+      if (useCallback) return callback(null, storageType, mongoClient, database);
+      return createReturnObj(storageType, mongoClient, database);
+    } else if (this.dburl === null && dbInfo === null) {
+      // priority 3 - use the filesystem
+      log.debug('Using filesystem');
+      storageType = Storage.FILESYSTEM;
+      if (useCallback) return callback(null, storageType, null, null);
+      return createReturnObj(storageType, mongoClient, database);
+    }
+  }
+
+  /**
+   * @callback connectCallback
+   * @param {boolean} isAlive
+   * @param {MongoClient} mongoClient
+   */
+
+  /**
+   * @typedef ConnectionInfo
+   * @property {boolean} isAlive
+   * @property {MongoClient} mongoClient
+   */
+
+  /**
+   * Call to connect and authenticate to the adapter database
+   *
+   * @function connect
+   * @param {connectCallback | undefined} callback - the callback to invoke. If undefined, the function acts like a promise. If defined, the function returns void.
+   *
+   * @returns {Promise<ConnectionInfo>}
+   */
+  async connect(callback = undefined) {
+    const origin = `${this.myid}-dbUtil-connect`;
+    const useCallback = typeof callback === 'function';
+    log.trace(origin);
+
+    let mongoClient = null;
+
+    // const options = (replSetEnabled === true) ? { replSet: opts } : { server: opts };
+    log.debug(`${origin}: Connecting to MongoDB with options ${JSON.stringify(this.dboptions)}`);
+
+    try {
+      // Now we will start the process of connecting to mongo db
+      mongoClient = await MongoClient.connect(this.dburl, this.dboptions);
+      mongoClient.on('close', () => {
+        this.alive = false;
+        log.error(`${origin}: MONGO CONNECTION LOST...`);
+      });
+
+      mongoClient.on('reconnect', async () => {
+        // we still need to check if we are properly authenticated
+        // so we just list collections to test it.
+        try {
+          await this.clientDB.listCollections().toArray();
+          log.info(`${origin}: MONGO CONNECTION BACK...`);
+          this.alive = true;
+          this.adapterMongoClient = mongoClient;
+        } catch (error) {
+          log.error(`${origin}: ${error}`);
+          this.alive = false;
+        }
+      });
+
+      log.info(`${origin}: mongo running @${this.dburl}/${this.database}`);
+      this.clientDB = mongoClient.db(this.database);
+      this.adapterMongoClient = mongoClient;
+
+      // we don't have authentication defined but we still need to check if Mongo does not
+      // require one, so we just list collections to test if it's doable.
+      try {
+        await this.clientDB.listCollections().toArray();
+        log.info(`${origin}: MongoDB connection has been established`);
+        this.alive = true;
+        this.adapterMongoClient = mongoClient;
+      } catch (error) {
+        log.error(`${origin}: ${error}`);
+        this.alive = false;
+      }
+      if (useCallback) return callback(this.alive, mongoClient);
+      return { isAlive: this.alive, mongoClient };
+    } catch (err) {
+      log.error(`${origin}: Error! Exiting... Must start MongoDB first ${err}`);
+      this.alive = false;
+      if (useCallback) return callback(this.alive, mongoClient);
+      return { isAlive: this.alive, mongoClient };
+    }
+  }
+
+  /**
+   * Call to disconnect from the adapter database
+   *
+   * @function disconnect
+   */
+  async disconnect() {
+    if (this.adapterMongoClient) {
+      await this.adapterMongoClient.close();
+    }
+  }
+
+  /**
+   * @callback collectionCallback
+   * @param {string} err - the error message
+   * @param {string} collectionName
+   */
+  /**
+   * createCollection creates the provided collection in the file system or database.
+   *
+   * @function createCollection
+   * @param {string} collectionName - the name of the collection to create
+   * @param {DBInfo} dbInfo - the url for the database to connect to (dburl, dboptions, database)
+   * @param {boolean} fsWrite - turn on write to the file system
+   * @param {collectionCallback | undefined} callback - the callback to invoke. If undefined, the function acts like a promise. If defined, the function returns void.
+   *
+   * @returns {Promise<string>} - the name of the collection
+   * @throws {string} - the error message
+   */
+  async createCollection(collectionName, dbInfo, fsWrite, callback = undefined) {
+    const origin = `${this.myid}-dbUtil-createCollection`;
+    const useCallback = typeof callback === 'function';
+    log.trace(origin);
+
+    // verify the required data has been provided
+    if (!collectionName || (typeof collectionName !== 'string')) {
+      const msg = `${origin}: Missing Collection Name or not string`;
+      log.warn(msg);
+      if (useCallback) return callback(msg, null);
+      throw msg;
+    }
+
+    let res;
+    try {
+      res = await this.determineStorage(dbInfo);
+    } catch (err) {
+      const msg = `${origin}: ${err}`;
+      if (useCallback) return callback(msg, null);
+      throw msg;
+    }
+
+    // if using file storage
+    if (res.storageType === Storage.FILESYSTEM) {
+      // if there is no adapter directory - can not do anything so error
+      if (!fs.existsSync(`${this.baseDir}`)) {
+        const msg = `${origin}: Not able to create storage - missing base directory!`;
+        log.warn(msg);
+        if (useCallback) return callback(msg, null);
+        throw msg;
+      }
+      // if there is no storage directory - create it
+      if (!fs.existsSync(`${this.baseDir}/storage`)) {
+        fs.mkdirSync(`${this.baseDir}/storage`);
+      }
+      // if the collection already exists - no need to create it
+      if (fs.existsSync(`${this.baseDir}/storage/${collectionName}`)) {
+        log.debug(`${origin}: storage file collection already exists`);
+        return collectionName;
+      }
+      // create the new collection on the file system
+      fs.mkdirSync(`${this.baseDir}/storage/${collectionName}`);
+      log.debug(`${origin}: storage file collection ${collectionName} created`);
+      return collectionName;
+    }
+
+    // if using MongoDB storage
+    if (res.storageType === Storage.DBINFO || res.storageType === Storage.ADAPTERDB) {
+      let collection;
+      try {
+        collection = await res.mongoClient.db(res.database).createCollection(collectionName);
+      } catch (err) {
+        // error we get back if the collection already existed - not a true error
+        if (err.codeName === 'NamespaceExists') {
+          log.debug(`${origin}: database collection already exists`);
+          if (useCallback) return callback(null, collectionName);
+          return collectionName;
+        }
+        const msg = `${origin}: Error creating collection ${err}`;
+        log.warn(msg);
+        if (useCallback) return callback(msg, null);
+        throw msg;
+      }
+
+      log.spam(`${origin}: db response ${res}`);
+      log.debug(`${origin}: database collection ${collectionName} created`);
+      if (useCallback) return callback(null, collectionName);
+      return collectionName;
+    }
+
+    log.warn(`${origin}: Unexpected end of function reached`);
+  }
+
+  /**
+   * removeCollection removes the provided collection from the file system or database.
+   *
+   * @function removeCollection
+   * @param {string} collectionName - the name of the collection to remove
+   * @param {DBInfo} dbInfo - the url for the database to connect to (dburl, dboptions, database)
+   * @param {boolean} fsWrite - turn on write to the file system
+   * @param {collectionCallback | undefined} callback - the callback to invoke. If undefined, the function acts like a promise. If defined, the function returns void.
+   *
+   * @returns {Promise<string>} - the name of the collection
+   * @throws {string} - the error message
+   */
+  async removeCollection(collectionName, dbInfo, fsWrite, callback = undefined) {
+    const origin = `${this.myid}-dbUtil-removeCollection`;
+    const useCallback = typeof callback === 'function';
+    log.trace(origin);
+
+    // verify the required data has been provided
+    if (!collectionName || (typeof collectionName !== 'string')) {
+      const msg = `${origin}: Missing Collection Name or not string`;
+      log.warn(msg);
+      if (useCallback) return callback(msg, null);
+      throw msg;
+    }
+
+    let res;
+    try {
+      res = await this.determineStorage(dbInfo);
+    } catch (err) {
+      const msg = `${origin}: ${err}`;
+      if (useCallback) return callback(msg, null);
+      throw msg;
+    }
+
+    // if using file storage
+    if (res.storageType === Storage.FILESYSTEM) {
+      const deld = deleteJSON(collectionName);
+      if (deld) {
+        log.debug(`${origin}: storage file collection ${collectionName} removed`);
+        if (useCallback) return callback(null, deld);
+        return deld;
+      }
+      const msg = `${origin}: could not remove storage file collection`;
+      log.debug(msg);
+      if (useCallback) return callback(msg, null);
+      throw msg;
+    }
+
+    // if using MongoDB storage
+    if (res.storageType === Storage.DBINFO || res.storageType === Storage.ADAPTERDB) {
+      // get the list of collections from the database - can we just drop?
+      let collections;
+      try {
+        collections = await res.mongoClient.db(res.database).listCollections().toArray();
+      } catch (err) {
+        const msg = `${origin}: Failed to get collections ${err}`;
+        log.warn(msg);
+        if (useCallback) return callback(msg, null);
+        throw msg;
+      }
+
+      // go through the collections to get the correct one for removal
+      const collectionToRemove = collections.find((collection) => collection.name === collectionName);
+      // If it doesn't exist, drop is implicit and successful
+      if (!collectionToRemove) {
+        if (useCallback) return callback(null, collectionName);
+        return collectionName;
+      }
+
+      // now that we found it, remove it
+      try {
+        const dropRes = await res.mongoClient.db(res.database).collection(collectionName).drop({});
+        log.spam(`${origin}: db response ${dropRes}`);
+        log.debug(`${origin}: database collection ${collectionName} removed`);
+        if (useCallback) return callback(null, collectionName);
+        return collectionName;
+      } catch (err) {
+        const msg = `${origin}: Failed to remove collection ${err}`;
+        if (useCallback) return callback(msg, null);
+        throw msg;
+      }
+    }
+
+    log.warn(`${origin}: Unexpected end of function reached`);
+  }
+
+  /**
+   * @callback createCallback
+   * @param {string} err - the error message
+   * @param {string} data - the modification made
+   */
+  /**
+   * Call to create an item in the database
+   *
+   * @function create
+   * @param {string} collectionName - the collection to save the item in. (required)
+   * @param {string} data - the modification to make. (required)
+   * @param {DBInfo} dbInfo - the url for the database to connect to (dburl, dboptions, database)
+   * @param {boolean} fsWrite - turn on write to the file system
+   * @param {createCallback | undefined} callback - the callback to invoke. If undefined, the function acts like a promise. If defined, the function returns void.
+   *
+   * @returns {Promise<string>} - the modification made
+   * @throws {string} - the error message
+   */
+  async create(collectionName, data, dbInfo, fsWrite, callback = undefined) {
+    const origin = `${this.myid}-dbUtil-create`;
+    const useCallback = typeof callback === 'function';
+    log.trace(origin);
+
+    // verify the required data has been provided
+    if (!collectionName) {
+      const msg = `${origin}: Missing Collection Name`;
+      log.warn(msg);
+      if (useCallback) return callback(msg, null);
+      throw msg;
+    } else if (!data) {
+      const msg = `${origin}: Missing data to add`;
+      log.warn(msg);
+      if (useCallback) return callback(msg, null);
+      throw msg;
+    } else if ((data.entity && !data.action) || (!data.entity && data.action)) {
+      const msg = `${origin}: Inconsistent entity/action set`;
+      log.warn(msg);
+      if (useCallback) return callback(msg, null);
+      throw msg;
+    }
+
+    // create the unique identifier (should we let mongo do this?)
+    const dataInfo = data;
+    if (!{}.hasOwnProperty.call(dataInfo, '_id')) {
+      dataInfo._id = uuid.v4();
+    }
+
+    let res;
+    try {
+      res = await this.determineStorage(dbInfo);
+    } catch (err) {
+      const msg = `${origin}: ${err}`;
+      if (useCallback) return callback(msg, null);
+      throw msg;
+    }
+
+    // if using file storage
+    if (res.storageType === Storage.FILESYSTEM) {
+      // save it to file in the adapter storage directory
+      const saved = saveAsJson(collectionName, data);
+      if (!saved) {
+        const msg = `${origin}: Data has not been saved to file storage`;
+        log.warn(msg);
+        if (useCallback) return callback(msg, null);
+        throw msg;
+      }
+      log.debug(`${origin}: Data saved in file storage`);
+      if (useCallback) return callback(null, saved);
+      return saved;
+    }
+
+    // if using MongoDB storage
+    if (res.storageType === Storage.DBINFO || res.storageType === Storage.ADAPTERDB) {
+      // Add the data to the database
+      // insertOne has only 2 parameters: the data to be added & callback. Not an identifier.
+      try {
+        const result = await res.mongoClient.db(res.database).collection(collectionName).insertOne(dataInfo);
+        if (res.storageType === Storage.DBINFO && res.mongoClient) res.mongoClient.close();
+        log.spam(`${origin}: db response ${result}`);
+        log.debug(`${origin}: Data saved in database`);
+        if (useCallback) return callback(null, data);
+        return data;
+      } catch (err) {
+        if (res.storageType === Storage.DBINFO && res.mongoClient) res.mongoClient.close();
+        const msg = `${origin}: Failed to insert data in collection ${err}`;
+        log.warn(msg);
+        if (useCallback) return callback(msg, null);
+        throw msg;
+      }
+    }
+
+    log.warn(`${origin}: Unexpected end of function reached`);
+  }
+
+  /**
+   * Call to create an index in the database
+   *
+   * @function createIndex
+   * @param {string} collectionName - the collection to index. (required)
+   * @param {string} fieldOrSpec - what to index. (required)
+   * @param {CreateIndexesOptions | undefined} options - the creation options
+   * @param {DBInfo} dbInfo - the url for the database to connect to (dburl, dboptions, database)
+   * @param {createCallback | undefined} callback - the callback to invoke. If undefined, the function acts like a promise. If defined, the function returns void.
+   *
+   * @returns {Promise<string>} - the created index
+   * @throws {string} - the error message
+   */
+  async createIndex(collectionName, fieldOrSpec, options, dbInfo, callback = undefined) {
+    const origin = `${this.myid}-dbUtil-createIndex`;
+    const useCallback = typeof callback === 'function';
+    log.trace(origin);
+
+    // verify the required data has been provided
+    if (!collectionName) {
+      const msg = `${origin}: Missing Collection Name`;
+      log.warn(msg);
+      if (useCallback) return callback(msg, null);
+      throw msg;
+    }
+    if (!fieldOrSpec) {
+      const msg = `${origin}: Missing Specs`;
+      log.warn(msg);
+      if (useCallback) return callback(msg, null);
+      throw msg;
+    }
+
+    let res;
+    try {
+      res = await this.determineStorage(dbInfo);
+    } catch (err) {
+      const msg = `${origin}: ${err}`;
+      if (useCallback) callback(msg, null);
+      throw msg;
+    }
+
+    // if using file storage
+    if (res.storageType === Storage.FILESYSTEM) {
+      // no database - no index
+      const msg = `${origin}: No database - no index`;
+      log.warn(msg);
+      if (useCallback) return callback(msg, null);
+      throw msg;
+    }
+
+    // if using MongoDB storage
+    if (res.storageType === Storage.DBINFO || res.storageType === Storage.ADAPTERDB) {
+      // create the index on the collection
+      let indexRes;
+      try {
+        indexRes = await res.mongoClient.db(res.database).collection(collectionName).createIndex(fieldOrSpec, options || {});
+        if (res.storageType === Storage.DBINFO && res.mongoClient) res.mongoClient.close();
+        log.debug(`${origin}: Data in collection indexed`);
+        if (useCallback) return callback(null, indexRes);
+        return indexRes;
+      } catch (err) {
+        if (res.storageType === Storage.DBINFO && res.mongoClient) res.mongoClient.close();
+        const msg = `${origin}: Failed to index data in collection ${err}`;
+        log.warn(msg);
+        if (useCallback) return callback(msg, null);
+        throw msg;
+      }
+    }
+
+    log.warn(`${origin}: Unexpected end of function reached`);
+  }
+
+  /**
+   * @callback countCallback
+   * @param {string} err - the error message
+   * @param {int} data - the count
+   */
+  /**
+   * Call to count the documents in a collection
+   *
+   * @function countDocuments
+   * @param {string} collectionName - the collection to count documents in. (required)
+   * @param {object} query - the query to minimize documents you count. (required)
+   * @param {CountDocumentsOptions | undefined} options
+   * @param {DBInfo} dbInfo - the url for the database to connect to (dburl, dboptions, database)
+   * @param {boolean} fsWrite - turn on write to the file system
+   * @param {countCallback | undefined} callback - the callback to invoke. If undefined, the function acts like a promise. If defined, the function returns void.
+   *
+   * @returns {Promise<number>} - the count of the documents
+   * @throws {string} - the error message
+   */
+  async countDocuments(collectionName, query, options, dbInfo, fsWrite, callback = undefined) {
+    const origin = `${this.myid}-dbUtil-countDocuments`;
+    const useCallback = typeof callback === 'function';
+    log.trace(origin);
+
+    // verify the required data has been provided
+    if (!collectionName) {
+      const msg = `${origin}: Missing Collection Name`;
+      log.warn(msg);
+      if (useCallback) return callback(msg, null);
+      throw msg;
+    }
+
+    let res;
+    try {
+      res = await this.determineStorage(dbInfo);
+    } catch (err) {
+      const msg = `${origin}: ${err}`;
+      if (useCallback) callback(msg, null);
+      throw msg;
+    }
+
+    // if using file storage
+    if (res.storageType === Storage.FILESYSTEM) {
+      // get a count from the JSON
+      const data = countJSON(collectionName, query);
+      if (!data || data === -1) {
+        const msg = `${origin}: Could not count data from file storage`;
+        log.warn(msg);
+        if (useCallback) return callback(msg, null);
+        throw msg;
+      }
+      log.debug(`${origin}: Count from file storage ${data}`);
+      if (useCallback) return callback(null, data);
+      return data;
+    }
+
+    // if using MongoDB storage
+    if (res.storageType === Storage.DBINFO || res.storageType === Storage.ADAPTERDB) {
+      // get the count from mongo
+      let count;
+      try {
+        count = await res.mongoClient.db(res.database).collection(collectionName).countDocuments(query, options || {});
+        if (res.storageType === Storage.DBINFO && res.mongoClient) res.mongoClient.close();
+        log.debug(`${origin}: Count from database ${res}`);
+        if (useCallback) return callback(null, count);
+        return count;
+      } catch (err) {
+        if (res.storageType === Storage.DBINFO && res.mongoClient) res.mongoClient.close();
+        const msg = `${origin}: Failed to count collection ${err}`;
+        log.warn(msg);
+        if (useCallback) return callback(msg, null);
+        throw msg;
+      }
+    }
+
+    log.warn(`${origin}: Unexpected end of function reached`);
+  }
+
+  /**
+   * Delete items from a collection
+   *
+   * @function delete
+   * @param {string} collectionName - the collection to remove document from. (required)
+   * @param {object} filter - the filter for the document(s) to remove. (required)
+   * @param {DeleteOptions} options
+   * @param {boolean} multiple
+   * @param {DBInfo} dbInfo - the url for the database to connect to (dburl, dboptions, database)
+   * @param {boolean} fsWrite - turn on write to the file system
+   * @param {createCallback | undefined} callback - the callback to invoke. If undefined, the function acts like a promise. If defined, the function returns void.
+   *
+   * @returns {Promise<DeleteResult>}
+   * @throws {string} - the error message
+   */
+  async delete(collectionName, filter, options, multiple, dbInfo, fsWrite, callback = undefined) {
+    const origin = `${this.myid}-dbUtil-delete`;
+    const useCallback = typeof callback === 'function';
+    log.trace(origin);
+
+    // verify the required data has been provided
+    if (!collectionName) {
+      const msg = `${origin}: Missing Collection Name`;
+      log.warn(msg);
+      if (useCallback) return callback(msg, null);
+      throw msg;
+    }
+    if (!filter) {
+      const msg = `${origin}: Missing Filter`;
+      log.warn(msg);
+      if (useCallback) return callback(msg, null);
+      throw msg;
+    }
+    if (multiple === undefined || multiple === null) {
+      const msg = `${origin}: Missing Multiple flag`;
+      log.warn(msg);
+      if (useCallback) return callback(msg, null);
+      throw msg;
+    }
+
+    let res;
+    try {
+      res = await this.determineStorage(dbInfo);
+    } catch (err) {
+      const msg = `${origin}: ${err}`;
+      if (useCallback) callback(msg, null);
+      throw msg;
+    }
+
+    // if using file storage
+    if (res.storageType === Storage.FILESYSTEM) {
+      // verify the collection exists
+      if (!fs.existsSync(`${adapterDir}/storage/${collectionName}.json`)) {
+        const msg = `${origin}: Collection ${collectionName} does not exist`;
+        log.warn(msg);
+        if (useCallback) return callback(msg, null);
+        throw msg;
+      }
+      // remove the item from the collection
+      const deld = removeFromJSON(collectionName, filter, multiple);
+      if (!deld) {
+        const msg = `${origin}: Data has not been deleted from file storage`;
+        log.warn(msg);
+        if (useCallback) return callback(msg, null);
+        throw msg;
+      }
+      log.debug(`${origin}: Data has been deleted from file storage`);
+      if (useCallback) return callback(null, deld);
+      return deld;
+    }
+
+    // if using MongoDB storage
+    if (res.storageType === Storage.DBINFO || res.storageType === Storage.ADAPTERDB) {
+      // Calls either 'collection.deleteOne' or 'collection.deleteMany'. Both methods take the same parameters
+      const methodName = multiple ? 'deleteMany' : 'deleteOne';
+      let delRes;
+      try {
+        delRes = await res.mongoClient.db(res.database).collection(collectionName)[methodName](filter, options);
+        if (res.storageType === Storage.DBINFO && res.mongoClient) res.mongoClient.close();
+        log.debug(`${origin}: Data has been deleted from database`);
+        if (useCallback) return callback(null, delRes);
+        return delRes;
+      } catch (err) {
+        if (res.storageType === Storage.DBINFO && res.mongoClient) res.mongoClient.close();
+        const msg = `${origin}: Failed to delete data from database ${err}`;
+        log.warn(msg);
+        if (useCallback) return callback(msg, null);
+        throw msg;
+      }
+    }
+
+    log.warn(`${origin}: Unexpected end of function reached`);
+  }
+
+  /**
+   * @callback replaceCallback
+   * @param {string} err - the error message
+   * @param {Document | UpdateResult<Document>} data - the update result
+   */
+
+  /**
+   * Replace an item in a collection
+   *
+   * @function replaceOne
+   * @param {string} collectionName - the collection to replace document in. (required)
+   * @param {object} filter - the filter for the document to replace. (required)
+   * @param {object} doc - the document to insert. (required)
+   * @param {ReplaceOptions} options
+   * @param {DBInfo} dbInfo - the url for the database to connect to (dburl, dboptions, database)
+   * @param {boolean} fsWrite - turn on write to the file system
+   * @param {replaceCallback | undefined} callback - the callback to invoke. If undefined, the function acts like a promise. If defined, the function returns void.
+   *
+   * @returns {Promise<Document | UpdateResult<Document>>}
+   * @throws {string} - the error message
+   */
+  async replaceOne(collectionName, filter, doc, options, dbInfo, fsWrite, callback = undefined) {
+    const origin = `${this.myid}-dbUtil-replaceOne`;
+    const useCallback = typeof callback === 'function';
+    log.trace(origin);
+
+    // verify the required data has been provided
+    if (!collectionName) {
+      const msg = `${origin}: Missing Collection Name`;
+      log.warn(msg);
+      if (useCallback) return callback(msg, null);
+      throw msg;
+    }
+    if (!filter) {
+      const msg = `${origin}: Missing Filter`;
+      log.warn(msg);
+      if (useCallback) return callback(msg, null);
+      throw msg;
+    }
+    if (!doc) {
+      const msg = `${origin}: Missing Document`;
+      log.warn(msg);
+      if (useCallback) return callback(msg, null);
+      throw msg;
+    }
+
+    let res;
+    try {
+      res = await this.determineStorage(dbInfo);
+    } catch (err) {
+      const msg = `${origin}: ${err}`;
+      if (useCallback) callback(msg, null);
+      throw msg;
+    }
+
+    // if using file storage
+    if (res.storageType === Storage.FILESYSTEM) {
+      // remove the data from the collection
+      const rem = removeFromJSON(collectionName, filter, false);
+      if (rem) {
+        // add the data into the collection
+        const sav = saveAsJson(collectionName, doc);
+        if (sav) {
+          log.debug(`${origin}: Data replaced in file storage`);
+          if (useCallback) return callback(null, sav);
+          return sav;
+        }
+        const msg = `${origin}: Could not save doc into file storage`;
+        log.warn(msg);
+        if (useCallback) return callback(msg, null);
+        throw msg;
+      }
+      const msg = `${origin}: Could not delete from file storage`;
+      log.warn(msg);
+      if (useCallback) return callback(msg, null);
+      throw msg;
+    }
+
+    // if using MongoDB storage
+    if (res.storageType === Storage.DBINFO || res.storageType === Storage.ADAPTERDB) {
+      // replace an items in mongo
+      let replaceRes;
+      try {
+        replaceRes = await res.mongoClient.db(res.database).collection(collectionName).replaceOne(filter, doc, options);
+        if (res.storageType === Storage.DBINFO && res.mongoClient) res.mongoClient.close();
+        log.debug(`${origin}: Data replaced in file storage`);
+        if (useCallback) return callback(null, replaceRes);
+        return replaceRes;
+      } catch (err) {
+        if (res.storageType === Storage.DBINFO && res.mongoClient) res.mongoClient.close();
+        const msg = `${origin}: Failed to replace data in database ${err}`;
+        log.warn(msg);
+        if (useCallback) return callback(msg, null);
+        throw msg;
+      }
+    }
+
+    log.warn(`${origin}: Unexpected end of function reached`);
+  }
+
+  /**
+   * @callback findCallback
+   * @param {string} err - the error message
+   * @param {Document[]} data - the found data
+   */
+
+  /**
+   * @typedef FindOptsParam
+   * @property {Filter<Document>} filter
+   * @property {Sort} sort
+   * @property {number} start - start position
+   * @property {number} limit - max number of results
+   */
+
+  /**
+   * Call to find items in the database
+   *
+   * @function find
+   * @param {string} collectionName - the collection name to search in. (required)
+   * @param {FindOptsParam} options - the options to use to find data.
+   * @param {DBInfo} dbInfo - the url for the database to connect to (dburl, dboptions, database)
+   * @param {boolean} fsWrite - turn on write to the file system
+   * @param {findCallback | undefined} callback - the callback to invoke. If undefined, the function acts like a promise. If defined, the function returns void.
+   *
+   * @returns {Promise<Document[]>}
+   * @throws {string} - the error message
+   */
+  async find(collectionName, options, dbInfo, fsWrite, callback = undefined) {
+    const origin = `${this.myid}-dbUtil-find`;
+    const useCallback = typeof callback === 'function';
+    log.trace(origin);
+
+    // verify the required data has been provided
+    if (!collectionName) {
+      const msg = `${origin}: Missing Collection Name`;
+      log.warn(msg);
+      if (useCallback) return callback(msg, null);
+      throw msg;
+    }
+
+    // get the collection so we can run the remove on the collection
+    let filter = {};
+    let start = 0;
+    let sort = {};
+    let limit = 10;
+    if (options) {
+      filter = options.filter || {};
+      start = options.start || 0;
+      sort = options.sort || {};
+      if (Object.hasOwnProperty.call(options, 'limit')) {
+        ({ limit } = options);
+      }
+    }
+
+    // If limit is not specified, default to 10.
+    // Note: limit may be 0, which is equivalent to setting no limit.
+
+    // Replace filter with regex to allow for substring lookup
+    // TODO: Need to create a new filter object instead of mutating the exsisting one
+    const filterKeys = Object.keys(filter).filter((key) => (key[0] !== '$' && typeof filter[key] === 'string'));
+    filterKeys.map((key) => {
+      try {
+        const escapedFilter = filter[key].replace(/([.?*+^$[\]\\(){}|-])/g, '\\$1');
+        const regexedFilter = new RegExp(`.*${escapedFilter}.*`, 'i');
+        filter[key] = {
+          $regex: regexedFilter
+        };
+      } catch (e) {
+        delete filter[key];
+      }
+      return key;
+    });
+
+    let res;
+    try {
+      res = await this.determineStorage(dbInfo);
+    } catch (err) {
+      const msg = `${origin}: ${err}`;
+      if (useCallback) callback(msg, null);
+      throw msg;
+    }
+
+    // if using file storage
+    if (res.storageType === Storage.FILESYSTEM) {
+      // Find it from file in the adapter
+      let toReturn = getFromJson(collectionName, options);
+      if (collectionName === 'adapter_configs') {
+        log.debug(`${origin}: Data retrieved from file storage`);
+        if (useCallback) return callback(null, [toReturn]);
+        return [toReturn];
+      }
+      if (toReturn && toReturn.length > limit) {
+        let curEnd = start + limit;
+        if (curEnd < toReturn.length) {
+          curEnd = toReturn.length;
+        }
+        toReturn = toReturn.slice(start, curEnd);
+      }
+      log.debug(`${origin}: Data retrieved from file storage`);
+      if (useCallback) return callback(null, toReturn);
+      return toReturn;
+    }
+
+    // if using MongoDB storage
+    if (res.storageType === Storage.DBINFO || res.storageType === Storage.ADAPTERDB) {
+      // Find the data in the database
+      let findRes;
+      try {
+        findRes = await res.mongoClient.db(res.database).collection(collectionName).find(filter)
+          .sort(sort)
+          .skip(start)
+          .limit(limit)
+          .toArray();
+        log.debug(`${origin}: Data retrieved from database`);
+        if (useCallback) return callback(null, findRes);
+        return findRes;
+      } catch (err) {
+        const msg = `${origin}: Data could not be retrieved from database`;
+        log.warn(msg);
+        if (useCallback) return callback(msg, null);
+        throw msg;
+      } finally {
+        if (res.storageType === Storage.DBINFO && res.mongoClient) res.mongoClient.close();
+      }
+    }
+
+    log.warn(`${origin}: Unexpected end of function reached`);
+  }
+
+  /**
+   * @callback findAndModifyCallback
+   * @param {string} err - the error message
+   * @param {WithId<Document>} data - the response
+   */
+
+  /**
+   * Call to find an item in a collection and modify it.
+   *
+   * @function findAndModify
+   * @param {string} collectionName - the collection to find things from. (required)
+   * @param {Filter<Document>} filter - the filter used to find objects. (optional)
+   * @param {Array} sort - how to sort the items (first one in order will be modified). (optional)
+   * @param {UpdateFilter<Document>} data - the modification to make. (required)
+   * @param {boolean} upsert - option for the whether to insert new objects. (optional)
+   * @param {DBInfo} dbInfo - the url for the database to connect to (dburl, dboptions, database) (optional)
+   * @param {boolean} fsWrite - turn on write to the file system (optional)
+   * @param {findAndModifyCallback | undefined} callback - the callback to invoke. If undefined, the function acts like a promise. If defined, the function returns void.
+   *
+   * @return {Promise<WithId<Document>>}
+   * @throws {string} - the error message
+   */
+  async findAndModify(collectionName, filter, sort, data, upsert, dbInfo, fsWrite, callback = undefined) {
+    const origin = `${this.myid}-dbUtil-findAndModify`;
+    const useCallback = typeof callback === 'function';
+    log.trace(origin);
+
+    // verify the required data has been provided
+    if (collectionName === undefined || collectionName === null || collectionName === '') {
+      const msg = `${origin}: Missing Collection Name`;
+      log.warn(msg);
+      if (useCallback) return callback(msg, null);
+      throw msg;
+    }
+    if (data === undefined || data === null || typeof data !== 'object' || Object.keys(data).length === 0) {
+      const msg = `${origin}: Missing data for modification`;
+      log.warn(msg);
+      if (useCallback) return callback(msg, null);
+      throw msg;
+    }
+
+    const options = {
+      sort,
+      upsert,
+      returnOriginal: false
+    };
+
+    let res;
+    try {
+      res = await this.determineStorage(dbInfo);
+    } catch (err) {
+      const msg = `${origin}: ${err}`;
+      if (useCallback) callback(msg, null);
+      throw msg;
+    }
+
+    // if using file storage
+    if (res.storageType === Storage.FILESYSTEM) {
+      // save it to file in the adapter storage directory
+      const saved = saveAsJson(collectionName, data);
+      if (!saved) {
+        const msg = `${origin}: Data has not been saved`;
+        log.error(msg);
+        if (useCallback) return callback(msg, null);
+        throw msg;
+      }
+      log.debug(`${origin}: Data modified in file storage`);
+      if (useCallback) return callback(null, saved);
+      return saved;
+    }
+
+    // if using MongoDB storage
+    if (res.storageType === Storage.DBINFO || res.storageType === Storage.ADAPTERDB) {
+      // find and modify the data in the database
+      let findRes;
+      try {
+        findRes = await res.mongoClient.db(res.database).collection(collectionName).findOneAndUpdate((filter || {}), data, options);
+        if (findRes) log.debug(`${origin}: Data modified in database`);
+        if (useCallback) return callback(null, findRes);
+        return findRes;
+      } catch (err) {
+        const msg = `${origin}: Failed to modified data in database ${err}`;
+        log.warn(msg);
+        if (useCallback) return callback(msg, null);
+        throw msg;
+      } finally {
+        if (res.storageType === Storage.DBINFO && res.mongoClient) res.mongoClient.close();
+      }
+    }
+
+    log.warn(`${origin}: Unexpected end of function reached`);
+  }
+}
+
+module.exports = DBUtil;