@itentialopensource/adapter-utils 4.49.0 → 5.0.0

package/lib/cacheHandler.js

@@ -0,0 +1,968 @@
+/* @copyright Itential, LLC 2023 */
+
+// Set globals
+/* global log */
+/* eslint global-require:warn */
+/* eslint import/no-dynamic-require: warn */
+
+const AsyncLockCl = require('async-lock');
+const path = require('path');
+const jsonQuery = require('json-query');
+
+const lock = new AsyncLockCl();
+let id = null;
+
+/**
+ * @summary Creates a Cache Entity object to organize the cache by entity type.
+ * @function createCacheEntity
+ * @param {String} entityName - type of entity
+ * @param {Array} entityList - array of data
+ * @param {Object} interval - id of interval
+ * @returns {Object} - created Cache Entity object
+ */
+function createCacheEntity(entityName, entityList, interval, sortEntities = true) {
+  const origin = `${id}-cacheHandler-createCacheEntity`;
+  log.trace(origin);
+  return ({
+    lockKey: entityName,
+    entityType: entityName,
+    list: entityList,
+    intervalId: interval,
+    sort: sortEntities
+  });
+}
+
+/**
+ * @summary Generates name for data object. Currently non-unique
+ * @function generateName
+ * @returns {String} - generated name
+ */
+function generateName() {
+  log.warn('Name for entity not found, generated name placeholder');
+  return 'GeneratedName';
+}
+
+/**
+ * @summary Deletes cache data and properties. Data deletion
+ * saves considerable memory.
+ * @function deleteCacheData
+ * @param {Object} cacheHandler - this class instance
+ */
+function deleteCacheData(cacheHandler) {
+  const origin = `${id}-cacheHandler-deleteCacheData`;
+  log.trace(origin);
+
+  const handler = cacheHandler;
+
+  // get all keys to make sure nothing else is editing the cache at this time
+  const keysArr = [];
+  handler.cache.forEach((entity) => {
+    keysArr.push(entity.lockKey);
+  });
+
+  lock.acquire(keysArr, () => {
+    if (!handler.enabled) {
+      handler.cache.forEach((entity) => {
+        clearInterval(entity.intervalId);
+      });
+      handler.cache = [];
+      handler.propertiesMap.clear();
+      log.warn(`${origin}: Cache deletion confirmed`);
+    }
+  });
+}
+
+/**
+ * @summary changes the frequency of which to update an entity type
+ * @function changeUpdateFrequency
+ * @param {Array of Objects} currentCache - cache
+ * @param {String} entityType - entity type of which to update
+ * @param {Integer} newFrequency - how often to updaate (in minutes)
+ * @param {cacheHandler} cacheHandler - instance of class to edit
+ */
+function changeUpdateFrequency(currentCache, entityType, newFrequency, cacheHandler) {
+  const origin = `${id}-cacheHandler-changeUpdateFrequency`;
+  log.trace(origin);
+
+  const cache = currentCache;
+  for (let i = 0; i < cache.length; i += 1) {
+    if (cache[i].entityType === entityType) {
+      cacheHandler.populateCache(entityType);
+      clearInterval(cache[i].intervalId);
+      cache[i].intervalId = setInterval(() => {
+        cacheHandler.populateCache(entityType);
+      }, newFrequency * 60000);
+      log.info(`Changed update frequency for ${entityType}.`);
+      return;
+    }
+  }
+  log.error(`Failed to change interval. ${entityType} does not exist in cache.`);
+}
+
+/**
+ * @summary Given an array of populate objects from a single entityType, initialize missing fields.
+ * @function validatePopulate
+ * @param {Array} populateArr - array of populate objects from properties
+ */
+function validatePopulate(populateArr) {
+  const origin = `${id}-cacheHandler-validatePopulate`;
+  log.trace(origin);
+
+  const arr = populateArr;
+  for (let i = 0; i < arr.length; i += 1) {
+    if (!arr[i].path) {
+      log.error(`No path in populate in properties for cache! Path is ${path}`);
+      // let brokerHandler handle this
+    } else {
+      arr[i].method = arr[i].method || 'GET';
+      arr[i].query = arr[i].query || {};
+      arr[i].body = arr[i].body || {};
+      arr[i].headers = arr[i].headers || {};
+      arr[i].handleFailure = arr[i].handleFailure || 'ignore';
+      arr[i].requestFields = arr[i].requestFields || {};
+      arr[i].responseDatakey = arr[i].responseDatakey || '';
+      arr[i].responseFields = arr[i].responseFields || {};
+    }
+  }
+}
+
+/**
+ * @summary Checks the properties passed in for the cache are valid,
+ * giving them default values if possible and removing them otherwise.
+ * @function validateProperties
+ * @param {Object} obj - properties json object
+ * @returns - adjusted valid properties with defaults
+ */
+function validateProperties(obj) {
+  const origin = `${id}-cacheHandler-validateProperties`;
+  log.trace(origin);
+
+  if (obj === null) {
+    log.error(`${origin}: Null properties!`);
+    throw new Error('Null properties!');
+  }
+
+  const props = obj;
+  // check cache, cache.enabled, cache.entities exist
+  if (!props.cache) {
+    // no cache, no props
+    props.cache = {
+      enabled: false,
+      entities: []
+    };
+  } else {
+    if (!props.cache.enabled) {
+      props.cache.enabled = false;
+      log.error(`${origin}: Passed in cache properties but did not specify enabled!`);
+    }
+    if (!props.cache.entities) {
+      props.cache.entities = [];
+      log.error(`${origin}: Passed in cache properties but did not specify entities (not even as empty array)!`);
+    }
+  }
+  // check each entity type in entities
+  for (let i = 0; i < props.cache.entities.length; i += 1) {
+    // must have entityType and populate
+    if (!props.cache.entities[i].entityType || !props.cache.entities[i].populate || props.cache.entities[i].populate.length === 0) {
+      log.warn(`${origin}: removed invalid cache property at index ${i}`);
+      // remove rather than throw error
+      props.cache.entities.splice(i, 1);
+      i -= 1; // already pointing at next after splice
+      break;
+    }
+
+    // default frequency = 24 hrs, caps if <60 or >10080
+    if (!props.cache.entities[i].frequency) {
+      props.cache.entities[i].frequency = 24 * 60; // 1 day in minutes
+    } else if (props.cache.entities[i].frequency < 15) { // 15 min minimum
+      log.warn(`${origin}: Passed in frequency of ${props.cache.entities[i].frequency} is too small! Set to 15 minute.`);
+      props.cache.entities[i].frequency = 15;
+    } else if (props.cache.entities[i].frequency > 10080) { // 1 week max
+      log.warn(`${origin}: Passed in frequency of ${props.cache.entities[i].frequency} is too large! Set to 1 week.`);
+      props.cache.entities[i].frequency = 10080;
+    }
+
+    // default flushOnFail = false
+    if (!props.cache.entities[i].flushOnFail) {
+      props.cache.entities[i].flushOnFail = false;
+    }
+
+    // Limits entity type to 1000 entities
+    if (!props.cache.entities[i].limit) {
+      props.cache.entities[i].limits = 1000;
+    }
+
+    if (!props.cache.entities[i].retryAttempts) {
+      props.cache.entities[i].retryAttempts = 5; // currently only used for startup
+    }
+
+    // make sure task and filterField exists for every object in cachedTasks
+    if (!props.cache.entities[i].cachedTasks) {
+      props.cache.entities[i].cachedTasks = [];
+    } else {
+      props.cache.entities[i].cachedTasks.forEach((cacheTask) => {
+        const curTask = cacheTask; // for lint
+        if (!curTask.name) {
+          log.error('Cached task has no task name!');
+          curTask.name = ''; // prevent errors, will not ever match a task name
+        }
+        if (!curTask.filterField) {
+          // if no filter then assume is a get all call, would need to explicitly check this
+          curTask.filterField = '';
+        }
+      });
+    }
+
+    // fill in missing populate fields
+    validatePopulate(props.cache.entities[i].populate);
+  }
+  return props;
+}
+
+/**
+ * @summary Compares two objects by name for sorting purposes.
+ * @function compareByName
+ * @param {Object} a - object to compare
+ * @param {Object} b - object to compare
+ * @returns {int} - specifying data order in relation to each other
+ */
+function compareByName(a, b) {
+  if (a.name > b.name) {
+    return 1;
+  }
+  if (a.name < b.name) {
+    return -1;
+  }
+  return 0;
+}
+
+/**
+ * @summary tests if object is an options object. Defaults:
+ *  - filter = ""
+ *  - start = 0 and limit = MAX_SAFE_INTEGER
+ *  - sort = false
+ * @function validateOptions
+ * @param {Object} obj - object to be tested
+ * @return {Object} - modified object with defaults
+ */
+function validateOptions(obj) {
+  const origin = `${id}-cacheHandler-validateOptions`;
+  log.trace(origin);
+
+  const opt = obj || {};
+
+  const keys = Object.keys(opt);
+  if (!keys.includes('filter')) {
+    opt.filter = {}; // filter for nothing means get everything
+  }
+  if (!keys.includes('start') && !keys.includes('limit')) {
+    opt.start = 0;
+    opt.limit = Number.MAX_SAFE_INTEGER; // return everything, maxed at max safe integer
+  } else if (!(keys.includes('start') && keys.includes('limit')) || opt.start < 0 || opt.limit < 0) {
+    log.error(`${origin}: incomplete or invalid options with start and limit!`);
+    throw new Error('Invalid options on accessing cache');
+  }
+
+  return opt;
+}
+
+/**
+ * @summary Helper method that retrieves entities from the cache
+ * @function retrieveCacheEntriesHelper
+ *
+ * @param {Object} cache - cache to search through
+ * @param {String} entityType - entity type to retrieve
+ * @param {Object} options - options on how to filter and return the data
+ */
+function retrieveCacheEntriesHelper(cache, entityType, options, callback) {
+  // go through the cache to find the entity we care about
+  for (let i = 0; i < cache.length; i += 1) {
+    // check if entity type we care about
+    if (cache[i].entityType === entityType) {
+      // Lock the cache for the entity so we can take what is in the cache
+      // Lock and instant unlock as wait until lock is free
+      return lock.acquire(cache[i].lockKey, (done) => {
+        // return the entity's list and free the lock
+        done(cache[i].list);
+      }, (ret) => {
+        let arr = [];
+
+        // populate re-attempt fails
+        if (!ret) {
+          log.error(`${origin}: No list in retrieve helper, re-attempt populate failed!`);
+          return callback(null, 'No list in retrieve cache helper!');
+        }
+
+        // no cache data found
+        if (ret.length === 0) {
+          log.info('Cache data retrieved and found to be empty.');
+          return callback(arr);
+        }
+
+        // apply filter
+        if (!options.filter || Object.keys(options.filter).length === 0) {
+          // no filter, get everything
+          log.info('No filter applied to cache retrieved');
+          arr = ret;
+        } else {
+          // get cache data with substring in name
+          const filterField = Object.keys(options.filter)[0];
+          const filter = options.filter[filterField];
+          for (let j = 0; j < ret.length; j += 1) {
+            if (ret[j][filterField].includes(filter)) {
+              arr.push(ret[j]);
+            }
+          }
+        }
+
+        // don't need to sort if we know it's OOB
+        if (options.limit * options.start >= arr.length) {
+          log.warn(`${options.start} is not a valid page, too large!`);
+          return callback([]);
+        }
+
+        // pagination: get entities limit*start to limit*(start+1) - 1
+        if (options.start >= 0 && options.limit > 0) { // Probably should handle this in properties.
+          // if last "page" then will be returning fewer items
+          const end = Math.min(arr.length, options.limit * (options.start + 1));
+          arr = arr.slice(options.limit * options.start, end);
+        }
+        return callback(arr); // for linting
+      });
+    }
+  }
+  log.error(`${origin}: No entityType ${entityType} in retrieve helper!`);
+  return callback(null, `No ${entityType} in retrieve cache helper!`); // error not found
+}
+
+/**
+ * @summary Removes all data of a certain entity type
+ * @function removeCacheEntry
+ * @param {Array} cache - the cache to remove from
+ * @param {String} entityType - the entity type of which to remove
+ */
+function removeCacheEntry(cache, entityType) {
+  const origin = `${id}-cacheHandler-removeCacheEntry`;
+  log.trace(origin);
+
+  for (let i = 0; i < cache.length; i += 1) {
+    if (cache[i].entityType === entityType) {
+      lock.acquire(cache[i].lockKey, () => {
+        clearInterval(cache[i].intervalId);
+        cache[i].splice(i, 1);
+      });
+      log.info(`Successfully removed ${entityType} from cache.`);
+      return;
+    }
+  }
+  log.error(`${origin}: Did not find cache type ${entityType} to remove!`);
+}
+
+/**
+ * @function getDataFromSources
+ * @summary INTERNAL FUNCTION: get data from source(s) - nested
+ * @param {*} loopField - fields
+ * @param {Array} sources - sources to look into
+ * @returns {*} - nested field value
+ */
+function getDataFromSources(loopField, sources) {
+  let fieldValue = loopField;
+
+  // go through the sources to find the field
+  for (let s = 0; s < sources.length; s += 1) {
+    // find the field value using jsonquery
+    const nestedValue = jsonQuery(loopField, { data: sources[s] }).value;
+
+    // if we found in source - set and no need to check other sources
+    if (nestedValue) {
+      fieldValue = nestedValue;
+      break;
+    }
+  }
+
+  return fieldValue;
+}
+
+/**
+ * @summary Manipulates data from call to only represent the specified fields and id and name if they do not have one
+ *
+ * @function parseResponseFields
+ * @param {Boolean} responseFull - whether or not to take in all fields
+ * @param {Object of output to call result} responseFields - fields to take in
+ * - ie response field of "name": "id" will give the iap call output a name field
+ *  with the value being the id
+ * @param {Array of Objects} allData - data returned from IAP Call
+ * @return {Array of Objects} - modified data
+ */
+function parseResponseFields(responseFields, allData, end, requestFields) {
+  const origin = `${id}-cacheHandler-createCacheData`;
+  log.trace(origin);
+
+  const parsedData = [];
+  const rfKeys = Object.keys(responseFields);
+  log.debug(rfKeys);
+
+  let ostypePrefix = '';
+  if (responseFields.ostypePrefix) {
+    ostypePrefix = responseFields.ostypePrefix;
+  }
+
+  let statusValue = true;
+  if (responseFields.statusValue) {
+    statusValue = responseFields.statusValue;
+  }
+
+  allData.response.forEach((currData) => {
+    const newObj = currData;
+
+    for (let rf = 0; rf < rfKeys.length; rf += 1) {
+      if (rfKeys[rf] !== 'ostypePrefix') {
+        let fieldValue = getDataFromSources(responseFields[rfKeys[rf]], [currData, { fake: 'fakedata' }, requestFields]);
+
+        // if the field is ostype - need to add prefix
+        if (rfKeys[rf] === 'ostype' && typeof fieldValue === 'string') {
+          fieldValue = ostypePrefix + fieldValue;
+        }
+        // if there is a status to set, set it
+        if (rfKeys[rf] === 'status') {
+          // if really looking for just a good response
+          if (responseFields[rfKeys[rf]] === 'return2xx' && allData.icode === statusValue.toString()) {
+            newObj.isAlive = true;
+          } else if (fieldValue.toString() === statusValue.toString()) {
+            newObj.isAlive = true;
+          } else {
+            newObj.isAlive = false;
+          }
+        }
+        // if we found a good value
+        newObj[rfKeys[rf]] = fieldValue;
+      }
+    }
+
+    if (!Object.prototype.hasOwnProperty.call(currData, 'name')) {
+      newObj.name = generateName();
+    }
+
+    // no longer requiring id field
+
+    parsedData.push(newObj);
+  });
+  return parsedData;
+}
+
+/**
+ * @summary makes IAP Calls through Generic Handler
+ *
+ * @function makeIAPCall
+ * @param calls - calls to make
+ * @param requestHandler - instance of requestHandler to make call
+ * @param callback - data or error
+ */
+function makeIAPCall(calls, requestHandler, callback) { // todo pass in properties from cacheHandler
+  const callPromises = [];
+  log.debug('Starting an iap call from cache.');
+  for (let i = 0; i < calls.length; i += 1) {
+    log.debug('Response :', calls[i].responseFields);
+    callPromises.push(new Promise((resolve, reject) => {
+      requestHandler.genericAdapterRequest(calls[i].path, calls[i].method, calls[i].query, calls[i].body, calls[i].headers, (callRet, callErr) => {
+        if (callErr) {
+          log.error('Make iap call failed with error');
+          log.error(callErr);
+          if (callErr.icode === 'AD.301') {
+            log.error(callErr.IAPerror.displayString);
+            return reject(callErr);
+          }
+          if (calls[i].handleFailure === 'ignore') {
+            log.info(`Call failed for path ${calls[i].path} with error code ${callErr.icode}. Ignoring.`);
+            return resolve([]);
+          }
+          log.warn(`Call failed for path ${calls[i].path} with error code ${callErr.icode}. Rejecting.`);
+          return reject(callErr);
+        }
+
+        // callRet is the object returned from that call
+        log.info(`Sucessful Call. Adding to cache from path ${calls[i].path}`);
+        const result = callRet;
+        if (calls[i].responseDatakey) {
+          result.response = jsonQuery(calls[i].responseDatakey, { data: result.response }).value;
+        }
+        return resolve(parseResponseFields(calls[i].responseFields, result, i, calls[i].requestFields));
+      });
+    }));
+  }
+
+  let returnData = [];
+  // Reworking the brokerHandler calls
+  return Promise.all(callPromises).then((results) => { // array of each Promise result
+    for (let i = 0; i < results.length; i += 1) {
+      returnData = returnData.concat(results[i]);
+    }
+    callback(returnData, null); // Returns data if all calls succeed
+  }, (error) => {
+    callback(null, error); // Throws error if at least one call failed
+  });
+}
+
+/**
+ * @summary Helper method that tries to populate the cache after 5 seconds
+ *
+ * @param {String} entityType - entity type to populate
+ * @param {CacheHandler} cacheHandler - instance of cacheHandler
+ * @param {Integer} attemptsRemaining - amount of attempts left to retry
+ * @param {function} callback - returns whether or not that cache was successfully populated
+ */
+function populateTimeout(entityType, cacheHandler, attemptsRemaining, callback) {
+  let ar = attemptsRemaining;
+  log.info(`Retrying to populate cache for ${entityType}. Attempts Remaining: ${ar}`);
+  setTimeout(() => {
+    cacheHandler.populateCache(entityType).then((result) => {
+      if (result && result[0] === 'success') {
+        return callback(true);
+      }
+      ar -= 1;
+      if (ar === 0) {
+        return callback(false);
+      }
+      return populateTimeout(entityType, cacheHandler, ar, callback);
+    });
+  }, 10000);
+}
+
+/**
+ * @summary tries to populate the cache again after populate fails on startup
+ *
+ * @param {String} entityType - entity type to populate
+ * @param {CacheHandler} cacheHandler - instance of CacheHandler
+ * @param {Integer} attempts - amount of attempts to retry
+ */
+async function retryPopulate(entityType, cacheHandler, attempts) {
+  log.info('Retrying populate...');
+  return new Promise((resolve) => {
+    populateTimeout(entityType, cacheHandler, attempts, (success) => {
+      if (success) {
+        return resolve(`Cache updated for ${entityType}.`);
+      }
+      return resolve(`Populate cache for ${entityType} failed.`);
+    });
+  });
+}
+
+// Exposed handler class
+class CacheHandler {
+  /**
+   * Adapter Cache Handler
+   * @constructor
+   */
+  constructor(prongId, properties, directory, reqH) {
+    id = prongId; // accessable to non-exposed methods
+    this.baseDir = directory;
+    this.requestHandler = reqH; // Request Handler object with id, props, dir
+
+    this.cache = []; // array of CacheEntity objects
+
+    // set up the properties I care about
+    this.propertiesMap = new Map(); // entityType->properties map
+    this.refreshProperties(properties);
+  }
+
+  /**
+   * refreshProperties is used to set up all of the properties for the cache handler.
+   * It allows properties to be changed later by simply calling refreshProperties rather
+   * than having to restart the cache handler.
+   *
+   * @function refreshProperties
+   * @param {Object} properties - an object containing all of the properties
+   */
+  refreshProperties(properties) {
+    const origin = `${id}-cacheHandler-refreshProperties`;
+    log.trace(origin);
+
+    if (properties === null) {
+      log.error(`${origin}: Cache Handler received no properties!`);
+      return;
+    }
+
+    this.props = validateProperties(properties);
+
+    const wasEnabled = this.enabled || false; // in case undefined
+    this.enabled = this.props.cache.enabled;
+    if (!this.enabled) {
+      // destroy cache and properties for memory efficiency
+      if (wasEnabled) {
+        deleteCacheData(this);
+      }
+      log.warn('Cache was disabled from enabled, removed cache memory.');
+      return;
+    }
+
+    // check entityType was not deleted (if so flush that cache type)
+    this.propertiesMap.forEach((value, key) => {
+      for (let i = 0; i < this.props.cache.entities.length; i += 1) {
+        if (!this.props.cache.entities[i].name === key) { // currently in cache, but not props
+          removeCacheEntry(this.cache, key);
+          this.propertiesMap.delete(key);
+          log.warn(`${origin}: Removed cache entity type due to change in properties.`);
+          log.info(`Deleted props for entity type ${key} with val ${value}`);
+          return; // continue forEach
+        }
+      }
+    });
+
+    // set properties map
+    this.props.cache.entities.forEach((entity) => {
+      if (!this.propertiesMap.has(entity.entityType)) {
+        // entity does not exist yet
+        this.propertiesMap.set(entity.entityType, {});
+        this.propertiesMap.get(entity.entityType).frequency = entity.frequency;
+        this.propertiesMap.get(entity.entityType).flushOnFail = entity.flushOnFail;
+        this.propertiesMap.get(entity.entityType).populate = entity.populate;
+        this.propertiesMap.get(entity.entityType).limit = entity.limit;
+        this.propertiesMap.get(entity.entityType).retryAttempts = entity.retryAttempts;
+
+        const newIntervalId = setInterval(() => {
+          this.populateCache(entity.entityType);
+        }, entity.frequency * 60000);
+
+        const sort = Object.prototype.hasOwnProperty.call(entity, 'sort') ? entity.sort : true; // default true
+        this.cache.push(createCacheEntity(entity.entityType, null, newIntervalId, sort));
+        this.populateCache(entity.entityType).then((result) => {
+          if (result && result[0] === 'error') {
+            retryPopulate(entity.entityType, this, entity.retryAttempts).then((nextResult) => {
+              log.info(nextResult);
+            });
+          }
+        }).catch((error) => {
+          log.error('Failed populate cache!');
+          if (error.icode === 'AD.301') {
+            log.error(`Destroying interval, check path and call in properties for ${entity.entityType}`);
+            clearInterval(newIntervalId);
+            this.cache.removeCacheEntry(this.cache, entity.entityType);
+          }
+        });
+      } else {
+        // entity props frequency updated
+        if (entity.frequency !== this.propertiesMap.get(entity.entityType).frequency) {
+          changeUpdateFrequency(this.cache, entity.entityType, entity.frequency, this);
+        }
+
+        this.propertiesMap.get(entity.entityType).frequency = entity.frequency;
+        this.propertiesMap.get(entity.entityType).flushOnFail = entity.flushOnFail;
+        this.propertiesMap.get(entity.entityType).populate = entity.populate;
+        this.propertiesMap.get(entity.entityType).limit = entity.limit;
+        this.propertiesMap.get(entity.entityType).retryAttempts = entity.retryAttempts;
+      }
+    });
+  }
+
+  /**
+   * @summary Populates/updates cache with entities' data from IAP call
+   * @function populateCache
+   * @param {String/Array of Strings} entities - entities of which we want to store
+   * @param {Boolean} onlyIfNecessary - If true and cache has been populated after acquiring the lock, does nothing
+   * @return {Array of Strings} - whether each entity succeeded or errored
+   */
+  async populateCache(entities) {
+    const origin = `${id}-cacheHandler-populateCache`;
+    log.trace(origin);
+
+    if (!this.enabled) {
+      return Promise.reject(new Error('Cache is not enabled!'));
+    }
+
+    log.info('Populating cache.');
+
+    // support string and array input
+    let entityArr = entities;
+    if (!Array.isArray(entities)) {
+      entityArr = [entities];
+    }
+
+    // need all properties to make iap call
+    for (let i = 0; i < entityArr.length; i += 1) {
+      const entityType = entityArr[i];
+      if (!this.propertiesMap.has(entityType)) {
+        log.error(`${entityType} is an untracked entity type! Check properties.`);
+        return Promise.reject(new Error(`${entityType} is an untracked entity type! Check properties.`));
+      }
+    }
+
+    try {
+      const promiseArr = [];
+      // for each entityType need to populate
+      for (let j = 0; j < entityArr.length; j += 1) {
+        const entityType = entityArr[j];
+        // try to find current entityType in cache
+        for (let i = 0; i < this.cache.length; i += 1) {
+          if (this.cache[i].entityType === entityType) { // will always be found
+            promiseArr.push(new Promise((resolve, reject) => {
+              makeIAPCall(this.propertiesMap.get(entityType).populate, this.requestHandler, (cacheData, error) => {
+                // error message if applicable
+                let errorMsg = null;
+                if (error) {
+                  errorMsg = `${origin}: ${entityType} failed with error ${error.IAPerror.displayString}.`;
+                } else if (cacheData.length > this.propertiesMap.get(entityType).limit) {
+                  errorMsg = `${origin}: ${entityType} failed. Data surpassed the limit.`;
+                } else if (cacheData.length === 0) {
+                  errorMsg = `${origin}: ${entityType} failed. Nothing found.`;
+                }
+                // handle error
+                if (errorMsg !== null) {
+                  /**
+                   * removed unnecessary check if cache[i].list is null here
+                   * setting null var to null again is fine and shortens critical section
+                   * */
+
+                  if (this.propertiesMap.get(entityType).flushOnFail) {
+                    lock.acquire(this.cache[i].lockKey, (done) => {
+                      this.cache[i].list = null;
+                      done(`${errorMsg} Flushed.`);
+                    }, (ret) => {
+                      log.error(ret);
+                    });
+                  } else {
+                    log.error(`${errorMsg} Keeping old data.`);
+                  }
+
+                  // bad path
+                  if (error && error.icode === 'AD.301') {
+                    return reject(error);
+                  }
+                  return resolve('error');
+                }
+                // sort cache data, default true
+                if (this.cache[i].sort) {
+                  cacheData.sort(compareByName);
+                }
+
+                // no error, edit cache
+                lock.acquire(this.cache[i].lockKey, (done) => {
+                  if (this.enabled) { // in case properties changes during iap call
+                    this.cache[i].list = cacheData;
+                    done(`${entityType}: cache updated.`);
+                  } else {
+                    done('Populate cancelled due to disabled cache!');
+                  }
+                }, (ret) => {
+                  log.info(ret);
+                });
+                return resolve('success');
+              });
+            }));
+            break;
+          }
+        }
+      }
+      const arr = await Promise.allSettled(promiseArr);
+      return new Promise((resolve, reject) => {
+        const valueArray = [];
+        for (let p = 0; p < arr.length; p += 1) {
+          if (arr[p].status === 'rejected') {
+            reject(arr[p].reason);
+          }
+          valueArray.push(arr[p].value);
+        }
+        resolve(valueArray);
+      });
+    } catch (e) {
+      // re-throw any exception
+      throw new Error(e);
+    }
+  }
+
+  /**
+   * @summary Retrieves an entity's data from cache under specified filters
+   * @function retrieveCacheEntries
+   * @param {String} entityType - entity type that we want to retrieve items from
+   * @param {Object} opts - Options of what we want to search for and return
+   *                           Keys:
+   *                              Filter (String) - Substring in which to see which's data
+   *                                                names contain
+   *                              Start (int) - Pagination Number (Not sure how to implement)
+   *                              Limit (int) - limit of how much data to send back
+   *                              Sort (Boolean) - whether we sort the data or not
+   * @return {Array} - array of data objects
+   */
+  retrieveCacheEntries(entityType, opts, callback) {
+    const origin = `${id}-cacheHandler-retrieveCacheEntries`;
+    log.trace(origin);
+
+    if (!this.enabled) {
+      log.warn(`${origin}: Cache is not enabled!`);
+      return callback(null, 'Cache is not enabled.');
+    }
+
+    if (typeof entityType !== 'string') {
+      log.error(`${entityType} is not of type String`);
+      return callback(null, `${entityType} is not of type String`);
+    }
+
+    try {
+      const options = validateOptions(opts); // may throw error
+      // go through the cache to find the entity we care about
+      for (let i = 0; i < this.cache.length; i += 1) {
+        // check if entity type we care about
+        if (this.cache[i].entityType === entityType) {
+          // Lock the cache for the entity so we can take what is in the cache
+          // Lock and instant unlock as wait until lock is free
+          return lock.acquire(this.cache[i].lockKey, (done) => {
+            // return the entity's list and free the lock
+            done(this.cache[i].list);
+          }, (ret) => {
+            // last iap call failed and flushed, attempt new iap call
+            if (!ret) {
+              // this returns a promise but below we return a callback...
+              return this.populateCache(entityType).then((result) => {
+                if (result && result[0] === 'success') {
+                  return retrieveCacheEntriesHelper(this.cache, entityType, options, callback);
+                }
+                return callback(null, `Retrieve call for ${entityType} failed, and populate re-attempt failed`);
+              });
+            }
+
+            // normal retrieve
+            return retrieveCacheEntriesHelper(this.cache, entityType, options, callback);
+          });
+        }
+      }
+      log.error(`${origin}: Failed to find ${entityType} in retrieve!`);
+      return callback(null, `Retrieve call for ${entityType} failed.`);
+    } catch (e) {
+      // re-throw any exception
+      throw new Error(e);
+    }
+  }
+
+  /**
+   * @summary Method to check if the entity is in the cache. Faster than
+   * retrieveCacheEntries hence the seperation
+   *
+   * @function isEntityCached
+   * @param {String} entityType - the entity type to check for
+   * @param {Array of String} entityNames - the specific entities we are looking for of that type
+   *
+   * @return {Array of Boolean} - whether associated entity is found
+   */
+  isEntityCached(entityType, entityNames, callback) {
+    const origin = `${id}-cacheHandler-isEntityCached`;
+    log.trace(origin);
+    // Possibly supporting checking if entityType is cache? (set entityId = null)
+    if (!this.enabled) {
+      log.warn(`${origin}: Cache is not enabled!`);
+      return callback(null, 'Cache is not enabled!');
+    }
+
+    let names = entityNames;
+    if (!Array.isArray(entityNames)) {
+      names = [entityNames];
+    }
+
+    try {
+      for (let i = 0; i < this.cache.length; i += 1) {
+        if (this.cache[i].entityType === entityType) {
+          const returnVals = [];
+          return lock.acquire(this.cache[i].lockKey, (done) => {
+            done(this.cache[i].list);
+          }, (ret) => {
+            // null list means previously flushed or failed called
+            if (!ret) {
+              log.info(`Did not find ${entityType}, attempting populate in isEntityCached`);
+              return this.populateCache(entityType).then((result) => { // populate will re-acquire lock
+                if (result && result[0] === 'success') {
+                  // re-acquire lock to read list
+                  lock.acquire(this.cache[i].lockKey, (done) => {
+                    names.forEach((name) => {
+                      for (let j = 0; j < this.cache[i].list.length; j += 1) {
+                        if (this.cache[i].list[j].name === name) {
+                          returnVals.push('found');
+                          return; // forEach
+                        }
+                      }
+                      returnVals.push('notfound');
+                    });
+                    done(returnVals);
+                  }, (retVals) => {
+                    log.info(retVals);
+                    return callback(retVals);
+                  });
+                }
+                return callback(null, `isEntityCached call for ${entityType} failed.`);
+              });
+            }
+
+            names.forEach((curName) => {
+              for (let j = 0; j < ret.length; j += 1) {
+                if (ret[j].name === curName) {
+                  returnVals.push('found');
+                  return; // continue the forEach
+                }
+              }
+              returnVals.push('notfound');
+            });
+            log.info(returnVals);
+            return callback(returnVals);
+          }); // end of lock acquire
+        }
+      }
+      log.error(`${entityType} not found in cache.`);
+      return callback(false);
+    } catch (e) {
+      log.error(e);
+      return callback(false);
+    }
+  }
+
+  /**
+   * @function isEntityTypeToBeCached
+   * @summary Checks whether the cache properties for that entity type
+   * is set up, meaning that the entity type should be cached
+   * @param {String} entityType - the entity type to check for
+   * @returns {Boolean} - whether the entity type is set to be cached
+   */
+  isEntityTypeToBeCached(entityType, callback) {
+    const origin = `${id}-cacheHandler-isEntityTypeToBeCached`;
+    log.trace(origin);
+
+    if (!this.enabled) {
+      log.error('Cache is not enabled!');
+      return callback(null, 'Cache is not enabled!');
+    }
+
+    return callback(this.propertiesMap.has(entityType));
+  }
+
+  /**
+   * @function isTaskCached
+   * @summary Checks whether the task should be cached, for adapter
+   * (non-broker level) calls
+   * @param {String} entityType - entity type from identifyRequest
+   * @param {String} task - method name
+   */
+  isTaskCached(entityType, task) {
+    const origin = `${id}-cacheHandler-isTaskCached`;
+    log.trace(origin);
+
+    // do not bother with callbacks here
+    if (!this.enabled || !this.propertiesMap.has(entityType)) {
+      return false;
+    }
+
+    // find entity type
+    for (let i = 0; i < this.props.cache.entities.length; i += 1) {
+      if (this.props.cache.entities[i].entityType === entityType) {
+        // check if contains task
+        for (let c = 0; c < this.props.cache.entities[i].cachedTasks.length; c += 1) {
+          if (this.props.cache.entity[i].cachedTasks[c].name === task) {
+            return true;
+          }
+        }
+        // does not have task under that entityType
+        return false;
+      }
+    }
+    // entityType not cached, should never reach here
+    log.error(`${origin}: EntityType not found in cache, should never reach here.`);
+    return false;
+  }
+}
+
+module.exports = CacheHandler;