@itentialopensource/adapter-utils 5.1.6 → 5.2.0

package/CHANGELOG.md

@@ -1,4 +1,24 @@
 
+## 5.2.0 [10-04-2023]
+
+* Broker Changes
+
+Closes ADAPT-2848
+
+See merge request itentialopensource/adapter-utils!277
+
+---
+
+## 5.1.7 [09-09-2023]
+
+* Added pagination to expanded generic handler
+
+Closes ADAPT-2849
+
+See merge request itentialopensource/adapter-utils!276
+
+---
+
 ## 5.1.6 [09-06-2023]
 
 * add fixes to connector that are missing

package/lib/cacheHandler.js

@@ -7,7 +7,6 @@
 
 const AsyncLockCl = require('async-lock');
 const path = require('path');
-const jsonQuery = require('json-query');
 
 const lock = new AsyncLockCl();
 let id = null;
@@ -32,16 +31,6 @@ function createCacheEntity(entityName, entityList, interval, sortEntities = true
   });
 }
 
-/**
- * @summary Generates name for data object. Currently non-unique
- * @function generateName
- * @returns {String} - generated name
- */
-function generateName() {
-  log.warn('Name for entity not found, generated name placeholder');
-  return 'GeneratedName';
-}
-
 /**
  * @summary Deletes cache data and properties. Data deletion
  * saves considerable memory.
@@ -358,98 +347,6 @@ function removeCacheEntry(cache, entityType) {
   log.error(`${origin}: Did not find cache type ${entityType} to remove!`);
 }
 
-/**
- * @function getDataFromSources
- * @summary INTERNAL FUNCTION: get data from source(s) - nested
- * @param {*} loopField - fields
- * @param {Array} sources - sources to look into
- * @returns {*} - nested field value
- */
-function getDataFromSources(loopField, sources) {
-  let fieldValue = loopField;
-
-  // go through the sources to find the field
-  for (let s = 0; s < sources.length; s += 1) {
-    // find the field value using jsonquery
-    const nestedValue = jsonQuery(loopField, { data: sources[s] }).value;
-
-    // if we found in source - set and no need to check other sources
-    if (nestedValue) {
-      fieldValue = nestedValue;
-      break;
-    }
-  }
-
-  return fieldValue;
-}
-
-/**
- * @summary Manipulates data from call to only represent the specified fields and id and name if they do not have one
- *
- * @function parseResponseFields
- * @param {Boolean} responseFull - whether or not to take in all fields
- * @param {Object of output to call result} responseFields - fields to take in
- * - ie response field of "name": "id" will give the iap call output a name field
- *  with the value being the id
- * @param {Array of Objects} allData - data returned from IAP Call
- * @return {Array of Objects} - modified data
- */
-function parseResponseFields(responseFields, allData, end, requestFields) {
-  const origin = `${id}-cacheHandler-createCacheData`;
-  log.trace(origin);
-
-  const parsedData = [];
-  const rfKeys = Object.keys(responseFields);
-  log.debug(rfKeys);
-
-  let ostypePrefix = '';
-  if (responseFields.ostypePrefix) {
-    ostypePrefix = responseFields.ostypePrefix;
-  }
-
-  let statusValue = true;
-  if (responseFields.statusValue) {
-    statusValue = responseFields.statusValue;
-  }
-
-  allData.response.forEach((currData) => {
-    const newObj = currData;
-
-    for (let rf = 0; rf < rfKeys.length; rf += 1) {
-      if (rfKeys[rf] !== 'ostypePrefix') {
-        let fieldValue = getDataFromSources(responseFields[rfKeys[rf]], [currData, { fake: 'fakedata' }, requestFields]);
-
-        // if the field is ostype - need to add prefix
-        if (rfKeys[rf] === 'ostype' && typeof fieldValue === 'string') {
-          fieldValue = ostypePrefix + fieldValue;
-        }
-        // if there is a status to set, set it
-        if (rfKeys[rf] === 'status') {
-          // if really looking for just a good response
-          if (responseFields[rfKeys[rf]] === 'return2xx' && allData.icode === statusValue.toString()) {
-            newObj.isAlive = true;
-          } else if (fieldValue.toString() === statusValue.toString()) {
-            newObj.isAlive = true;
-          } else {
-            newObj.isAlive = false;
-          }
-        }
-        // if we found a good value
-        newObj[rfKeys[rf]] = fieldValue;
-      }
-    }
-
-    if (!Object.prototype.hasOwnProperty.call(currData, 'name')) {
-      newObj.name = generateName();
-    }
-
-    // no longer requiring id field
-
-    parsedData.push(newObj);
-  });
-  return parsedData;
-}
-
 /**
  * @summary makes IAP Calls through Generic Handler
  *
@@ -458,13 +355,18 @@ function parseResponseFields(responseFields, allData, end, requestFields) {
  * @param requestHandler - instance of requestHandler to make call
  * @param callback - data or error
  */
-function makeIAPCall(calls, requestHandler, callback) { // todo pass in properties from cacheHandler
+function makeIAPCall(calls, requestHandler, callback) {
   const callPromises = [];
-  log.debug('Starting an iap call from cache.');
+
   for (let i = 0; i < calls.length; i += 1) {
     log.debug('Response :', calls[i].responseFields);
     callPromises.push(new Promise((resolve, reject) => {
-      requestHandler.genericAdapterRequest(calls[i].path, calls[i].method, calls[i].query, calls[i].body, calls[i].headers, (callRet, callErr) => {
+      const metadata = {};
+      if (calls[i].pagination) {
+        metadata.pagination = calls[i].pagination;
+        metadata.pagination.responseDatakey = calls[i].responseDatakey || '';
+      }
+      requestHandler.iapMakeGenericCall(metadata, calls[i].path, calls[i], [{ fake: 'fakedata' }], [], (callRet, callErr) => {
         if (callErr) {
           log.error('Make iap call failed with error');
           log.error(callErr);
@@ -472,21 +374,13 @@ function makeIAPCall(calls, requestHandler, callback) { // todo pass in properti
             log.error(callErr.IAPerror.displayString);
             return reject(callErr);
           }
-          if (calls[i].handleFailure === 'ignore') {
-            log.info(`Call failed for path ${calls[i].path} with error code ${callErr.icode}. Ignoring.`);
-            return resolve([]);
-          }
           log.warn(`Call failed for path ${calls[i].path} with error code ${callErr.icode}. Rejecting.`);
           return reject(callErr);
         }
 
         // callRet is the object returned from that call
         log.info(`Sucessful Call. Adding to cache from path ${calls[i].path}`);
-        const result = callRet;
-        if (calls[i].responseDatakey) {
-          result.response = jsonQuery(calls[i].responseDatakey, { data: result.response }).value;
-        }
-        return resolve(parseResponseFields(calls[i].responseFields, result, i, calls[i].requestFields));
+        return resolve(callRet);
       });
     }));
   }

package/lib/genericHandler.js

@@ -5,6 +5,54 @@
 /* eslint consistent-return: warn */
 
 /* NodeJS internal utilities */
+const jsonQuery = require('json-query');
+
+// INTERNAL FUNCTIONS
+
+/**
+ * @summary Update the offset for paginated call.
+ * @function incrementOffset
+ * @param {String} offsetType - type of offset (page or limit)
+ * @param {Number} data - previous offset
+ * @param {Number} limit - call limit
+ * @returns {Number} - new offset
+ */
+function incrementOffset(offsetType, previousOffset, limit) {
+  let newOffset;
+  if (offsetType === 'limit') {
+    newOffset = previousOffset + limit;
+  } else if (offsetType === 'page') {
+    newOffset = previousOffset + 1;
+  } else {
+    throw new Error(`Offset Type : ${offsetType} not supported or insufficient data was provided`);
+  }
+  return newOffset;
+}
+
+/**
+ * @summary Function to set the value at a specified property path
+ * @function setNestedProperty
+ * @param {Object} oldValue - Object to update
+ * @param {String} responseDataKey - path to property to update in . separated string
+ * @param {Object} newValue - Object with data to use in update
+ */
+function setNestedProperty(oldValue, responseDatakey, newValue) {
+  const newRes = jsonQuery(responseDatakey, { data: newValue.response }).value;
+  const oldRes = jsonQuery(responseDatakey, { data: oldValue.response }).value;
+  const path = responseDatakey.split('.');
+  let currentObj = oldValue.response;
+  for (let i = 0; i < path.length - 1; i += 1) {
+    const segment = path[i];
+    if (Object.hasOwnProperty.call(currentObj, segment)) {
+      currentObj = currentObj[segment];
+    } else {
+      currentObj[segment] = {};
+      currentObj = currentObj[segment];
+    }
+  }
+  // Modifies value at path and updates original object
+  currentObj[path[path.length - 1]] = oldRes.concat(newRes);
+}
 
 class GenericHandler {
   /**
@@ -156,7 +204,7 @@ class GenericHandler {
           }
           reqObj.addlHeaders = { ...reqObj.addlHeaders, ...signature };
           this.requestHandlerInst.identifyRequest('.generic', action, reqObj, returnF, (irReturnData, irReturnError) => {
-            // if we received an error or their is no response on the results
+            // if we received an error or there is no response on the results
             // return an error
             if (irReturnError) {
               /* HERE IS WHERE YOU CAN ALTER THE ERROR MESSAGE */
@@ -174,6 +222,24 @@ class GenericHandler {
           });
         });
       }
+
+      // Does not support AWS adapters due to the auth call above
+      if (metadata && metadata.pagination) {
+        return this.expandedGenericAdapterRequestPaginated(metadata.pagination, action, reqObj, returnF, meth, (returnData, returnError) => {
+          if (returnError) {
+            /* HERE IS WHERE YOU CAN ALTER THE ERROR MESSAGE */
+            return callback(null, returnError);
+          }
+          if (!Object.hasOwnProperty.call(returnData, 'response')) {
+            const errorObj = this.requestHandlerInst.formatErrorObject(this.myid, meth, 'Invalid Response', ['genericAdapterRequest'], null, null, null);
+            log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+            return callback(null, errorObj);
+          }
+          /* HERE IS WHERE YOU CAN ALTER THE RETURN DATA */
+          // return the response
+          return callback(returnData, null);
+        });
+      }
       return this.requestHandlerInst.identifyRequest('.generic', action, reqObj, returnF, (irReturnData, irReturnError) => {
         // if we received an error or their is no response on the results
         // return an error
@@ -198,6 +264,85 @@ class GenericHandler {
     }
   }
 
+  /**
+   * Makes the requested generic call
+   *
+   * @function expandedGenericAdapterRequestPaginated
+   * @param {Object} metadata - metadata for the call (optional).
+   *                 Can be a stringified Object.
+   * @param {String} uriPath - the path of the api call - do not include the host, port, base path or version (optional)
+   * @param {String} restMethod - the rest method (GET, POST, PUT, PATCH, DELETE) (optional)
+   * @param {Object} pathVars - the parameters to be put within the url path (optional).
+   *                 Can be a stringified Object.
+   * @param {Object} queryData - the parameters to be put on the url (optional).
+   *                 Can be a stringified Object.
+   * @param {Object} requestBody - the body to add to the request (optional).
+   *                 Can be a stringified Object.
+   * @param {Object} addlHeaders - additional headers to be put on the call (optional).
+   *                 Can be a stringified Object.
+   * @param {Object} paginationObject - object specifying pagination variables and increment method
+   * @param {getCallback} callback - a callback function to return the result (Generics)
+   *                 or the error
+   */
+  expandedGenericAdapterRequestPaginated(paginationObject, action, reqObj, returnF, meth, callback) {
+    const origin = `${this.myid}-requestHandler-expandedGenericAdapterRequestPaginated`;
+    log.trace(origin);
+    let results;
+    // Set up variables for calls
+    const { offsetVar } = paginationObject;
+    const pagType = paginationObject.requestLocation; // Body or query supported
+    let pagLocation;
+    if (pagType === 'body') {
+      pagLocation = 'payload';
+    } else if (pagType === 'query') {
+      pagLocation = 'uriQuery';
+    } else {
+      const err = new Error(`Pagination Type : ${pagType} not supported or insufficient data was provided`);
+      return callback(null, err);
+    }
+    const limit = reqObj[pagLocation][paginationObject.limitVar];
+    const recursiveCall = (currentOffset) => {
+      try {
+        const myReqObj = {
+          ...reqObj
+        };
+        myReqObj[pagLocation][offsetVar] = currentOffset;
+        this.requestHandlerInst.identifyRequest('.generic', action, myReqObj, returnF, (result, error) => {
+          if (error) {
+            return callback(null, error);
+          }
+
+          if (!Object.hasOwnProperty.call(result, 'response')) {
+            const errorObj = this.requestHandlerInst.formatErrorObject(this.myid, meth, 'Invalid Response', ['genericAdapterRequestPaginated'], null, null, null);
+            log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+            return callback(null, errorObj);
+          }
+          // Results hasn't been populated yet
+          if (!results) {
+            results = { ...result };
+          } else if (paginationObject.responseDatakey) {
+            setNestedProperty(results, paginationObject.responseDatakey, result);
+          } else {
+            results.response = results.response.concat(result.response);
+          }
+          if (result.response.length === limit) {
+            const newOffset = incrementOffset(paginationObject.incrementBy, currentOffset, limit);
+            // Rescurse
+            recursiveCall(newOffset);
+          } else {
+            return callback(results, null);
+          }
+        });
+      } catch (e) {
+        // handle any exception
+        const errorObj = this.transUtil.checkAndReturn(e, origin, 'Expanded Generic Adapter Request with Pagination Failed');
+        return callback(null, errorObj);
+      }
+    };
+
+    recursiveCall(reqObj[pagLocation][offsetVar]);
+  }
+
   /**
    * Makes the requested generic call
    *

package/lib/requestHandler.js

@@ -329,24 +329,109 @@ function dbCalls(collectionName, entity, action, data, callback) {
 /*
  * INTERNAL FUNCTION: get data from source(s) - nested
  */
-function getDataFromSources(loopField, sources) {
+function getDataFromSources(loopField, sources, props) {
   let fieldValue = loopField;
-
+  let foundProp = false;
   // go through the sources to find the field
   for (let s = 0; s < sources.length; s += 1) {
     // find the field value using jsonquery
     const nestedValue = jsonQuery(loopField, { data: sources[s] }).value;
 
     // if we found in source - set and no need to check other sources
-    if (nestedValue) {
+    if (nestedValue && Object.keys(nestedValue).length !== 0) {
       fieldValue = nestedValue;
+      foundProp = true;
       break;
     }
   }
 
+  // Check for field in adapter properties
+  if (!foundProp && props && props[loopField]) {
+    fieldValue = props[loopField];
+  }
+
   return fieldValue;
 }
 
+/**
+ * @summary  Extracts the keys to be replaced in request fields or response fields.
+ *
+ * @function extractKeysFromBraces
+ * @param {String} value - String in which to look for {} and extract values
+ *
+ * @return {Array} Array containing 0 or more keys found in input String
+ */
+function extractKeysFromBraces(value) {
+  const regex = /\{(.*?)\}/g;
+  const matches = value.match(regex);
+  if (matches) {
+    return matches.map((key) => key.replace(/{|}/g, ''));
+  }
+  return [];
+}
+
+/**
+ * @summary  Given a string containing a logical OR in braces {||}, return the LHS if value is found
+ * in sources, otherwise return the RHS
+ *
+ * @function setConditionalValue
+ * @param {String} valueField - String containing operands
+ * @param {Array} source - Array containing sources to search for operand values
+ * @param {Object} props - Object containing Adapter props as secondary source
+ *
+ * @return {String} Final value with data replaced in found keys.
+ */
+function setConditionalValue(valueField, sources, props) {
+  if (!valueField.includes('{||}')) {
+    throw new Error('This method is only to be used with Strings containing a logical OR in braces {||}');
+  }
+  const operands = valueField.split('{||}'); // Array of left side and right side
+  const operandA = extractKeysFromBraces(operands[0]); // [name]
+  const operandB = extractKeysFromBraces(operands[1]); // [serial]
+  let finalValue = operands[0]; // {name}
+  for (let i = 0; i < operandA.length; i += 1) {
+    const fieldValue = getDataFromSources(operandA[i], sources, props);
+    if (fieldValue === operandA[i]) {
+      // did not find value - break here to try second part of conditional
+      [, finalValue] = operands;
+      break;
+    }
+    finalValue = finalValue.replace(`{${operandA[i]}}`, fieldValue);
+  }
+
+  // Check if broke out of loop due to missing value
+  if (finalValue !== operands[1]) {
+    return finalValue;
+  }
+
+  // Look for values for keys to the right of {||}
+  for (let j = 0; j < operandB.length; j += 1) {
+    const fieldValue = getDataFromSources(operandB[j], sources, props);
+    if (fieldValue === operandB[j]) {
+      throw new Error(`Could not find value in sources for ${operandA} or ${operandB}`);
+    }
+    finalValue = finalValue.replace(`{${operandB[j]}}`, fieldValue);
+  }
+
+  return finalValue;
+}
+
+function setResponseDataFromSources(loopField, sources, props) {
+  if (loopField.includes('{||}')) {
+    return setConditionalValue(loopField, sources, props);
+  }
+  let myField = loopField;
+  const keys = extractKeysFromBraces(loopField);
+  // Handle if val not found ?
+  for (let k = 0; k < keys.length; k += 1) {
+    const responseKey = keys[k];
+    const fieldValue = getDataFromSources(responseKey, sources, props);
+    myField = myField.replace(`{${responseKey}}`, fieldValue);
+  }
+
+  return myField;
+}
+
 class RequestHandler {
   /**
    * Request Handler
@@ -553,19 +638,16 @@ class RequestHandler {
       let handleFail = 'fail';
       let ostypePrefix = '';
       let statusValue = 'true';
+      const sources = devResp.concat([callProps.requestFields]);
       if (callProps.path) {
         uriPath = `${callProps.path}`;
-
         // make any necessary changes to the path
-        if (devResp !== null && callProps.requestFields && Object.keys(callProps.requestFields).length > 0) {
-          const rqKeys = Object.keys(callProps.requestFields);
-
+        if (devResp !== null) {
           // get the field from the provided device
-          for (let rq = 0; rq < rqKeys.length; rq += 1) {
-            const fieldValue = getDataFromSources(callProps.requestFields[rqKeys[rq]], devResp);
-
-            // put the value into the path - if it has been specified in the path
-            uriPath = uriPath.replace(`{${rqKeys[rq]}}`, fieldValue);
+          const pathKeys = extractKeysFromBraces(uriPath);
+          for (let pathKey = 0; pathKey < pathKeys.length; pathKey += 1) {
+            const fieldValue = getDataFromSources(pathKeys[pathKey], sources, this.props);
+            uriPath = uriPath.replace(`{${pathKeys[pathKey]}}`, fieldValue);
           }
         }
       }
@@ -577,69 +659,41 @@ class RequestHandler {
         // go through the query params to check for variable values
         const cpKeys = Object.keys(callQuery);
         for (let cp = 0; cp < cpKeys.length; cp += 1) {
-          // if (callQuery[cpKeys[cp]].startsWith('{') && callQuery[cpKeys[cp]].endsWith('}')) {
-          // make any necessary changes to the query params
-          if (devResp !== null && callProps.requestFields && Object.keys(callProps.requestFields).length > 0) {
-            const rqKeys = Object.keys(callProps.requestFields);
-
-            // get the field from the provided device
-            for (let rq = 0; rq < rqKeys.length; rq += 1) {
-              if (callQuery[cpKeys[cp]] === rqKeys[rq]) {
-                const fieldValue = getDataFromSources(callProps.requestFields[rqKeys[rq]], devResp);
-                // put the value into the query - if it has been specified in the query
-                callQuery[cpKeys[cp]] = fieldValue;
-              }
-            }
+          // get array of values to replace
+          const matches = extractKeysFromBraces(callQuery[cpKeys[cp]]);
+          for (let m = 0; m < matches.length; m += 1) {
+            const queryKey = matches[m];
+            const fieldValue = getDataFromSources(queryKey, sources, this.props);
+            callQuery[cpKeys[cp]] = callQuery[cpKeys[cp]].replace(`{${queryKey}}`, fieldValue);
           }
-          // }
         }
       }
       if (callProps.body) {
         callBody = { ...callProps.body };
-
         // go through the body fields to check for variable values
         const cbKeys = Object.keys(callBody);
         for (let cb = 0; cb < cbKeys.length; cb += 1) {
-          // if (callBody[cbKeys[cb]].startsWith('{') && callBody[cbKeys[cb]].endsWith('}')) {
+          const matches = extractKeysFromBraces(callBody[cbKeys[cb]]);
+          for (let m = 0; m < matches.length; m += 1) {
           // make any necessary changes to the query params
-          if (devResp !== null && callProps.requestFields && Object.keys(callProps.requestFields).length > 0) {
-            const rqKeys = Object.keys(callProps.requestFields);
-
-            // get the field from the provided device
-            for (let rq = 0; rq < rqKeys.length; rq += 1) {
-              if (callBody[cbKeys[cb]] === rqKeys[rq]) {
-                const fieldValue = getDataFromSources(callProps.requestFields[rqKeys[rq]], devResp);
-
-                // put the value into the query - if it has been specified in the query
-                callBody[cbKeys[cb]] = fieldValue;
-              }
-            }
+            const bodyKey = matches[m];
+            const fieldValue = getDataFromSources(bodyKey, sources, this.props);
+            callBody[cbKeys[cb]] = callBody[cbKeys[cb]].replace(`{${bodyKey}}`, fieldValue);
           }
-          // }
         }
       }
       if (callProps.headers) {
         callHeaders = { ...callProps.headers };
-
-        // go through the body fields to check for variable values
+        // go through the header fields to check for variable values
         const chKeys = Object.keys(callHeaders);
         for (let ch = 0; ch < chKeys.length; ch += 1) {
-          // if (callHeaders[chKeys[ch]].startsWith('{') && callHeaders[chKeys[ch]].endsWith('}')) {
-          // make any necessary changes to the query params
-          if (devResp !== null && callProps.requestFields && Object.keys(callProps.requestFields).length > 0) {
-            const rqKeys = Object.keys(callProps.requestFields);
-
-            // get the field from the provided device
-            for (let rq = 0; rq < rqKeys.length; rq += 1) {
-              if (callHeaders[chKeys[ch]] === rqKeys[rq]) {
-                const fieldValue = getDataFromSources(callProps.requestFields[rqKeys[rq]], devResp);
-
-                // put the value into the query - if it has been specified in the query
-                callHeaders[chKeys[ch]] = fieldValue;
-              }
-            }
+          const matches = extractKeysFromBraces(callHeaders[chKeys[ch]]);
+          for (let m = 0; m < matches.length; m += 1) {
+            const headerKey = matches[m];
+            // make any necessary changes to the query params
+            const fieldValue = getDataFromSources(headerKey, sources, this.props);
+            callHeaders[chKeys[ch]] = callQuery[chKeys[ch]].replace(`{${headerKey}}`, fieldValue);
           }
-          // }
         }
       }
       if (callProps.handleFailure) {
@@ -654,9 +708,16 @@ class RequestHandler {
 
       // !! using Generic makes it easier on the Adapter Builder (just need to change the path)
       // !! you can also replace with a specific call if that is easier
+      if (callProps.pagination) {
+        metadata.pagination = callProps.pagination;
+        metadata.pagination.responseDatakey = callProps.responseDatakey || '';
+      }
       return this.expandedGenericAdapterRequest(metadata, uriPath, uriMethod, null, callQuery, callBody, callHeaders, (result, error) => {
         // if we received an error or their is no response on the results return an error
         if (error) {
+          if (this.props.stub && error.icode === 'AD.301') {
+            return callback(null, error);
+          }
           if (handleFail === 'fail') {
             return callback(null, error);
           }
@@ -690,7 +751,8 @@ class RequestHandler {
             const thisDevice = myResult.response[a];
             for (let rf = 0; rf < rfKeys.length; rf += 1) {
               if (rfKeys[rf] !== 'ostypePrefix') {
-                let fieldValue = getDataFromSources(callProps.responseFields[rfKeys[rf]], [thisDevice, devResp, callProps.requestFields]);
+                // devResp removed due to conditional issues
+                let fieldValue = setResponseDataFromSources(callProps.responseFields[rfKeys[rf]], [thisDevice, callProps.requestFields]);
 
                 // if the field is ostype - need to add prefix
                 if (rfKeys[rf] === 'ostype' && typeof fieldValue === 'string') {
@@ -740,8 +802,7 @@ class RequestHandler {
         for (let rf = 0; rf < rfKeys.length; rf += 1) {
           // skip ostypePrefix since it is not a field
           if (rfKeys[rf] !== 'ostypePrefix') {
-            let fieldValue = getDataFromSources(callProps.responseFields[rfKeys[rf]], [thisDevice, devResp, callProps.requestFields]);
-
+            let fieldValue = setResponseDataFromSources(callProps.responseFields[rfKeys[rf]], [thisDevice, devResp, callProps.requestFields]);
             // if the field is ostype - need to add prefix
             if (rfKeys[rf] === 'ostype' && typeof fieldValue === 'string') {
               fieldValue = ostypePrefix + fieldValue;
@@ -781,6 +842,7 @@ class RequestHandler {
       });
     } catch (e) {
       const errorObj = this.formatErrorObject(this.myid, meth, 'Caught Exception', null, null, null, e);
+      log.debug(`actual error: ${JSON.stringify(e)}`);
       log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
       return callback(null, errorObj);
     }

package/lib/restHandler.js

@@ -1191,7 +1191,9 @@ class RestHandler {
       const bodyString = buildPayload(entity, action, entitySchema, payload);
 
       if ((callMeth !== 'GET' || entitySchema.sendGetBody) && bodyString !== '{}') {
-        thisAHdata['Content-length'] = Buffer.byteLength(bodyString);
+        if (!thisAHdata['Content-length'] && !thisAHdata['Content-Length']) {
+          thisAHdata['Content-length'] = Buffer.byteLength(bodyString);
+        }
       }
 
       // set up the request to be sent
@@ -1319,7 +1321,9 @@ class RestHandler {
         const bodyString = buildPayload('.system', 'healthcheck', healthSchema, payload);
 
         if ((callMeth !== 'GET' || healthSchema.sendGetBody) && bodyString !== '{}') {
-          thisAHdata['Content-length'] = Buffer.byteLength(bodyString);
+          if (!thisAHdata['Content-length'] && !thisAHdata['Content-Length']) {
+            thisAHdata['Content-length'] = Buffer.byteLength(bodyString);
+          }
         }
 
         // set up the request to be sent

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@itentialopensource/adapter-utils",
-  "version": "5.1.6",
+  "version": "5.2.0",
   "description": "Itential Adapter Utility Libraries",
   "scripts": {
     "postinstall": "node utils/setup.js",

package/schemas/propertiesSchema.json

@@ -1046,6 +1046,38 @@
                 "description": "The method of the call to getDevicesFiltered",
                 "default": "GET"
               },
+              "pagination": {
+                "type": "object",
+                "description": "todo",
+                "properties": {
+                  "offsetVar": {
+                    "type": "string",
+                    "description": "Name of variable that defines how to go to next set of results"
+                  },
+                  "limitVar": {
+                    "type": "string",
+                    "description": "Name of variable that defines the max results returned in a request"
+                  },
+                  "incrementBy": {
+                    "type": "string",
+                    "enum": [
+                      "limit",
+                      "page"
+                    ],
+                    "description": "How to incremenet offset. Default limit",
+                    "default": "limit"
+                  },
+                  "requestLocation": {
+                    "type": "string",
+                    "enum": [
+                      "query",
+                      "body"
+                    ],
+                    "description": "Where in request the pagination data goes",
+                    "default": "query"
+                  }
+                }
+              },
               "query": {
                 "type": "object",
                 "description": "The json object with query parameters of the call to getDevicesFiltered",
@@ -1402,31 +1434,63 @@
                   "properties": {
                     "path": {
                       "type": "string",
-                      "description": "The fully qualified path of the call to getDevice (e.g. /rest/api/device/{deviceid})",
+                      "description": "The fully qualified path of the call to populate the cache (e.g. /rest/api/devices)",
                       "default": ""
                     },
                     "method": {
                       "type": "string",
-                      "description": "The method of the call to getDevice",
+                      "description": "The method of the call to populate the cache",
                       "default": "GET"
                     },
+                    "pagination": {
+                      "type": "object",
+                      "description": "todo",
+                      "properties": {
+                        "offsetVar": {
+                          "type": "string",
+                          "description": "Name of variable that defines how to go to next set of results"
+                        },
+                        "limitVar": {
+                          "type": "string",
+                          "description": "Name of variable that defines the max results returned in a request"
+                        },
+                        "incrementBy": {
+                          "type": "string",
+                          "enum": [
+                            "limit",
+                            "page"
+                          ],
+                          "description": "How to incremenet offset. Default limit",
+                          "default": "limit"
+                        },
+                        "requestLocation": {
+                          "type": "string",
+                          "enum": [
+                            "query",
+                            "body"
+                          ],
+                          "description": "Where in request the pagination data goes",
+                          "default": "query"
+                        }
+                      }
+                    },
                     "query": {
                       "type": "object",
-                      "description": "The json object with query parameters of the call to getDevice",
+                      "description": "The json object with query parameters of the call to populate the cache",
                       "additionalProperties": {
                         "type": ["string", "number"]
                       }
                     },
                     "body": {
                       "type": "object",
-                      "description": "The json object with body of the call to getDevice",
+                      "description": "The json object with body of the call to populate the cache",
                       "additionalProperties": {
                         "type": ["string", "number"]
                       }
                     },
                     "headers": {
                       "type": "object",
-                      "description": "The json object with headers of the call to getDevice",
+                      "description": "The json object with headers of the call to populate the cache",
                       "additionalProperties": {
                         "type": ["string", "number"]
                       }
@@ -1442,7 +1506,7 @@
                     },
                     "requestFields": {
                       "type": "object",
-                      "description": "The json object with response fields of the call to getDevice",
+                      "description": "The json object with response fields of the call to populate the cache",
                       "additionalProperties": {
                         "type": ["string", "number"]
                       },
@@ -1455,7 +1519,7 @@
                     },
                     "responseFields": {
                       "type": "object",
-                      "description": "The json object with response fields of the call to getDevice",
+                      "description": "The json object with response fields of the call to populate the cache",
                       "additionalProperties": {
                         "type": ["string", "number"]
                       }