@itentialopensource/adapter-utils 5.10.21 → 5.10.23

package/lib/connectorRest.js

@@ -789,7 +789,9 @@ function makeRequest(request, entitySchema, callProperties, startTrip, attempt,
           healthy = true;
           doneH2(true);
         }, (retH2) => {
-          log.debug(`${origin}: CALL RETURN ${JSON.stringify(propUtilInst.scrubSensitiveInfo(callResp, addSensitiveItems))}`);
+          // COMMENTING OUT THE COMPLETE LOG until we figure out how to handle scrubbing
+          // log.debug(`${origin}: CALL RETURN ${JSON.stringify(propUtilInst.scrubSensitiveInfo(callResp, addSensitiveItems))}`);
+          log.debug(`${origin}: CALL RETURNED`);
           useProt = undefined;
           callResp.reqHdr = request.header.headers;
           return callback(callResp);

package/lib/restHandler.js

@@ -343,7 +343,8 @@ function handleRestRequest(request, entityId, entitySchema, callProperties, filt
 
       // if id, log response (if all - too much to log all)
       if (entityId === 'nomap') {
-        log.debug(`${origin}: RESPONSE: ${resObj.response}`);
+        // log.debug(`${origin}: RESPONSE: ${resObj.response}`);
+        log.debug(`${origin}: RESPONSE NO MAP`);
 
         // if no mapping (do not care about response just that we did not
         // error) set response
@@ -353,20 +354,23 @@ function handleRestRequest(request, entityId, entitySchema, callProperties, filt
 
       // if the data is not json we can not perform the extended capabilities so just return it
       if (entitySchema && entitySchema.responseDatatype && entitySchema.responseDatatype.toUpperCase() === 'PLAIN') {
-        log.debug(`${origin}: RESPONSE: ${resObj.response}`);
+        // log.debug(`${origin}: RESPONSE: ${resObj.response}`);
+        log.debug(`${origin}: RESPONSE PLAIN`);
 
         // return the response
         return callback(retObject);
       }
       if (entitySchema && entitySchema.responseDatatype && entitySchema.responseDatatype.toUpperCase() === 'XML') {
-        log.debug(`${origin}: RESPONSE: ${resObj.response}`);
+        // log.debug(`${origin}: RESPONSE: ${resObj.response}`);
+        log.debug(`${origin}: RESPONSE XML`);
 
         // return the response
         return callback(retObject);
       }
 
       if (entitySchema && entitySchema.responseDatatype && entitySchema.responseDatatype.toUpperCase() === 'XML2JSON') {
-        log.debug(`${origin}: RESPONSE: ${resObj.response}`);
+        // log.debug(`${origin}: RESPONSE: ${resObj.response}`);
+        log.debug(`${origin}: RESPONSE XML2JSON`);
 
         try {
           const parser = new xml2js.Parser({ explicitArray: false, attrkey: '_attr' });
@@ -410,12 +414,14 @@ function handleRestRequest(request, entityId, entitySchema, callProperties, filt
           const responseBody = unzipresponse.toString('utf8'); // Convert buffer to string
           if (responseDatatype === 'GZIP2PLAIN') {
             retObject.response = responseBody;
-            log.debug(`${origin}: RESPONSE: ${retObject.response}`);
+            // log.debug(`${origin}: RESPONSE: ${retObject.response}`);
+            log.debug(`${origin}: RESPONSE GZIP2PLAIN`);
             return callback(retObject);
           } if (responseDatatype === 'GZIP2JSON') {
             try {
               retObject.response = JSON.parse(responseBody);
-              log.debug(`${origin}: RESPONSE: ${retObject.response}`);
+              // log.debug(`${origin}: RESPONSE: ${retObject.response}`);
+              log.debug(`${origin}: RESPONSE GZIP2JSON`);
               return callback(retObject); // Parse responseBody as JSON
             } catch (jsonErr) {
               log.warn(`${origin}: Error parsing JSON response: ${jsonErr}`);
@@ -459,9 +465,11 @@ function handleRestRequest(request, entityId, entitySchema, callProperties, filt
 
         // if the response is an array, log the first item
         if (Array.isArray(returnFieldData)) {
-          log.debug(`${origin}: RESPONSE (FIRST): ${JSON.stringify(returnFieldData[0])}`);
+          // log.debug(`${origin}: RESPONSE (FIRST): ${JSON.stringify(returnFieldData[0])}`);
+          log.debug(`${origin}: RESPONSE KEY FOUND`);
         } else {
-          log.debug(`${origin}: RESPONSE: ${JSON.stringify(returnFieldData)}`);
+          // log.debug(`${origin}: RESPONSE: ${JSON.stringify(returnFieldData)}`);
+          log.debug(`${origin}: RESPONSE KEY FOUND`);
         }
 
         // if the data is not an object - just return what it is
@@ -482,9 +490,11 @@ function handleRestRequest(request, entityId, entitySchema, callProperties, filt
 
       if (Array.isArray(retResponse)) {
         // if the response is an array, log the first item
-        log.debug(`${origin}: RESPONSE (FIRST): ${JSON.stringify(retResponse[0])}`);
+        // log.debug(`${origin}: RESPONSE (FIRST): ${JSON.stringify(retResponse[0])}`);
+        log.debug(`${origin}: RESPONSE NO KEY`);
       } else {
-        log.debug(`${origin}: RESPONSE: ${JSON.stringify(retResponse)}`);
+        // log.debug(`${origin}: RESPONSE: ${JSON.stringify(retResponse)}`);
+        log.debug(`${origin}: RESPONSE NO KEY`);
       }
 
       // if the data is not an object - just return what it is

package/lib/throttle.js

@@ -51,7 +51,7 @@ let qlock = null;
  */
 function getQueueItems(dbUI, collectionName, filter, callback) {
   const origin = `${id}-throttle-getQueueItems`;
-  log.spam(origin);
+  log.trace(origin);
 
   try {
     let myFilter = filter;
@@ -164,7 +164,7 @@ function getQueueItems(dbUI, collectionName, filter, callback) {
  */
 function claimTurn(dbUI, collectionName, queueItem, callback) {
   const origin = `${id}-throttle-claimTurn`;
-  log.spam(origin);
+  log.trace(origin);
 
   try {
     const cur = new Date();
@@ -233,7 +233,7 @@ function claimTurn(dbUI, collectionName, queueItem, callback) {
  */
 function freeQueueItem(dbUI, collectionName, queueItem, callback) {
   const origin = `${id}-throttle-freeQueueItem`;
-  log.spam(origin);
+  log.trace(origin);
 
   try {
     // If the number of Pronghorns is greater than 1, the queue is in the database
@@ -277,6 +277,60 @@ function freeQueueItem(dbUI, collectionName, queueItem, callback) {
   }
 }
 
+/*
+ * INTERNAL FUNCTION: freeMultipleQueueItem is used to free the turn from the given request.
+ *    This is done by deleting the items from the database or removing them from the
+ *    queue.
+ */
+function freeMultipleQueueItem(dbUI, collectionName, queueItems, callback) {
+  const origin = `${id}-throttle-freeMultipleQueueItem`;
+  log.trace(origin);
+
+  try {
+    // If the number of Pronghorns is greater than 1, the queue is in the database
+    if (numberPhs > 1) {
+      const idsToDelete = [];
+      for (let i = 0; i < queueItems.length; i += 1) {
+        idsToDelete.push(queueItems[i]._id);
+      }
+
+      // actual call to remove the requests from the queue in the database
+      // return MONGOQ.deleteOne({ _id: queueItem._id }, (error, result) => {
+      return dbUI.deleteMany(collectionName, { _id: { $in: idsToDelete } }, {}, false, null, false, (error, result) => {
+        if (error) {
+          const errorObj = transUtilInst.formatErrorObject(origin, 'Unable To Free Turn', [queueItems[0].request_id, queueItems[0].transNum, error], null, null, null);
+          log.error(`${origin}: ${errorObj.IAPerror.displayString}`);
+          return callback(null, errorObj);
+        }
+        return callback(result);
+      });
+    }
+
+    // If the number of Pronghorns is 1, the queue is in memory
+    // Lock the memQueue while finding and removing the item
+    return qlock.acquire(memQlock, (done) => {
+      for (let i = 0; i < queueItems.length; i += 1) {
+        const index = memQueue.indexOf(queueItems[i]);
+        if (index >= 0) {
+          log.debug(`${origin}: Free Queue Item ${queueItems[i].request_id} - ${queueItems[i].transNum}`);
+          memQueue.splice(index, 1);
+        } else {
+          log.warn(`${origin}: Queue Item Not Found ${queueItems[i].request_id} - ${queueItems[i].transNum}: May have been already removed`);
+        }
+      }
+
+      done('finished removing items from queue');
+    }, (ret) => {
+      log.info(ret);
+      return callback(ret);
+    });
+  } catch (e) {
+    // handle any exception
+    const errorObj = transUtilInst.checkAndReturn(e, origin, 'Issue freeing queue items');
+    return callback(null, errorObj);
+  }
+}
+
 /*
  * INTERNAL FUNCTION: freeExpiredQueueItems goes through the current active usage to
  *    determine if any have finished running long enough ago that the turn has expired.
@@ -287,7 +341,7 @@ function freeQueueItem(dbUI, collectionName, queueItem, callback) {
  */
 function freeExpiredQueueItems(callback) {
   const origin = `${id}-throttle-freeExpiredQueueItems`;
-  log.spam(origin);
+  log.trace(origin);
 
   try {
     const status = 'success';
@@ -310,35 +364,31 @@ function freeExpiredQueueItems(callback) {
         return callback(status);
       }
 
-      let handled = 0;
-
+      const freelist = [];
       // go through the queue items to see if any have expired
       for (let i = 0; i < currentUse.length; i += 1) {
         // if the transaction finished and the end + timeout is less than the current time
         if (currentUse[i].end !== null
             && (Number(currentUse[i].end) + Number(expireTimeout) < cur.getTime())) {
-          freeQueueItem(dbUtilInst, queueColl, currentUse[i], (flic, ferror) => {
-            if (ferror) {
-              return callback(null, ferror);
-            }
-
-            handled += 1;
-            // when we have handled everything - return
-            if (handled === currentUse.length) {
-              log.spam(`${origin}: ${flic}`);
-              return callback(status);
-            }
-          });
-        } else {
-          handled += 1;
-        }
-        // when we have handled everything - return
-        if (handled === currentUse.length) {
-          return callback(status);
+          freelist.push(currentUse[i]);
         }
       }
 
-      return callback(status);
+      // if there is nothing to free just return
+      if (freelist.length === 0) {
+        log.info(`${origin}: Nothing to be freed`);
+        return callback(status);
+      }
+
+      return freeMultipleQueueItem(dbUtilInst, queueColl, freelist, (flic, ferror) => {
+        if (ferror) {
+          log.warn(`Error received on FreeQueueItem: ${JSON.stringify(ferror)}`);
+          return callback(null, ferror);
+        }
+
+        log.info(`${origin}: Expired Queue Item Freed: ${flic}`);
+        return callback(status);
+      });
     });
   } catch (e) {
     // handle any exception
@@ -354,7 +404,7 @@ function freeExpiredQueueItems(callback) {
  */
 function checkTurnAvailable(myRequest, myTransNum, callback) {
   const origin = `${id}-throttle-checkTurnAvailable`;
-  log.spam(origin);
+  log.trace(origin);
 
   try {
     return freeExpiredQueueItems((freeRes, ferror) => {
@@ -362,7 +412,7 @@ function checkTurnAvailable(myRequest, myTransNum, callback) {
         return callback(null, ferror);
       }
 
-      log.spam(`${origin}: ${freeRes}`);
+      log.trace(`${origin}: ${freeRes}`);
 
       // only want to set filter if using DB - if memory want the entire queue so we get all priorities
       let useFilter = null;
@@ -438,7 +488,7 @@ function checkTurnAvailable(myRequest, myTransNum, callback) {
  */
 function gettingCloseInterval(myRequest, transNum, callback) {
   const origin = `${id}-throttle-gettingCloseInterval`;
-  log.spam(origin);
+  log.trace(origin);
 
   try {
     let intRun = false;
@@ -648,7 +698,7 @@ class SystemXThrottle {
 
         // Connection URL
         let murl = prongo.properties.url;
-        log.spam(`${origin}: ${murl}`);
+        log.trace(`${origin}: ${murl}`);
         const dbName = prongo.properties.db;
 
         // define some local variables to help in validating the properties.json file
@@ -1117,7 +1167,7 @@ class SystemXThrottle {
               return callback(null, gerror);
             }
 
-            log.spam(`${origin}: ${waitEnd}`);
+            log.trace(`${origin}: ${waitEnd}`);
             return claimTurn(this.dbUtil, queueColl, queueItem, callback);
           });
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@itentialopensource/adapter-utils",
-  "version": "5.10.21",
+  "version": "5.10.23",
   "description": "Itential Adapter Utility Libraries",
   "scripts": {
     "postinstall": "node utils/setup.js",