@natlibfi/melinda-rest-api-commons 4.2.3-alpha.1 → 4.2.4-alpha.11

package/dist/mongo.js

@@ -13,6 +13,7 @@ var _utils = require("./utils.js");
 var _moment = _interopRequireDefault(require("moment"));
 var _httpStatus = _interopRequireDefault(require("http-status"));
 var _mongoSanitize = _interopRequireDefault(require("mongo-sanitize"));
+var _debug = _interopRequireDefault(require("debug"));
 /* eslint-disable max-lines */
 
 //import isDeepStrictEqual from 'util';
@@ -46,25 +47,28 @@ var _mongoSanitize = _interopRequireDefault(require("mongo-sanitize"));
 }
 */
 
-async function _default(MONGO_URI, collection) {
+async function _default(MONGO_URI, collection, db = 'rest-api') {
   const logger = (0, _melindaBackendCommons.createLogger)();
+  const debug = (0, _debug.default)('@natlibfi/melinda-rest-api-commons/mongo');
+  const debugDev = debug.extend(':dev');
 
   // Connect to mongo (MONGO)
-  const client = await _mongodb.MongoClient.connect(MONGO_URI, {
-    useNewUrlParser: true,
-    useUnifiedTopology: true
-  });
-  const db = client.db('rest-api');
-  const gridFSBucket = new _mongodb.GridFSBucket(db, {
+  const client = await _mongodb.MongoClient.connect(MONGO_URI);
+  const dbConnection = client.db(db);
+  const gridFSBucket = new _mongodb.GridFSBucket(dbConnection, {
     bucketName: collection
   });
+  const operator = dbConnection.collection(collection);
+  debugDev(`mongo: ${MONGO_URI}, db: ${db}, collection: ${collection}`);
   return {
     createPrio,
     createBulk,
     checkAndSetState,
     checkAndSetImportJobState,
     query,
+    createProjection,
     queryById,
+    checkTimeOut,
     remove,
     readContent,
     removeContent,
@@ -86,6 +90,7 @@ async function _default(MONGO_URI, collection) {
     operation,
     operationSettings
   }) {
+    debugDev('createPrio');
     const time = (0, _moment.default)().toDate();
     const newQueueItem = {
       correlationId,
@@ -111,13 +116,17 @@ async function _default(MONGO_URI, collection) {
       modificationTime: time
     };
     try {
-      const result = await db.collection(collection).insertOne(newQueueItem);
+      debugDev(`Inserting: ${JSON.stringify(newQueueItem)} to ${collection}}`);
+      const result = await operator.insertOne(newQueueItem);
+      debugDev(JSON.stringify(result));
       if (result.acknowledged) {
         logger.info(`New PRIO queue item for ${operation} ${correlationId} has been made in ${collection}`);
         return;
       }
+      // TEST-COVERAGE: this error is not (currently) tested
       throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR);
     } catch (error) {
+      // TEST-COVERAGE: this error is not (currently) tested
       const errorMessage = error.payload || error.message || '';
       (0, _utils.logError)(error);
       throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);
@@ -164,16 +173,17 @@ async function _default(MONGO_URI, collection) {
 
     // eslint-disable-next-line functional/no-conditional-statements
     if (stream) {
+      debug(`Creating streamBulk`);
       try {
-        // No await here, promises later
-        db.collection(collection).insertOne(newQueueItem);
-        logger.info(`New BULK queue item for ${operation} ${correlationId} has been made in ${collection}!`);
-        return new Promise((resolve, reject) => {
+        await new Promise((resolve, reject) => {
           const outputStream = gridFSBucket.openUploadStream(correlationId);
           stream.on('error', reject).on('data', chunk => outputStream.write(chunk)).on('end', () => outputStream.end(undefined, undefined, () => {
             resolve(correlationId);
           }));
         });
+        logger.info(`New BULK queue item for ${operation} ${correlationId} has been made in ${collection}!`);
+        debug(`New BULK queue item for ${operation} ${correlationId} has been made in ${collection}!`);
+        return operator.insertOne(newQueueItem);
       } catch (error) {
         const errorMessage = error.payload || error.message || '';
         (0, _utils.logError)(error);
@@ -181,19 +191,23 @@ async function _default(MONGO_URI, collection) {
       }
     }
     logger.debug(`No stream`);
+    debug(`Creating noStreamBulk`);
     // eslint-disable-next-line functional/no-conditional-statements
     if (!stream) {
       try {
-        const result = await db.collection(collection).insertOne(newQueueItem);
+        const result = await operator.insertOne(newQueueItem);
         if (result.acknowledged) {
           logger.info(`New noStream BULK queue item for ${operation} ${correlationId} has been made in ${collection}`);
+          debug(`New noStream BULK queue item for ${operation} ${correlationId} has been made in ${collection}`);
           return {
             correlationId,
             queueItemState: _constants.QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS
           };
         }
+        // TEST-COVERAGE: this error is not (currently) tested
         throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR);
       } catch (error) {
+        // TEST-COVERAGE: this error is not (currently) tested
         const errorMessage = error.payload || error.message || '';
         (0, _utils.logError)(error);
         throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);
@@ -202,14 +216,15 @@ async function _default(MONGO_URI, collection) {
   }
 
   // Check state that the queueItem has not waited too long and set state
+  // Optionally set also queueItems errorStatus & errorMessage
   async function checkAndSetState({
     correlationId,
     state,
     errorMessage = undefined,
     errorStatus = undefined
   }) {
-    // checkTimeOut returns true, if queueItem is fresher than 1 minute and it's state is not ABORT/ERROR
-    // otherwise it sets queueItem to state ABORT (408, 'Timeout')
+    // checkTimeOut returns true, if queueItem is fresher than 1 minute and it's state is not ABORT/ERROR/DONE
+    // otherwise it sets queueItem and importJobStates to state ABORT and add errorStatus and errorMessage (408, 'Timeout')
     const timeOut = await checkTimeOut({
       correlationId
     });
@@ -224,16 +239,15 @@ async function _default(MONGO_URI, collection) {
     return false;
   }
 
-  // Check state that the queueItem has not waited too long and set state
+  // Check state that the queueItem has not waited too long and set importJobState
+  // Note: checkAndSetImportJobState cannot be used to set queueItems errorStatus & errorMessage (expect in case of a TimeOut)
   async function checkAndSetImportJobState({
     correlationId,
     operation,
-    importJobState,
-    errorMessage = '',
-    errorStatus = ''
+    importJobState
   }) {
-    // checkTimeOut returns true, if queueItem is fresher than 1 minute and it's state is not ABORT/ERROR
-    // otherwise it sets queueItem to state ABORT (408, 'Timeout')
+    // checkTimeOut returns true, if queueItem is fresher than 1 minute and it's state is not ABORT/ERROR/DONE
+    // otherwise it sets queueItem and importJobStates to state ABORT and errorStatue and errorMessage (408, 'Timeout')
     logger.debug(`${correlationId}, ${importJobState}, ${operation}`);
     const timeOut = await checkTimeOut({
       correlationId,
@@ -244,25 +258,25 @@ async function _default(MONGO_URI, collection) {
       return setImportJobState({
         correlationId,
         operation,
-        importJobState,
-        errorMessage,
-        errorStatus
+        importJobState
       });
     }
     return false;
   }
   async function query(params, showParams = {}) {
     logger.debug(`Querying: ${JSON.stringify(params)}, ${JSON.stringify(showParams)}`);
+    debug(`Querying: ${JSON.stringify(params)}, ${JSON.stringify(showParams)}`);
     const {
       limit = 1000,
       skip = 0,
       ...rest
     } = params;
-    const result = await db.collection(collection).find(rest, {
+    const result = await operator.find(rest, {
       projection: createProjection(showParams)
     }).limit(parseInt(limit, 10)).skip(parseInt(skip, 10)).toArray();
     logger.debug(`Query result: ${result.length > 0 ? 'Found!' : 'Not found!'}`);
     logger.silly(`${JSON.stringify(result)}`);
+    debug(`${JSON.stringify(result)}`);
     return result;
   }
 
@@ -272,6 +286,7 @@ async function _default(MONGO_URI, collection) {
 
   function createProjection(showParams = {}) {
     logger.silly(`Creating projection for query: ${JSON.stringify(showParams)}`);
+    debug(`Creating projection for query: ${JSON.stringify(showParams)}`);
     const {
       showAll = 0,
       showOperations = 0,
@@ -279,8 +294,21 @@ async function _default(MONGO_URI, collection) {
       showRecordLoadParams = 0,
       showImportJobState = 0
     } = showParams;
+    const combinedShowParams = {
+      showAll,
+      showOperations,
+      showOperationSettings,
+      showRecordLoadParams,
+      showImportJobState
+    };
     logger.debug(`showAll: ${showAll}, showOperations: ${showOperations}, showOperationSettings: ${showOperationSettings}, showRecordLoadParams: ${showRecordLoadParams}, showImportJobState: ${showImportJobState}`);
-    if (showAll) {
+    debug(`showAll: ${showAll}, showOperations: ${showOperations}, showOperationSettings: ${showOperationSettings}, showRecordLoadParams: ${showRecordLoadParams}, showImportJobState: ${showImportJobState}`);
+    debug(`${JSON.stringify(combinedShowParams)}`);
+
+    // NOTE: parseBoolean parses any non-empty, non-"false" string as true!
+    if ((0, _melindaCommons.parseBoolean)(showAll)) {
+      //if (showAll === true || showAll === '1' || showAll === 1) {
+      debug(`ShowAll: ${showAll}`);
       return {
         _id: 0
       };
@@ -291,52 +319,61 @@ async function _default(MONGO_URI, collection) {
       'showRecordLoadParams': 'recordLoadParams',
       'showImportJobState': 'importJobState'
     };
-    const result = Object.keys(showParams).filter(param => param !== 'showAll' && showParams[param] !== true && showParams[param] !== 1).filter(param => showParamToField[param]).map(param => showParamToField[param]);
+    const result = Object.keys(combinedShowParams)
+    //.filter(param => param !== 'showAll' && combinedShowParams[param] !== true && combinedShowParams[param] !== 1 && combinedShowParams[param] !== '1')
+    .filter(param => param !== 'showAll' && !(0, _melindaCommons.parseBoolean)(combinedShowParams[param])).filter(param => showParamToField[param]).map(param => showParamToField[param]);
     logger.silly(`We want to exclude from projection: ${JSON.stringify(result)}`);
+    debug(`We want to exclude from projection: ${JSON.stringify(result)}`);
     const excludeObject = Object.fromEntries(result.map(param => [param, 0]));
     logger.silly(`We want to exclude from projection: ${JSON.stringify(excludeObject)}`);
+    debug(`We want to exclude from projection: ${JSON.stringify(excludeObject)}`);
     return {
       _id: 0,
       ...excludeObject
     };
   }
+
+  // Note: edited in v4.2.4 to return queueItem *after* possible timout has aborted it
+  // Note: previous versions returned queueItem *before* its state/queueItemState is timeOuted
   async function queryById({
     correlationId,
     checkModTime = false
   }) {
-    const result = await db.collection(collection).findOne({
-      correlationId
-    });
     if (checkModTime) {
-      const timeOut = await checkTimeOut({
+      debug(`queryById checkModTime: ${checkModTime}`);
+      await checkTimeOut({
+        correlationId
+      });
+      return queryById({
         correlationId
       });
-      if (timeOut) {
-        return queryById({
-          correlationId
-        });
-      }
-      return result;
     }
+    const result = await operator.findOne({
+      correlationId
+    }, {
+      projection: {
+        _id: 0
+      }
+    });
     return result;
   }
 
   // Check that if the item has waited too long
   // If the last modification time for the queueItem is older than 1 minute
   // set state to ABORT and return false, otherwise return true
-  // If the state is already ABORT or ERROR return false
+  // If the state is already ABORT or ERROR or DONE return false
 
   async function checkTimeOut({
     correlationId
   }) {
     const {
       modificationTime,
-      queueitemState: oldState,
+      queueItemState: oldState,
       importJobState
-    } = await db.collection(collection).findOne({
+    } = await operator.findOne({
       correlationId
     });
-
+    //debug(`${modificationTime} - oldState: ${oldState}`);
     // should we check for DONE too?
     if ([_constants.QUEUE_ITEM_STATE.ABORT, _constants.QUEUE_ITEM_STATE.ERROR, _constants.QUEUE_ITEM_STATE.DONE].includes(oldState)) {
       logger.silly(`${correlationId} has already state: ${oldState}`);
@@ -380,26 +417,38 @@ async function _default(MONGO_URI, collection) {
     }
     return true;
   }
+
+  // eslint-disable-next-line max-statements
   async function remove(params) {
     logger.silly(`${JSON.stringify(params)}`);
     logger.info(`Removing from Mongo (${collection}) id: ${params.correlationId}`);
+    debug(`${JSON.stringify(params)}`);
+    debug(`Removing from Mongo (${collection}) id: ${params.correlationId}`);
     const clean = (0, _mongoSanitize.default)(params.correlationId);
     logger.silly(`mongo/remove: clean: ${JSON.stringify(clean)}`);
+    debug(`mongo/remove: clean: ${JSON.stringify(clean)}`);
     try {
       //const metadataResult = await getFileMetadata({filename: clean});
       //logger.debug(`mongo/remove: metadataResult: ${JSON.stringify(metadataResult)}`);
-      const noContent = await removeContent(params);
+      const noContent = await removeContent({
+        correlationId: params.correlationId
+      });
+      debug(`noContent (removeContent result): ${JSON.stringify(noContent)}`);
       if (noContent) {
-        await db.collection(collection).deleteOne({
+        const deleteResult = await operator.deleteOne({
           correlationId: clean
         });
+        debug(`deleteResult: ${JSON.stringify(deleteResult)}`);
         return true;
       }
     } catch (err) {
+      // TEST-COVERAGE: this error is not (currently) tested
       if (err instanceof _mongodb.MongoDriverError) {
+        debug(err);
         if (err.message.indexOf('File not found for id') !== -1) {
           logger.silly(`mongo/remove: File not found, removing queueItem ${JSON.stringify(clean)} from ${collection}`);
-          await db.collection(collection).deleteOne({
+          debug(`mongo/remove: File not found, removing queueItem ${JSON.stringify(clean)} from ${collection}`);
+          await operator.deleteOne({
             correlationId: clean
           });
           return true;
@@ -412,11 +461,13 @@ async function _default(MONGO_URI, collection) {
   }
   async function readContent(correlationId) {
     logger.info(`Reading content from mongo for id: ${correlationId} in ${collection}`);
+    debug(`Reading content from mongo for id: ${correlationId} in ${collection}`);
     const clean = (0, _mongoSanitize.default)(correlationId);
-    const result = await db.collection(collection).findOne({
+    debug(clean);
+    const result = await operator.findOne({
       correlationId: clean
     }); // njsscan-ignore: node_nosqli_injection
-
+    debug(result);
     if (result) {
       const {
         operationSettings
@@ -428,6 +479,8 @@ async function _default(MONGO_URI, collection) {
         };
       }
       logger.debug(`OperationSettings for ${correlationId}: ${JSON.stringify(operationSettings)}`);
+      debug(`OperationSettings for ${correlationId}: ${JSON.stringify(operationSettings)}`);
+      debug(`Content is only available for streamBulk jobs. ${correlationId} is not a streamBulk job.`);
       throw new _melindaCommons.Error(_httpStatus.default.BAD_REQUEST, {
         message: `Content is only available for streamBulk jobs. ${correlationId} is not a streamBulk job.`
       });
@@ -438,51 +491,64 @@ async function _default(MONGO_URI, collection) {
   }
   async function removeContent(params) {
     logger.info(`Removing content from mongo for id: ${params.correlationId} in ${collection}`);
+    debug(`Removing content from mongo for id: ${params.correlationId} in ${collection}`);
     const clean = (0, _mongoSanitize.default)(params.correlationId);
-    const result = await db.collection(collection).findOne({
-      correlationId: clean
-    }); // njsscan-ignore: node_nosqli_injection
-    logger.silly(`mongo/removeContent: result ${JSON.stringify(result)}`);
-    if (result) {
-      await gridFSBucket.delete(clean);
+    const result = await gridFSBucket.find({
+      filename: clean
+    }).toArray();
+    debug(`mongo/removeContent: find bucket result ${JSON.stringify(result)}`);
+    if (result.length > 0) {
+      // Note: we assume and handle just one file for filename!
+      const [firstBucketResult] = result;
+      const fileId = firstBucketResult._id;
+      await gridFSBucket.delete(fileId);
       return true;
     }
+    logger.debug(`No files for ${params.correlationId} to delete`);
     return true;
   }
-
-  // eslint-disable-next-line max-statements
   function getOne({
     queueItemState,
     importJobState = undefined
   }) {
+    //params example: "queueItemState": "IMPORTER.IN_QUEUE", "importJobState":{"importJobState.UPDATE": "IN_QUEUE"}
     logger.silly(`queueItemState: ${queueItemState}, importJobState: ${JSON.stringify(importJobState)}`);
+    //debug(`queueItemState: ${queueItemState}, importJobState: ${JSON.stringify(importJobState)}`);
     const cleanQueueItemState = queueItemState ? {
       queueItemState: (0, _mongoSanitize.default)(queueItemState)
     } : undefined;
+    const options = {
+      projection: {
+        _id: 0
+      }
+    };
     try {
       // Just queueItemState
       if (queueItemState && importJobState === undefined) {
         logger.silly(`Checking DB ${collection} for just ${JSON.stringify(cleanQueueItemState.queueItemState)}`);
-        return db.collection(collection).findOne({
+        //debug(`Checking DB ${collection} for just ${JSON.stringify(cleanQueueItemState.queueItemState)}`);
+        return operator.findOne({
           ...cleanQueueItemState
-        });
+        }, options);
       }
 
       // importJobState
       if (importJobState && queueItemState === undefined) {
         logger.silly(`Checking DB ${collection} for ${JSON.stringify(importJobState)}`);
-        return db.collection(collection).findOne({
+        //debug(`Checking DB ${collection} for ${JSON.stringify(importJobState)}`);
+        return operator.findOne({
           ...importJobState
-        });
+        }, options);
       }
 
       // importJobState and queueItemState
       if (importJobState && queueItemState) {
         logger.silly(`Checking DB ${collection} for ${queueItemState} and ${JSON.stringify(importJobState)}`);
-        return db.collection(collection).findOne({
+        //debug(`Checking DB ${collection} for ${queueItemState} and ${JSON.stringify(importJobState)}`);
+        return operator.findOne({
           ...cleanQueueItemState,
           ...importJobState
-        });
+        }, options);
       }
       logger.debug(`getOne not working!`);
       throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR, 'Missing parameters, cannot get queueItem');
@@ -499,6 +565,7 @@ async function _default(MONGO_URI, collection) {
       // Return content stream
       return gridFSBucket.openDownloadStreamByName(clean);
     } catch (error) {
+      // TEST-COVERAGE: this error is not (currently) tested
       const errorMessage = error.payload || error.message || '';
       (0, _utils.logError)(error);
       throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);
@@ -512,7 +579,7 @@ async function _default(MONGO_URI, collection) {
     logger.verbose(`Push ids (${handledIds.length}) and rejectedIds (${rejectedIds.length}) ${correlationId} to ${collection}`);
     logger.debug(`ids (${handledIds.length}): ${JSON.stringify(handledIds)}, rejectedIds ${rejectedIds.length}: ${JSON.stringify(rejectedIds)}`);
     const clean = (0, _mongoSanitize.default)(correlationId);
-    await db.collection(collection).updateOne({
+    await operator.updateOne({
       correlationId: clean
     }, {
       $set: {
@@ -537,7 +604,8 @@ async function _default(MONGO_URI, collection) {
     logger.silly(`Messages (${messages.length}): ${JSON.stringify(messages)}}`);
     const clean = (0, _mongoSanitize.default)(correlationId);
     const cleanMessageField = (0, _mongoSanitize.default)(messageField);
-    await db.collection(collection).updateOne({
+    debug(`${messageField} -> ${cleanMessageField}`);
+    await operator.updateOne({
       correlationId: clean
     }, {
       $set: {
@@ -562,6 +630,7 @@ async function _default(MONGO_URI, collection) {
     const stateInQueueItemStatesValidator = Object.values(_constants.QUEUE_ITEM_STATE.VALIDATOR).indexOf(state) > -1;
     const stateInQueueItemStatesImporter = Object.values(_constants.QUEUE_ITEM_STATE.IMPORTER).indexOf(state) > -1;
     if (!stateInQueueItemStates && !stateInQueueItemStatesValidator && !stateInQueueItemStatesImporter) {
+      debug(`Trying to set invalid state ${state}`);
       throw new _melindaCommons.Error(_httpStatus.default.BAD_REQUEST, 'Trying to set invalid state');
     }
     const clean = (0, _mongoSanitize.default)(correlationId);
@@ -575,7 +644,7 @@ async function _default(MONGO_URI, collection) {
     // Do not update value that are undefined
     // eslint-disable-next-line functional/immutable-data
     Object.keys(updateValues).forEach(key => updateValues[key] === undefined && delete updateValues[key]);
-    return db.collection(collection).findOneAndUpdate({
+    return operator.findOneAndUpdate({
       correlationId: clean
     }, {
       $set: updateValues
@@ -583,7 +652,7 @@ async function _default(MONGO_URI, collection) {
       projection: {
         _id: 0
       },
-      returnNewDocument: true
+      returnDocument: 'after'
     });
   }
   function setImportJobState({
@@ -593,13 +662,16 @@ async function _default(MONGO_URI, collection) {
   }) {
     // should this also get importJobState as previously created object? {}
     logger.info(`Setting queue-item importJobState: {${operation}: ${importJobState}} for ${correlationId}`);
+    debug(`Setting queue-item importJobState: {${operation}: ${importJobState}} for ${correlationId}`);
     const cleanCorrelationId = (0, _mongoSanitize.default)(correlationId);
     const cleanImportJobState = (0, _mongoSanitize.default)(importJobState);
     if (!(operation in _constants.OPERATIONS)) {
-      throw new _melindaCommons.Error('400', 'Invalid operation for import job state');
+      debug(`Error: ${operation} is not in ${JSON.stringify(_constants.OPERATIONS)}`);
+      throw new _melindaCommons.Error(_httpStatus.default.BAD_REQUEST, 'Invalid operation for import job state');
     }
     if (!(cleanImportJobState in _constants.IMPORT_JOB_STATE)) {
-      throw new _melindaCommons.Error('400', 'Invalid import job state');
+      debug(`Error: ${cleanImportJobState} is not in ${JSON.stringify(_constants.IMPORT_JOB_STATE)}`);
+      throw new _melindaCommons.Error(_httpStatus.default.BAD_REQUEST, 'Invalid import job state');
     }
     const newJobState = getNewJobState(operation, cleanImportJobState);
     function getNewJobState(operation, cleanImportJobState) {
@@ -618,9 +690,11 @@ async function _default(MONGO_URI, collection) {
           'importJobState.FIX': cleanImportJobState
         };
       }
+      // istanbul ignore next
+      // Ignoring for coverage - we error this already
       throw new _melindaCommons.Error('400', 'Invalid operation for import job state');
     }
-    return db.collection(collection).findOneAndUpdate({
+    return operator.findOneAndUpdate({
       correlationId: cleanCorrelationId
     }, {
       $set: {
@@ -631,7 +705,7 @@ async function _default(MONGO_URI, collection) {
       projection: {
         _id: 0
       },
-      returnNewDocument: true
+      returnDocument: 'after'
     });
   }
   async function setOperation({
@@ -641,13 +715,13 @@ async function _default(MONGO_URI, collection) {
     const newOperation = operation;
     const {
       operation: oldOperation
-    } = await db.collection(collection).findOne({
+    } = await operator.findOne({
       correlationId
     });
     logger.info(`Setting queue-item operation from ${oldOperation} to ${newOperation} for ${correlationId} to ${collection}`);
     const cleanCorrelationId = (0, _mongoSanitize.default)(correlationId);
     const cleanNewOperation = (0, _mongoSanitize.default)(newOperation);
-    const result = await db.collection(collection).findOneAndUpdate({
+    const result = await operator.findOneAndUpdate({
       correlationId: cleanCorrelationId
     }, {
       $set: {
@@ -658,9 +732,10 @@ async function _default(MONGO_URI, collection) {
       projection: {
         _id: 0
       },
-      returnNewDocument: true
+      returnDocument: 'after'
     });
-    return result.ok;
+    debugDev(`${JSON.stringify(result)}`);
+    return result.ok || result;
     // logger.debug(JSON.stringify(result));
   }
   async function setOperations({
@@ -669,7 +744,7 @@ async function _default(MONGO_URI, collection) {
     removeOperation = undefined
   }) {
     const cleanCorrelationId = (0, _mongoSanitize.default)(correlationId);
-    const queueItem = await db.collection(collection).findOne({
+    const queueItem = await operator.findOne({
       correlationId: cleanCorrelationId
     });
     logger.silly(`We found a queueItem: ${JSON.stringify(queueItem)}`);
@@ -679,7 +754,7 @@ async function _default(MONGO_URI, collection) {
     const operationsAfterRemoveAndAdd = operationsAfterRemove.includes(addOperation) ? operationsAfterRemove : [...operationsAfterRemove, addOperation];
     logger.silly(`operationsAfterRemove: ${operationsAfterRemove}`);
     logger.silly(`operationAfterRemoveAndAdd: ${operationsAfterRemoveAndAdd}`);
-    const result = await db.collection(collection).findOneAndUpdate({
+    const result = await operator.findOneAndUpdate({
       correlationId: cleanCorrelationId
     }, {
       $set: {
@@ -690,16 +765,17 @@ async function _default(MONGO_URI, collection) {
       projection: {
         _id: 0
       },
-      returnNewDocument: true
+      returnDocument: 'after'
     });
-    return result.ok;
+    debugDev(`${JSON.stringify(result)}`);
+    return result.ok || result;
     // logger.debug(JSON.stringify(result));
   }
   async function addBlobSize({
     correlationId
   }) {
     const cleanCorrelationId = (0, _mongoSanitize.default)(correlationId);
-    const result = await db.collection(collection).findOneAndUpdate({
+    const result = await operator.findOneAndUpdate({
       correlationId: cleanCorrelationId,
       queueItemState: _constants.QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS
     }, {
@@ -713,9 +789,10 @@ async function _default(MONGO_URI, collection) {
       projection: {
         _id: 0
       },
-      returnNewDocument: true
+      returnDocument: 'after'
     });
     logger.verbose(`AddBlobSizeResult in mongo: ${JSON.stringify(result)}`);
+    debugDev(`${JSON.stringify(result)}`);
     return result;
   }
   async function setBlobSize({
@@ -724,7 +801,7 @@ async function _default(MONGO_URI, collection) {
   }) {
     const cleanCorrelationId = (0, _mongoSanitize.default)(correlationId);
     const cleanBlobSize = (0, _mongoSanitize.default)(blobSize);
-    const result = await db.collection(collection).findOneAndUpdate({
+    const result = await operator.findOneAndUpdate({
       correlationId: cleanCorrelationId
     }, {
       $set: {
@@ -735,9 +812,10 @@ async function _default(MONGO_URI, collection) {
       projection: {
         _id: 0
       },
-      returnNewDocument: true
+      returnDocument: 'after'
     });
     logger.verbose(`SetBlobSizeResult in mongo: ${JSON.stringify(result)}`);
+    debugDev(`${JSON.stringify(result)}`);
     return result;
   }
 }