@natlibfi/melinda-rest-api-commons 2.2.0-alpha.2 → 2.2.0-alpha.20

package/dist/mongo.js

@@ -50,6 +50,7 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
 * for the JavaScript code in this file.
 *
 */
+//import isDeepStrictEqual from 'util';
 
 /* QueueItem:
 {
@@ -57,6 +58,7 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
   "cataloger":"xxx0000",
   "oCatalogerIn":"xxx0000"
   "operation":"UPDATE",
+  "operations":["UPDATE", "CREATE"]
   "contentType":"application/json",
   "recordLoadParams": {
     "pActiveLibrary": "XXX00",
@@ -66,9 +68,16 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
     "pOldNew": "NEW"
   },
   "queueItemState":"PENDING_QUEUING",
+  "importJobState": {
+    "CREATE": "EMPTY",
+    "UPDATE": "EMPTY"
+  },
+  blobSize: 1,
+  records: [],
+  errorMessage: '',
+  errorStatus: '',
   "creationTime":"2020-01-01T00:00:00.000Z",
   "modificationTime":"2020-01-01T00:00:01.000Z",
-  "handledIds": []
 }
 */
 async function _default(MONGO_URI, collection) {
@@ -86,6 +95,7 @@ async function _default(MONGO_URI, collection) {
     createPrio,
     createBulk,
     checkAndSetState,
+    checkAndSetImportJobState,
     query,
     queryById,
     remove,
@@ -94,8 +104,13 @@ async function _default(MONGO_URI, collection) {
     getOne,
     getStream,
     setState,
+    setImportJobState,
     pushIds,
-    pushMessages
+    pushMessages,
+    setOperation,
+    setOperations,
+    addBlobSize,
+    setBlobSize
   };
 
   async function createPrio({
@@ -103,26 +118,30 @@ async function _default(MONGO_URI, collection) {
     cataloger,
     oCatalogerIn,
     operation,
-    noop = undefined,
-    unique = undefined,
-    prio = true
+    operationSettings
   }) {
     const time = (0, _moment.default)().toDate();
     const newQueueItem = {
       correlationId,
       cataloger,
       operation,
+      operations: [operation],
       oCatalogerIn,
       operationSettings: {
-        unique,
-        noop,
-        prio
+        prio: true,
+        ...operationSettings
       },
       queueItemState: _constants.QUEUE_ITEM_STATE.VALIDATOR.PENDING_VALIDATION,
+      blobSize: 1,
+      importJobState: {
+        CREATE: _constants.IMPORT_JOB_STATE.EMPTY,
+        UPDATE: _constants.IMPORT_JOB_STATE.EMPTY
+      },
+      records: [],
+      errorMessage: '',
+      errorStatus: '',
       creationTime: time,
-      modificationTime: time,
-      handledIds: [],
-      rejectedIds: []
+      modificationTime: time
     };
 
     try {
@@ -140,7 +159,7 @@ async function _default(MONGO_URI, collection) {
     }
   }
 
-  function createBulk({
+  async function createBulk({
     correlationId,
     cataloger,
     oCatalogerIn,
@@ -148,7 +167,7 @@ async function _default(MONGO_URI, collection) {
     contentType,
     recordLoadParams,
     stream,
-    prio = false
+    operationSettings
   }) {
     const time = (0, _moment.default)().toDate();
     const newQueueItem = {
@@ -156,40 +175,63 @@ async function _default(MONGO_URI, collection) {
       cataloger,
       oCatalogerIn,
       operation,
+      operations: [operation],
       operationSettings: {
-        prio
+        prio: false,
+        ...operationSettings
       },
       contentType,
       recordLoadParams,
       queueItemState: stream ? _constants.QUEUE_ITEM_STATE.VALIDATOR.UPLOADING : _constants.QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS,
+      blobSize: 0,
+      importJobState: {
+        CREATE: _constants.IMPORT_JOB_STATE.EMPTY,
+        UPDATE: _constants.IMPORT_JOB_STATE.EMPTY
+      },
+      records: [],
+      errorMessage: '',
+      errorStatus: '',
       creationTime: time,
-      modificationTime: time,
-      handledIds: [],
-      rejectedIds: []
-    };
-
-    try {
-      // No await here, promises later
-      db.collection(collection).insertOne(newQueueItem);
-      logger.info(`New BULK queue item for ${operation} ${correlationId} has been made in ${collection}!`);
-    } catch (error) {
-      (0, _utils.logError)(error);
-      throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR);
+      modificationTime: time
+    }; // eslint-disable-next-line functional/no-conditional-statement
+
+    if (stream) {
+      try {
+        // No await here, promises later
+        db.collection(collection).insertOne(newQueueItem);
+        logger.info(`New BULK queue item for ${operation} ${correlationId} has been made in ${collection}!`);
+        return new Promise((resolve, reject) => {
+          const outputStream = gridFSBucket.openUploadStream(correlationId);
+          stream.on('error', reject).on('data', chunk => outputStream.write(chunk)).on('end', () => outputStream.end(undefined, undefined, () => {
+            resolve(correlationId);
+          }));
+        });
+      } catch (error) {
+        (0, _utils.logError)(error);
+        throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR);
+      }
     }
 
+    logger.debug(`No stream`); // eslint-disable-next-line functional/no-conditional-statement
+
     if (!stream) {
-      return {
-        correlationId,
-        queueItemState: _constants.QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS
-      };
-    }
+      try {
+        const result = await db.collection(collection).insertOne(newQueueItem);
+
+        if (result.acknowledged) {
+          logger.info(`New noStream BULK queue item for ${operation} ${correlationId} has been made in ${collection}`);
+          return {
+            correlationId,
+            queueItemState: _constants.QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS
+          };
+        }
 
-    return new Promise((resolve, reject) => {
-      const outputStream = gridFSBucket.openUploadStream(correlationId);
-      stream.on('error', reject).on('data', chunk => outputStream.write(chunk)).on('end', () => outputStream.end(undefined, undefined, () => {
-        resolve(correlationId);
-      }));
-    });
+        throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR);
+      } catch (error) {
+        (0, _utils.logError)(error);
+        throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR);
+      }
+    }
   } // Check state that the queueItem has not waited too long and set state
 
 
@@ -201,7 +243,9 @@ async function _default(MONGO_URI, collection) {
   }) {
     // checkTimeOut returns true, if queueItem is fresher than 1 minute and it's state is not ABORT/ERROR
     // otherwise it sets queueItem to state ABORT (408, 'Timeout')
-    const timeOut = await checkTimeOut(correlationId);
+    const timeOut = await checkTimeOut({
+      correlationId
+    });
 
     if (timeOut) {
       return setState({
@@ -212,29 +256,81 @@ async function _default(MONGO_URI, collection) {
       });
     }
 
+    return false;
+  } // Check state that the queueItem has not waited too long and set state
+
+
+  async function checkAndSetImportJobState({
+    correlationId,
+    operation,
+    importJobState,
+    errorMessage = '',
+    errorStatus = ''
+  }) {
+    // checkTimeOut returns true, if queueItem is fresher than 1 minute and it's state is not ABORT/ERROR
+    // otherwise it sets queueItem to state ABORT (408, 'Timeout')
+    logger.debug(`${correlationId}, ${importJobState}, ${operation}`);
+    const timeOut = await checkTimeOut({
+      correlationId,
+      operation,
+      importJobState
+    });
+
+    if (timeOut) {
+      return setImportJobState({
+        correlationId,
+        operation,
+        importJobState,
+        errorMessage,
+        errorStatus
+      });
+    }
+
     return false;
   }
 
-  async function query(params) {
-    const result = await db.collection(collection).find(params, {
+  async function query(params, showParams = {}) {
+    const {
+      showOperations = 0,
+      showOperationSettings = 0,
+      showRecordLoadParams = 0,
+      showImportJobState = 0
+    } = showParams;
+    const {
+      limit = 1000,
+      skip = 0,
+      ...rest
+    } = params;
+    const result = await db.collection(collection).find(rest, {
       projection: {
-        _id: 0
+        _id: 0,
+        operations: showOperations,
+        operationSettings: showOperationSettings,
+        recordLoadParams: showRecordLoadParams,
+        importJobState: showImportJobState
       }
-    }).toArray();
+    }).limit(parseInt(limit, 10)).skip(parseInt(skip, 10)).toArray();
     logger.debug(`Query result: ${result.length > 0 ? 'Found!' : 'Not found!'}`);
     return result;
   }
 
-  async function queryById(correlationId, checkModTime) {
+  async function queryById({
+    correlationId,
+    checkModTime = false
+  }) {
     const result = await db.collection(collection).findOne({
       correlationId
     });
 
     if (checkModTime) {
-      const timeOut = await checkTimeOut(correlationId);
+      const timeOut = await checkTimeOut({
+        correlationId
+      });
 
       if (timeOut) {
-        return queryById(correlationId);
+        return queryById({
+          correlationId
+        });
       }
 
       return result;
@@ -247,15 +343,18 @@ async function _default(MONGO_URI, collection) {
   // If the state is already ABORT or ERROR return false
 
 
-  async function checkTimeOut(correlationId) {
+  async function checkTimeOut({
+    correlationId
+  }) {
     const {
       modificationTime,
-      queueitemState: oldState
+      queueitemState: oldState,
+      importJobState
     } = await db.collection(collection).findOne({
       correlationId
-    });
+    }); // should we check for DONE too?
 
-    if ([_constants.QUEUE_ITEM_STATE.ABORT, _constants.QUEUE_ITEM_STATE.ERROR].includes(oldState)) {
+    if ([_constants.QUEUE_ITEM_STATE.ABORT, _constants.QUEUE_ITEM_STATE.ERROR, _constants.QUEUE_ITEM_STATE.DONE].includes(oldState)) {
       logger.silly(`${correlationId} has already state: ${oldState}`);
       return false;
     }
@@ -264,6 +363,26 @@ async function _default(MONGO_URI, collection) {
     logger.silly(`timeOut @ ${timeoutTime}`);
 
     if (timeoutTime.isBefore()) {
+      const finalImportJobStates = [_constants.IMPORT_JOB_STATE.ABORT, _constants.IMPORT_JOB_STATE.DONE, _constants.IMPORT_JOB_STATE.EMPTY, _constants.IMPORT_JOB_STATE.ERROR];
+
+      if (!finalImportJobStates.includes(importJobState.CREATE)) {
+        // eslint-disable-line
+        await setImportJobState({
+          correlationId,
+          operation: 'CREATE',
+          importJobState: _constants.IMPORT_JOB_STATE.ABORT
+        });
+      }
+
+      if (!finalImportJobStates.includes(importJobState.UPDATE)) {
+        // eslint-disable-line
+        await setImportJobState({
+          correlationId,
+          operation: 'UPDATE',
+          importJobState: _constants.IMPORT_JOB_STATE.ABORT
+        });
+      }
+
       await setState({
         correlationId,
         state: _constants.QUEUE_ITEM_STATE.ABORT,
@@ -319,13 +438,26 @@ async function _default(MONGO_URI, collection) {
     }); // njsscan-ignore: node_nosqli_injection
 
     if (result) {
-      return {
-        contentType: result.contentType,
-        readStream: gridFSBucket.openDownloadStreamByName(clean)
-      };
+      const {
+        operationSettings
+      } = result;
+
+      if (operationSettings.prio === false && operationSettings.noStream === false) {
+        return {
+          contentType: result.contentType,
+          readStream: gridFSBucket.openDownloadStreamByName(clean)
+        };
+      }
+
+      logger.debug(`OperationSetting for ${correlationId}: ${JSON.stringify(operationSettings)}`);
+      throw new _melindaCommons.Error(_httpStatus.default.BAD_REQUEST, {
+        message: `Content is only available for streamBulk jobs. ${correlationId} is not a streamBulk job.`
+      });
     }
 
-    throw new _melindaCommons.Error(_httpStatus.default.NOT_FOUND);
+    throw new _melindaCommons.Error(_httpStatus.default.NOT_FOUND, {
+      message: `Job ${correlationId} not found.`
+    });
   }
 
   async function removeContent(params) {
@@ -343,30 +475,43 @@ async function _default(MONGO_URI, collection) {
     }
 
     return true;
-  }
+  } // eslint-disable-next-line max-statements
+
 
   function getOne({
-    operation,
-    queueItemState
+    queueItemState,
+    importJobState = undefined
   }) {
-    const clean = {
+    logger.silly(`queueItemState: ${queueItemState}, importJobState: ${JSON.stringify(importJobState)}`);
+    const cleanQueueItemState = queueItemState ? {
       queueItemState: (0, _mongoSanitize.default)(queueItemState)
-    };
+    } : undefined;
 
     try {
-      if (operation === undefined) {
-        logger.silly(`Checking DB ${collection} for ${JSON.stringify(clean.queueItemState)}`);
-        return db.collection(collection).findOne({ ...clean
+      // Just queueItemState
+      if (queueItemState && importJobState === undefined) {
+        logger.silly(`Checking DB ${collection} for just ${JSON.stringify(cleanQueueItemState.queueItemState)}`);
+        return db.collection(collection).findOne({ ...cleanQueueItemState
+        });
+      } // importJobState
+
+
+      if (importJobState && queueItemState === undefined) {
+        logger.silly(`Checking DB ${collection} for ${JSON.stringify(importJobState)}`);
+        return db.collection(collection).findOne({ ...importJobState
+        });
+      } // importJobState and queueItemState
+
+
+      if (importJobState && queueItemState) {
+        logger.silly(`Checking DB ${collection} for ${queueItemState} and ${JSON.stringify(importJobState)}`);
+        return db.collection(collection).findOne({ ...cleanQueueItemState,
+          ...importJobState
         });
       }
 
-      const clean2 = {
-        operation: (0, _mongoSanitize.default)(operation)
-      };
-      logger.silly(`Checking DB ${collection} for ${clean.queueItemState} + ${clean2.operation}`);
-      return db.collection(collection).findOne({ ...clean,
-        ...clean2
-      });
+      logger.debug(`getOne not working!`);
+      throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR, 'Missing parameters, cannot get queueItem');
     } catch (error) {
       (0, _utils.logError)(error);
     }
@@ -469,5 +614,150 @@ async function _default(MONGO_URI, collection) {
       returnNewDocument: true
     });
   }
+
+  function setImportJobState({
+    correlationId,
+    operation,
+    importJobState
+  }) {
+    // should this also get importJobState as previously created object? {}
+    logger.info(`Setting queue-item importJobState: {${operation}: ${importJobState}} for ${correlationId}`);
+    const cleanCorrelationId = (0, _mongoSanitize.default)(correlationId);
+    const cleanImportJobState = (0, _mongoSanitize.default)(importJobState);
+
+    if (!(operation in _constants.OPERATIONS)) {
+      throw new _melindaCommons.Error('400', 'Invalid operation for import job state');
+    }
+
+    if (!(cleanImportJobState in _constants.IMPORT_JOB_STATE)) {
+      throw new _melindaCommons.Error('400', 'Invalid import job state');
+    }
+
+    const newJobState = operation === _constants.OPERATIONS.CREATE ? {
+      'importJobState.CREATE': cleanImportJobState
+    } : {
+      'importJobState.UPDATE': cleanImportJobState
+    };
+    return db.collection(collection).findOneAndUpdate({
+      correlationId: cleanCorrelationId
+    }, {
+      $set: { ...newJobState,
+        modificationTime: (0, _moment.default)().toDate()
+      }
+    }, {
+      projection: {
+        _id: 0
+      },
+      returnNewDocument: true
+    });
+  }
+
+  async function setOperation({
+    correlationId,
+    operation
+  }) {
+    const newOperation = operation;
+    const {
+      operation: oldOperation
+    } = await db.collection(collection).findOne({
+      correlationId
+    });
+    logger.info(`Setting queue-item operation from ${oldOperation} to ${newOperation} for ${correlationId} to ${collection}`);
+    const cleanCorrelationId = (0, _mongoSanitize.default)(correlationId);
+    const cleanNewOperation = (0, _mongoSanitize.default)(newOperation);
+    const result = await db.collection(collection).findOneAndUpdate({
+      correlationId: cleanCorrelationId
+    }, {
+      $set: {
+        operation: cleanNewOperation,
+        modificationTime: (0, _moment.default)().toDate()
+      }
+    }, {
+      projection: {
+        _id: 0
+      },
+      returnNewDocument: true
+    });
+    return result.ok; // logger.debug(JSON.stringify(result));
+  }
+
+  async function setOperations({
+    correlationId,
+    addOperation,
+    removeOperation = undefined
+  }) {
+    const cleanCorrelationId = (0, _mongoSanitize.default)(correlationId);
+    const queueItem = await db.collection(collection).findOne({
+      correlationId: cleanCorrelationId
+    });
+    logger.silly(`We found a queueItem: ${JSON.stringify(queueItem)}`);
+    const oldOperations = queueItem.operations;
+    logger.info(`Setting queue-item operations from ${JSON.stringify(oldOperations)} by adding ${addOperation} and removing ${removeOperation} for ${correlationId} to ${collection}`);
+    const operationsAfterRemove = oldOperations.filter(operation => operation !== removeOperation);
+    const operationsAfterRemoveAndAdd = operationsAfterRemove.includes(addOperation) ? operationsAfterRemove : [...operationsAfterRemove, addOperation];
+    logger.silly(`operationsAfterRemove: ${operationsAfterRemove}`);
+    logger.silly(`operationAfterRemoveAndAdd: ${operationsAfterRemoveAndAdd}`);
+    const result = await db.collection(collection).findOneAndUpdate({
+      correlationId: cleanCorrelationId
+    }, {
+      $set: {
+        operations: operationsAfterRemoveAndAdd,
+        modificationTime: (0, _moment.default)().toDate()
+      }
+    }, {
+      projection: {
+        _id: 0
+      },
+      returnNewDocument: true
+    });
+    return result.ok; // logger.debug(JSON.stringify(result));
+  }
+
+  async function addBlobSize({
+    correlationId
+  }) {
+    const cleanCorrelationId = (0, _mongoSanitize.default)(correlationId);
+    const result = await db.collection(collection).findOneAndUpdate({
+      correlationId: cleanCorrelationId,
+      queueItemState: _constants.QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS
+    }, {
+      $inc: {
+        blobSize: 1
+      },
+      $set: {
+        modificationTime: (0, _moment.default)().toDate()
+      }
+    }, {
+      projection: {
+        _id: 0
+      },
+      returnNewDocument: true
+    });
+    logger.verbose(`AddBlobSizeResult in mongo: ${JSON.stringify(result)}`);
+    return result;
+  }
+
+  async function setBlobSize({
+    correlationId,
+    blobSize
+  }) {
+    const cleanCorrelationId = (0, _mongoSanitize.default)(correlationId);
+    const cleanBlobSize = (0, _mongoSanitize.default)(blobSize);
+    const result = await db.collection(collection).findOneAndUpdate({
+      correlationId: cleanCorrelationId
+    }, {
+      $set: {
+        blobSize: cleanBlobSize,
+        modificationTime: (0, _moment.default)().toDate()
+      }
+    }, {
+      projection: {
+        _id: 0
+      },
+      returnNewDocument: true
+    });
+    logger.verbose(`SetBlobSizeResult in mongo: ${JSON.stringify(result)}`);
+    return result;
+  }
 }
 //# sourceMappingURL=mongo.js.map