@natlibfi/melinda-rest-api-commons 2.1.3 → 2.2.0-alpha.11

package/dist/mongo.js

@@ -57,6 +57,7 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
   "cataloger":"xxx0000",
   "oCatalogerIn":"xxx0000"
   "operation":"UPDATE",
+  "operations":["UPDATE", "CREATE"]
   "contentType":"application/json",
   "recordLoadParams": {
     "pActiveLibrary": "XXX00",
@@ -66,6 +67,10 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
     "pOldNew": "NEW"
   },
   "queueItemState":"PENDING_QUEUING",
+  "importJobState": {
+    "CREATE": "EMPTY",
+    "UPDATE": "EMPTY"
+  },
   "creationTime":"2020-01-01T00:00:00.000Z",
   "modificationTime":"2020-01-01T00:00:01.000Z",
   "handledIds": []
@@ -86,6 +91,7 @@ async function _default(MONGO_URI, collection) {
     createPrio,
     createBulk,
     checkAndSetState,
+    checkAndSetImportJobState,
     query,
     queryById,
     remove,
@@ -94,8 +100,13 @@ async function _default(MONGO_URI, collection) {
     getOne,
     getStream,
     setState,
+    setImportJobState,
     pushIds,
-    pushMessages
+    pushMessages,
+    setOperation,
+    setOperations,
+    addBlobSize,
+    setBlobSize
   };
 
   async function createPrio({
@@ -103,22 +114,25 @@ async function _default(MONGO_URI, collection) {
     cataloger,
     oCatalogerIn,
     operation,
-    noop = undefined,
-    unique = undefined,
-    prio = true
+    operationSettings
   }) {
     const time = (0, _moment.default)().toDate();
     const newQueueItem = {
       correlationId,
       cataloger,
       operation,
+      operations: [operation],
       oCatalogerIn,
       operationSettings: {
-        unique,
-        noop,
-        prio
+        prio: true,
+        ...operationSettings
       },
       queueItemState: _constants.QUEUE_ITEM_STATE.VALIDATOR.PENDING_VALIDATION,
+      importJobState: {
+        CREATE: _constants.IMPORT_JOB_STATE.EMPTY,
+        UPDATE: _constants.IMPORT_JOB_STATE.EMPTY
+      },
+      blobSize: 1,
       creationTime: time,
       modificationTime: time,
       handledIds: [],
@@ -140,7 +154,7 @@ async function _default(MONGO_URI, collection) {
     }
   }
 
-  function createBulk({
+  async function createBulk({
     correlationId,
     cataloger,
     oCatalogerIn,
@@ -148,7 +162,7 @@ async function _default(MONGO_URI, collection) {
     contentType,
     recordLoadParams,
     stream,
-    prio = false
+    operationSettings
   }) {
     const time = (0, _moment.default)().toDate();
     const newQueueItem = {
@@ -156,45 +170,76 @@ async function _default(MONGO_URI, collection) {
       cataloger,
       oCatalogerIn,
       operation,
+      operations: [operation],
       operationSettings: {
-        prio
+        prio: false,
+        ...operationSettings
       },
       contentType,
       recordLoadParams,
-      queueItemState: _constants.QUEUE_ITEM_STATE.VALIDATOR.UPLOADING,
+      queueItemState: stream ? _constants.QUEUE_ITEM_STATE.VALIDATOR.UPLOADING : _constants.QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS,
+      blobSize: 0,
+      importJobState: {
+        CREATE: _constants.IMPORT_JOB_STATE.EMPTY,
+        UPDATE: _constants.IMPORT_JOB_STATE.EMPTY
+      },
       creationTime: time,
       modificationTime: time,
       handledIds: [],
       rejectedIds: []
-    };
-
-    try {
-      // No await here, promises later
-      db.collection(collection).insertOne(newQueueItem);
-      logger.info(`New BULK queue item for ${operation} ${correlationId} has been made in ${collection}!`);
-    } catch (error) {
-      (0, _utils.logError)(error);
-      throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR);
+    }; // eslint-disable-next-line functional/no-conditional-statement
+
+    if (stream) {
+      try {
+        // No await here, promises later
+        db.collection(collection).insertOne(newQueueItem);
+        logger.info(`New BULK queue item for ${operation} ${correlationId} has been made in ${collection}!`);
+        return new Promise((resolve, reject) => {
+          const outputStream = gridFSBucket.openUploadStream(correlationId);
+          stream.on('error', reject).on('data', chunk => outputStream.write(chunk)).on('end', () => outputStream.end(undefined, undefined, () => {
+            resolve(correlationId);
+          }));
+        });
+      } catch (error) {
+        (0, _utils.logError)(error);
+        throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR);
+      }
     }
 
-    return new Promise((resolve, reject) => {
-      const outputStream = gridFSBucket.openUploadStream(correlationId);
-      stream.on('error', reject).on('data', chunk => outputStream.write(chunk)).on('end', () => outputStream.end(undefined, undefined, () => {
-        resolve(correlationId);
-      }));
-    });
+    logger.debug(`No stream`); // eslint-disable-next-line functional/no-conditional-statement
+
+    if (!stream) {
+      try {
+        const result = await db.collection(collection).insertOne(newQueueItem);
+
+        if (result.acknowledged) {
+          logger.info(`New noStream BULK queue item for ${operation} ${correlationId} has been made in ${collection}`);
+          return {
+            correlationId,
+            queueItemState: _constants.QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS
+          };
+        }
+
+        throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR);
+      } catch (error) {
+        (0, _utils.logError)(error);
+        throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR);
+      }
+    }
   } // Check state that the queueItem has not waited too long and set state
 
 
   async function checkAndSetState({
     correlationId,
     state,
-    errorMessage = '',
-    errorStatus = ''
+    errorMessage = undefined,
+    errorStatus = undefined
   }) {
     // checkTimeOut returns true, if queueItem is fresher than 1 minute and it's state is not ABORT/ERROR
     // otherwise it sets queueItem to state ABORT (408, 'Timeout')
-    const timeOut = await checkTimeOut(correlationId);
+    const timeOut = await checkTimeOut({
+      correlationId
+    });
 
     if (timeOut) {
       return setState({
@@ -205,6 +250,36 @@ async function _default(MONGO_URI, collection) {
       });
     }
 
+    return false;
+  } // Check state that the queueItem has not waited too long and set state
+
+
+  async function checkAndSetImportJobState({
+    correlationId,
+    operation,
+    importJobState,
+    errorMessage = '',
+    errorStatus = ''
+  }) {
+    // checkTimeOut returns true, if queueItem is fresher than 1 minute and it's state is not ABORT/ERROR
+    // otherwise it sets queueItem to state ABORT (408, 'Timeout')
+    logger.debug(`${correlationId}, ${importJobState}, ${operation}`);
+    const timeOut = await checkTimeOut({
+      correlationId,
+      operation,
+      importJobState
+    });
+
+    if (timeOut) {
+      return setImportJobState({
+        correlationId,
+        operation,
+        importJobState,
+        errorMessage,
+        errorStatus
+      });
+    }
+
     return false;
   }
 
@@ -218,16 +293,23 @@ async function _default(MONGO_URI, collection) {
     return result;
   }
 
-  async function queryById(correlationId, checkModTime) {
+  async function queryById({
+    correlationId,
+    checkModTime = false
+  }) {
     const result = await db.collection(collection).findOne({
       correlationId
     });
 
     if (checkModTime) {
-      const timeOut = await checkTimeOut(correlationId);
+      const timeOut = await checkTimeOut({
+        correlationId
+      });
 
       if (timeOut) {
-        return queryById(correlationId);
+        return queryById({
+          correlationId
+        });
       }
 
       return result;
@@ -240,15 +322,18 @@ async function _default(MONGO_URI, collection) {
   // If the state is already ABORT or ERROR return false
 
 
-  async function checkTimeOut(correlationId) {
+  async function checkTimeOut({
+    correlationId
+  }) {
     const {
       modificationTime,
-      queueitemState: oldState
+      queueitemState: oldState,
+      importJobState
     } = await db.collection(collection).findOne({
       correlationId
-    });
+    }); // should we check for DONE too?
 
-    if ([_constants.QUEUE_ITEM_STATE.ABORT, _constants.QUEUE_ITEM_STATE.ERROR].includes(oldState)) {
+    if ([_constants.QUEUE_ITEM_STATE.ABORT, _constants.QUEUE_ITEM_STATE.ERROR, _constants.QUEUE_ITEM_STATE.DONE].includes(oldState)) {
       logger.silly(`${correlationId} has already state: ${oldState}`);
       return false;
     }
@@ -257,6 +342,26 @@ async function _default(MONGO_URI, collection) {
     logger.silly(`timeOut @ ${timeoutTime}`);
 
     if (timeoutTime.isBefore()) {
+      const finalImportJobStates = [_constants.IMPORT_JOB_STATE.ABORT, _constants.IMPORT_JOB_STATE.DONE, _constants.IMPORT_JOB_STATE.EMPTY, _constants.IMPORT_JOB_STATE.ERROR];
+
+      if (!finalImportJobStates.includes(importJobState.CREATE)) {
+        // eslint-disable-line
+        await setImportJobState({
+          correlationId,
+          operation: 'CREATE',
+          importJobState: _constants.IMPORT_JOB_STATE.ABORT
+        });
+      }
+
+      if (!finalImportJobStates.includes(importJobState.UPDATE)) {
+        // eslint-disable-line
+        await setImportJobState({
+          correlationId,
+          operation: 'UPDATE',
+          importJobState: _constants.IMPORT_JOB_STATE.ABORT
+        });
+      }
+
       await setState({
         correlationId,
         state: _constants.QUEUE_ITEM_STATE.ABORT,
@@ -336,30 +441,43 @@ async function _default(MONGO_URI, collection) {
     }
 
     return true;
-  }
+  } // eslint-disable-next-line max-statements
+
 
   function getOne({
-    operation,
-    queueItemState
+    queueItemState,
+    importJobState = undefined
   }) {
-    const clean = {
+    logger.silly(`queueItemState: ${queueItemState}, importJobState: ${JSON.stringify(importJobState)}`);
+    const cleanQueueItemState = queueItemState ? {
       queueItemState: (0, _mongoSanitize.default)(queueItemState)
-    };
+    } : undefined;
 
     try {
-      if (operation === undefined) {
-        logger.silly(`Checking DB ${collection} for ${JSON.stringify(clean.queueItemState)}`);
-        return db.collection(collection).findOne({ ...clean
+      // Just queueItemState
+      if (queueItemState && importJobState === undefined) {
+        logger.silly(`Checking DB ${collection} for just ${JSON.stringify(cleanQueueItemState.queueItemState)}`);
+        return db.collection(collection).findOne({ ...cleanQueueItemState
+        });
+      } // importJobState
+
+
+      if (importJobState && queueItemState === undefined) {
+        logger.silly(`Checking DB ${collection} for ${JSON.stringify(importJobState)}`);
+        return db.collection(collection).findOne({ ...importJobState
+        });
+      } // importJobState and queueItemState
+
+
+      if (importJobState && queueItemState) {
+        logger.silly(`Checking DB ${collection} for ${queueItemState} and ${JSON.stringify(importJobState)}`);
+        return db.collection(collection).findOne({ ...cleanQueueItemState,
+          ...importJobState
         });
       }
 
-      const clean2 = {
-        operation: (0, _mongoSanitize.default)(operation)
-      };
-      logger.silly(`Checking DB ${collection} for ${clean.queueItemState} + ${clean2.operation}`);
-      return db.collection(collection).findOne({ ...clean,
-        ...clean2
-      });
+      logger.debug(`getOne not working!`);
+      throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR, 'Missing parameters, cannot get queueItem');
     } catch (error) {
       (0, _utils.logError)(error);
     }
@@ -428,20 +546,97 @@ async function _default(MONGO_URI, collection) {
   function setState({
     correlationId,
     state,
-    errorMessage = '',
-    errorStatus = ''
+    errorMessage = undefined,
+    errorStatus = undefined
   }) {
-    const errorString = errorMessage || errorStatus ? `, Error message: '${errorMessage}', Error status: '${errorStatus}'` : '';
+    const errorString = errorMessage || errorStatus ? `, Error message: '${errorMessage || ''}', Error status: '${errorStatus || ''}'` : '';
     logger.info(`Setting queue-item state ${state} for ${correlationId}${errorString} to ${collection}`);
+    const stateInQueueItemStates = Object.values(_constants.QUEUE_ITEM_STATE).indexOf(state) > -1;
+    const stateInQueueItemStatesValidator = Object.values(_constants.QUEUE_ITEM_STATE.VALIDATOR).indexOf(state) > -1;
+    const stateInQueueItemStatesImporter = Object.values(_constants.QUEUE_ITEM_STATE.IMPORTER).indexOf(state) > -1;
+
+    if (!stateInQueueItemStates && !stateInQueueItemStatesValidator && !stateInQueueItemStatesImporter) {
+      throw new _melindaCommons.Error(_httpStatus.default.BAD_REQUEST, 'Trying to set invalid state');
+    }
+
     const clean = (0, _mongoSanitize.default)(correlationId);
+    const updateValues = {
+      queueItemState: state,
+      modificationTime: (0, _moment.default)().toDate(),
+      errorMessage,
+      errorStatus
+    }; // Do not update value that are undefined
+    // eslint-disable-next-line functional/immutable-data
+
+    Object.keys(updateValues).forEach(key => updateValues[key] === undefined && delete updateValues[key]);
     return db.collection(collection).findOneAndUpdate({
       correlationId: clean
+    }, {
+      $set: updateValues
+    }, {
+      projection: {
+        _id: 0
+      },
+      returnNewDocument: true
+    });
+  }
+
+  function setImportJobState({
+    correlationId,
+    operation,
+    importJobState
+  }) {
+    // should this also get importJobState as previously created object? {}
+    logger.info(`Setting queue-item importJobState: {${operation}: ${importJobState}} for ${correlationId}`);
+    const cleanCorrelationId = (0, _mongoSanitize.default)(correlationId);
+    const cleanImportJobState = (0, _mongoSanitize.default)(importJobState);
+
+    if (!(operation in _constants.OPERATIONS)) {
+      throw new _melindaCommons.Error('400', 'Invalid operation for import job state');
+    }
+
+    if (!(cleanImportJobState in _constants.IMPORT_JOB_STATE)) {
+      throw new _melindaCommons.Error('400', 'Invalid import job state');
+    }
+
+    const newJobState = operation === _constants.OPERATIONS.CREATE ? {
+      'importJobState.CREATE': cleanImportJobState
+    } : {
+      'importJobState.UPDATE': cleanImportJobState
+    };
+    return db.collection(collection).findOneAndUpdate({
+      correlationId: cleanCorrelationId
+    }, {
+      $set: { ...newJobState,
+        modificationTime: (0, _moment.default)().toDate()
+      }
+    }, {
+      projection: {
+        _id: 0
+      },
+      returnNewDocument: true
+    });
+  }
+
+  async function setOperation({
+    correlationId,
+    operation
+  }) {
+    const newOperation = operation;
+    const {
+      operation: oldOperation
+    } = await db.collection(collection).findOne({
+      correlationId
+    });
+    logger.info(`Setting queue-item operation from ${oldOperation} to ${newOperation} for ${correlationId} to ${collection}`);
+    const cleanCorrelationId = (0, _mongoSanitize.default)(correlationId);
+    const cleanNewOperation = (0, _mongoSanitize.default)(newOperation);
+    const result = await db.collection(collection).findOneAndUpdate({
+      correlationId: cleanCorrelationId
     }, {
       $set: {
-        queueItemState: state,
-        modificationTime: (0, _moment.default)().toDate(),
-        errorMessage,
-        errorStatus
+        operation: cleanNewOperation,
+        modificationTime: (0, _moment.default)().toDate()
       }
     }, {
       projection: {
@@ -449,6 +644,85 @@ async function _default(MONGO_URI, collection) {
       },
       returnNewDocument: true
     });
+    return result.ok; // logger.debug(JSON.stringify(result));
+  }
+
+  async function setOperations({
+    correlationId,
+    addOperation,
+    removeOperation = undefined
+  }) {
+    const cleanCorrelationId = (0, _mongoSanitize.default)(correlationId); //logger.info(`Setting queue-item operation from ${JSON.stringify(oldOperations)} by adding ${addOperation} and removing ${removeOperation} for ${correlationId} to ${collection}`);
+
+    const queueItem = await db.collection(collection).findOne({
+      correlationId: cleanCorrelationId
+    });
+    logger.debug(`We fould a queueItem: ${JSON.stringify(queueItem)}`);
+    const oldOperations = queueItem.operations;
+    logger.info(`Setting queue-item operation from ${JSON.stringify(oldOperations)} by adding ${addOperation} and removing ${removeOperation} for ${correlationId} to ${collection}`);
+    const operationsAfterRemove = oldOperations.filter(operation => operation !== removeOperation);
+    const operationsAfterRemoveAndAdd = [addOperation, ...operationsAfterRemove];
+    const result = await db.collection(collection).findOneAndUpdate({
+      correlationId: cleanCorrelationId
+    }, {
+      $set: {
+        operations: operationsAfterRemoveAndAdd,
+        modificationTime: (0, _moment.default)().toDate()
+      }
+    }, {
+      projection: {
+        _id: 0
+      },
+      returnNewDocument: true
+    });
+    return result.ok; // logger.debug(JSON.stringify(result));
+  }
+
+  async function addBlobSize({
+    correlationId
+  }) {
+    const cleanCorrelationId = (0, _mongoSanitize.default)(correlationId);
+    const result = await db.collection(collection).findOneAndUpdate({
+      correlationId: cleanCorrelationId,
+      queueItemState: _constants.QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS
+    }, {
+      $inc: {
+        blobSize: 1
+      },
+      $set: {
+        modificationTime: (0, _moment.default)().toDate()
+      }
+    }, {
+      projection: {
+        _id: 0
+      },
+      returnNewDocument: true
+    });
+    logger.verbose(`AddBlobSizeResult in mongo: ${JSON.stringify(result)}`);
+    return result;
+  }
+
+  async function setBlobSize({
+    correlationId,
+    blobSize
+  }) {
+    const cleanCorrelationId = (0, _mongoSanitize.default)(correlationId);
+    const cleanBlobSize = (0, _mongoSanitize.default)(blobSize);
+    const result = await db.collection(collection).findOneAndUpdate({
+      correlationId: cleanCorrelationId
+    }, {
+      $set: {
+        blobSize: cleanBlobSize,
+        modificationTime: (0, _moment.default)().toDate()
+      }
+    }, {
+      projection: {
+        _id: 0
+      },
+      returnNewDocument: true
+    });
+    logger.verbose(`SetBlobSizeResult in mongo: ${JSON.stringify(result)}`);
+    return result;
   }
 }
 //# sourceMappingURL=mongo.js.map