@natlibfi/melinda-rest-api-commons 2.2.0-alpha.1 → 2.2.0-alpha.10

package/README.md

@@ -10,27 +10,30 @@ Shared modules for microservices of Melinda rest api import system
 
 #### QueueItemState flow for bulk jobs (multiple records, no validations):
 
-|microservice | states                                                                   |
-|-------------| -------------------------------------------------------------------------|
-|`http`       | -> VALIDATOR.UPLOADING -> VALIDATOR.PENDING_QUEUING ->                  |
-|`validator`  | VALIDATOR.QUEUING_IN_PROGRESS -> IMPORTER.IN_QUEUE ->                   |
-|`importer`   | IMPORTER.IMPORTING -> IMPORTER.IN_PROCESS -> IMPORTER.IMPORTING -> DONE  |
-|             |                                                                          |
-|any          | any -> ERROR if errored
+| microservice | states                                                 |
+|--------------|--------------------------------------------------------|
+| `http`       | -> VALIDATOR.UPLOADING -> VALIDATOR.PENDING_QUEUING -> |
+| `validator`  | VALIDATOR.QUEUING_IN_PROGRESS -> IMPORTER.IN_QUEUE ->  |
+| `importer`   | IMPORTER.IMPORTING -> IMPORT_JOB_STATEs* -> DONE       |
+|              |                                                        |
+| any          | any -> ERROR if errored                                |
 
+*IMPORT_JOB_STATEs: EMPTY or IN_QUEUE, PROCESSING -> final: DONE, ERROR, ABORT
 
 #### QueueItemState flow for prio jobs (single record):
 
-|microservice | states                                                                   |
-|-------------| -------------------------------------------------------------------------|
-|`http`       | -> VALIDATOR.PENDING_VALIDATION ->                                       |
-|`validator`  | VALIDATOR.VALIDATING -> IMPORTER.IN_QUEUE ->                             |
-|`importer`   | IMPORTER.IMPORTING -> IMPORTER.IN_PROCESS -> DONE                        |
-|             |                                                                          |
-|`http`       | any -> ABORT if job has stayed in an active state too long               |
-|`validator`  | VALIDATOR.VALIDATING -> DONE for noop (no-operation) jobs                |
-|             |                                                                          |
-| any         | any -> ERROR if errored                                                  |
+| microservice | states                                                     |
+|--------------|------------------------------------------------------------|
+| `http`       | -> VALIDATOR.PENDING_VALIDATION ->                         |
+| `validator`  | VALIDATOR.VALIDATING -> IMPORTER.IN_QUEUE ->               |
+| `importer`   | IMPORTER.IMPORTING -> IMPORT_JOB_STATEs* -> DONE           |
+|              |                                                            |
+| `http`       | any -> ABORT if job has stayed in an active state too long |
+| `validator`  | VALIDATOR.VALIDATING -> DONE for noop (no-operation) jobs  |
+|              |                                                            |
+| any          | any -> ERROR if errored                                    |
+
+*IMPORT_JOB_STATEs: EMPTY or IN_QUEUE, PROCESSING -> final: DONE, ERROR, ABORT
 
 ### CHUNK_SIZE
 Chunck size to execution time ratio (From file to queue: ~450 records in 15sec)

package/dist/constants.js

@@ -3,7 +3,7 @@
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-exports.conversionFormats = exports.QUEUE_ITEM_STATE = exports.OPERATIONS = exports.CHUNK_SIZE = void 0;
+exports.QUEUE_ITEM_STATE = exports.OPERATIONS = exports.IMPORT_JOB_STATE = exports.CONVERSION_FORMATS = exports.CHUNK_SIZE = void 0;
 
 /**
 *
@@ -52,6 +52,17 @@ const QUEUE_ITEM_STATE = {
   ABORT: 'ABORT'
 };
 exports.QUEUE_ITEM_STATE = QUEUE_ITEM_STATE;
+const IMPORT_JOB_STATE = {
+  EMPTY: 'EMPTY',
+  QUEUING: 'QUEUING',
+  IN_QUEUE: 'IN_QUEUE',
+  IMPORTING: 'IMPORTING',
+  PROCESSING: 'PROCESSING',
+  DONE: 'DONE',
+  ERROR: 'ERROR',
+  ABORT: 'ABORT'
+};
+exports.IMPORT_JOB_STATE = IMPORT_JOB_STATE;
 const OPERATIONS = {
   CREATE: 'CREATE',
   UPDATE: 'UPDATE'
@@ -59,10 +70,11 @@ const OPERATIONS = {
 exports.OPERATIONS = OPERATIONS;
 const CHUNK_SIZE = 100;
 exports.CHUNK_SIZE = CHUNK_SIZE;
-const conversionFormats = {
-  MARCXML: 1,
-  ISO2709: 2,
-  JSON: 3
+const CONVERSION_FORMATS = {
+  MARCXML: 'MARCXML',
+  ISO2709: 'ISO2709',
+  JSON: 'JSON',
+  ALEPHSEQ: 'ALEPHSEQ'
 };
-exports.conversionFormats = conversionFormats;
+exports.CONVERSION_FORMATS = CONVERSION_FORMATS;
 //# sourceMappingURL=constants.js.map

package/dist/constants.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/constants.js"],"names":["QUEUE_ITEM_STATE","VALIDATOR","PENDING_QUEUING","PENDING_VALIDATION","QUEUING_IN_PROGRESS","UPLOADING","VALIDATING","WAITING_FOR_RECORDS","IMPORTER","IMPORTING","IN_PROCESS","IN_QUEUE","DONE","ERROR","ABORT","OPERATIONS","CREATE","UPDATE","CHUNK_SIZE","conversionFormats","MARCXML","ISO2709","JSON"],"mappings":";;;;;;;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAEA;AAEO,MAAMA,gBAAgB,GAAG;AAC9BC,EAAAA,SAAS,EAAE;AACTC,IAAAA,eAAe,EAAE,iBADR;AAETC,IAAAA,kBAAkB,EAAE,oBAFX;AAGTC,IAAAA,mBAAmB,EAAE,qBAHZ;AAITC,IAAAA,SAAS,EAAE,WAJF;AAKTC,IAAAA,UAAU,EAAE,YALH;AAMTC,IAAAA,mBAAmB,EAAE;AANZ,GADmB;AAS9BC,EAAAA,QAAQ,EAAE;AACRC,IAAAA,SAAS,EAAE,WADH;AAERC,IAAAA,UAAU,EAAE,YAFJ;AAGRC,IAAAA,QAAQ,EAAE;AAHF,GAToB;AAc9BC,EAAAA,IAAI,EAAE,MAdwB;AAe9BC,EAAAA,KAAK,EAAE,OAfuB;AAgB9BC,EAAAA,KAAK,EAAE;AAhBuB,CAAzB;;AAmBA,MAAMC,UAAU,GAAG;AACxBC,EAAAA,MAAM,EAAE,QADgB;AAExBC,EAAAA,MAAM,EAAE;AAFgB,CAAnB;;AAKA,MAAMC,UAAU,GAAG,GAAnB;;AAEA,MAAMC,iBAAiB,GAAG;AAC/BC,EAAAA,OAAO,EAAE,CADsB;AAE/BC,EAAAA,OAAO,EAAE,CAFsB;AAG/BC,EAAAA,IAAI,EAAE;AAHyB,CAA1B","sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020-2021 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\n// QueueItemState flows: see README.md\n\nexport const QUEUE_ITEM_STATE = {\n  VALIDATOR: {\n    PENDING_QUEUING: 'PENDING_QUEUING',\n    PENDING_VALIDATION: 'PENDING_VALIDATION',\n    QUEUING_IN_PROGRESS: 'QUEUING_IN_PROGRESS',\n    UPLOADING: 'UPLOADING',\n    VALIDATING: 'VALIDATING',\n    WAITING_FOR_RECORDS: 'WAITING_FOR_RECORDS'\n  },\n  IMPORTER: {\n    IMPORTING: 'IMPORTING',\n    IN_PROCESS: 'IN_PROCESS',\n    IN_QUEUE: 'IN_QUEUE'\n  },\n  DONE: 'DONE',\n  ERROR: 'ERROR',\n  ABORT: 'ABORT'\n};\n\nexport const OPERATIONS = {\n  CREATE: 'CREATE',\n  UPDATE: 'UPDATE'\n};\n\nexport const CHUNK_SIZE = 100;\n\nexport const conversionFormats = {\n  MARCXML: 1,\n  ISO2709: 2,\n  JSON: 3\n};\n"],"file":"constants.js"}
+{"version":3,"sources":["../src/constants.js"],"names":["QUEUE_ITEM_STATE","VALIDATOR","PENDING_QUEUING","PENDING_VALIDATION","QUEUING_IN_PROGRESS","UPLOADING","VALIDATING","WAITING_FOR_RECORDS","IMPORTER","IMPORTING","IN_PROCESS","IN_QUEUE","DONE","ERROR","ABORT","IMPORT_JOB_STATE","EMPTY","QUEUING","PROCESSING","OPERATIONS","CREATE","UPDATE","CHUNK_SIZE","CONVERSION_FORMATS","MARCXML","ISO2709","JSON","ALEPHSEQ"],"mappings":";;;;;;;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAEA;AAEO,MAAMA,gBAAgB,GAAG;AAC9BC,EAAAA,SAAS,EAAE;AACTC,IAAAA,eAAe,EAAE,iBADR;AAETC,IAAAA,kBAAkB,EAAE,oBAFX;AAGTC,IAAAA,mBAAmB,EAAE,qBAHZ;AAITC,IAAAA,SAAS,EAAE,WAJF;AAKTC,IAAAA,UAAU,EAAE,YALH;AAMTC,IAAAA,mBAAmB,EAAE;AANZ,GADmB;AAS9BC,EAAAA,QAAQ,EAAE;AACRC,IAAAA,SAAS,EAAE,WADH;AAERC,IAAAA,UAAU,EAAE,YAFJ;AAGRC,IAAAA,QAAQ,EAAE;AAHF,GAToB;AAc9BC,EAAAA,IAAI,EAAE,MAdwB;AAe9BC,EAAAA,KAAK,EAAE,OAfuB;AAgB9BC,EAAAA,KAAK,EAAE;AAhBuB,CAAzB;;AAmBA,MAAMC,gBAAgB,GAAG;AAC9BC,EAAAA,KAAK,EAAE,OADuB;AAE9BC,EAAAA,OAAO,EAAE,SAFqB;AAG9BN,EAAAA,QAAQ,EAAE,UAHoB;AAI9BF,EAAAA,SAAS,EAAE,WAJmB;AAK9BS,EAAAA,UAAU,EAAE,YALkB;AAM9BN,EAAAA,IAAI,EAAE,MANwB;AAO9BC,EAAAA,KAAK,EAAE,OAPuB;AAQ9BC,EAAAA,KAAK,EAAE;AARuB,CAAzB;;AAWA,MAAMK,UAAU,GAAG;AACxBC,EAAAA,MAAM,EAAE,QADgB;AAExBC,EAAAA,MAAM,EAAE;AAFgB,CAAnB;;AAKA,MAAMC,UAAU,GAAG,GAAnB;;AAEA,MAAMC,kBAAkB,GAAG;AAChCC,EAAAA,OAAO,EAAE,SADuB;AAEhCC,EAAAA,OAAO,EAAE,SAFuB;AAGhCC,EAAAA,IAAI,EAAE,MAH0B;AAIhCC,EAAAA,QAAQ,EAAE;AAJsB,CAA3B","sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020-2021 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\n// QueueItemState flows: see README.md\n\nexport const QUEUE_ITEM_STATE = {\n  VALIDATOR: {\n    PENDING_QUEUING: 'PENDING_QUEUING',\n    PENDING_VALIDATION: 'PENDING_VALIDATION',\n    QUEUING_IN_PROGRESS: 'QUEUING_IN_PROGRESS',\n    UPLOADING: 'UPLOADING',\n    VALIDATING: 'VALIDATING',\n    WAITING_FOR_RECORDS: 'WAITING_FOR_RECORDS'\n  },\n  IMPORTER: {\n    IMPORTING: 'IMPORTING',\n    IN_PROCESS: 'IN_PROCESS',\n    IN_QUEUE: 'IN_QUEUE'\n  },\n  DONE: 'DONE',\n  ERROR: 'ERROR',\n  ABORT: 'ABORT'\n};\n\nexport const IMPORT_JOB_STATE = {\n  EMPTY: 'EMPTY',\n  QUEUING: 'QUEUING',\n  IN_QUEUE: 'IN_QUEUE',\n  IMPORTING: 'IMPORTING',\n  PROCESSING: 'PROCESSING',\n  DONE: 'DONE',\n  ERROR: 'ERROR',\n  ABORT: 'ABORT'\n};\n\nexport const OPERATIONS = {\n  CREATE: 'CREATE',\n  UPDATE: 'UPDATE'\n};\n\nexport const CHUNK_SIZE = 100;\n\nexport const CONVERSION_FORMATS = {\n  MARCXML: 'MARCXML',\n  ISO2709: 'ISO2709',\n  JSON: 'JSON',\n  ALEPHSEQ: 'ALEPHSEQ'\n};\n"],"file":"constants.js"}

package/dist/helpers/conversion.js

@@ -51,24 +51,28 @@ function _default() {
   return {
     serialize,
     unserialize
-  };
+  }; // This uses format as a number form contants CONVERSION_FORMATS
 
   function serialize(record, format) {
     logger.verbose(`Serializing record to ${format}`);
 
     try {
-      if (format === _constants.conversionFormats.MARCXML) {
+      if (format === _constants.CONVERSION_FORMATS.MARCXML) {
         return _marcRecordSerializers.MARCXML.to(record);
       }
 
-      if (format === _constants.conversionFormats.ISO2709) {
+      if (format === _constants.CONVERSION_FORMATS.ISO2709) {
         return _marcRecordSerializers.ISO2709.to(record);
       }
 
-      if (format === _constants.conversionFormats.JSON) {
+      if (format === _constants.CONVERSION_FORMATS.JSON) {
         return _marcRecordSerializers.Json.to(record);
       }
 
+      if (format === _constants.CONVERSION_FORMATS.ALEPHSEQ) {
+        return _marcRecordSerializers.AlephSequential.to(record);
+      }
+
       throw new _melindaCommons.Error(_httpStatus.default.UNSUPPORTED_MEDIA_TYPE);
     } catch (err) {
       (0, _utils.logError)(err);
@@ -80,7 +84,8 @@ function _default() {
 
       throw new _melindaCommons.Error(_httpStatus.default.BAD_REQUEST, 'Error while serializing record');
     }
-  }
+  } // eslint-disable-next-line max-statements
+
 
   function unserialize(data, format, validationOptions = {
     subfieldValues: false
@@ -91,21 +96,26 @@ function _default() {
     logger.silly(`Data: ${JSON.stringify(data)}`);
 
     try {
-      if (format === _constants.conversionFormats.MARCXML) {
+      if (format === _constants.CONVERSION_FORMATS.MARCXML) {
         logger.silly('Unserialize format marcxml');
         return _marcRecordSerializers.MARCXML.from(data, validationOptions);
       }
 
-      if (format === _constants.conversionFormats.ISO2709) {
+      if (format === _constants.CONVERSION_FORMATS.ISO2709) {
         logger.silly('Unserialize format iso2709');
         return _marcRecordSerializers.ISO2709.from(data, validationOptions);
       }
 
-      if (format === _constants.conversionFormats.JSON) {
+      if (format === _constants.CONVERSION_FORMATS.JSON) {
         logger.silly('Unserialize format json');
         return _marcRecordSerializers.Json.from(data, validationOptions);
       }
 
+      if (format === _constants.CONVERSION_FORMATS.ALEPHSEQ) {
+        logger.silly('Unserialize format aleph sequential');
+        return _marcRecordSerializers.AlephSequential.from(data, validationOptions);
+      }
+
       throw new _melindaCommons.Error(_httpStatus.default.UNSUPPORTED_MEDIA_TYPE);
     } catch (err) {
       (0, _utils.logError)(err);

package/dist/helpers/conversion.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/helpers/conversion.js"],"names":["logger","serialize","unserialize","record","format","verbose","conversionFormats","MARCXML","to","ISO2709","JSON","Json","ConversionError","httpStatus","UNSUPPORTED_MEDIA_TYPE","err","BAD_REQUEST","data","validationOptions","subfieldValues","silly","debug","stringify","from"],"mappings":";;;;;;;AA4BA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AAjCA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AASe,oBAAY;AACzB,QAAMA,MAAM,GAAG,0CAAf;AAEA,SAAO;AAACC,IAAAA,SAAD;AAAYC,IAAAA;AAAZ,GAAP;;AAEA,WAASD,SAAT,CAAmBE,MAAnB,EAA2BC,MAA3B,EAAmC;AACjCJ,IAAAA,MAAM,CAACK,OAAP,CAAgB,yBAAwBD,MAAO,EAA/C;;AACA,QAAI;AACF,UAAIA,MAAM,KAAKE,6BAAkBC,OAAjC,EAA0C;AACxC,eAAOA,+BAAQC,EAAR,CAAWL,MAAX,CAAP;AACD;;AAED,UAAIC,MAAM,KAAKE,6BAAkBG,OAAjC,EAA0C;AACxC,eAAOA,+BAAQD,EAAR,CAAWL,MAAX,CAAP;AACD;;AAED,UAAIC,MAAM,KAAKE,6BAAkBI,IAAjC,EAAuC;AACrC,eAAOC,4BAAKH,EAAL,CAAQL,MAAR,CAAP;AACD;;AAED,YAAM,IAAIS,qBAAJ,CAAoBC,oBAAWC,sBAA/B,CAAN;AACD,KAdD,CAcE,OAAOC,GAAP,EAAY;AACZ,2BAASA,GAAT;;AACA,UAAIA,GAAG,YAAYH,qBAAnB,EAAoC;AAAE;AACpC,cAAMG,GAAN;AACD;;AACD,YAAM,IAAIH,qBAAJ,CAAoBC,oBAAWG,WAA/B,EAA4C,gCAA5C,CAAN;AACD;AACF;;AAED,WAASd,WAAT,CAAqBe,IAArB,EAA2Bb,MAA3B,EAAmCc,iBAAiB,GAAG;AAACC,IAAAA,cAAc,EAAE;AAAjB,GAAvD,EAAgF;AAC9EnB,IAAAA,MAAM,CAACK,OAAP,CAAgB,6BAA4BD,MAAO,EAAnD;AACAJ,IAAAA,MAAM,CAACoB,KAAP,CAAc,WAAUhB,MAAO,EAA/B;AACAJ,IAAAA,MAAM,CAACqB,KAAP,CAAc,uBAAsBX,IAAI,CAACY,SAAL,CAAeJ,iBAAf,CAAkC,EAAtE;AACAlB,IAAAA,MAAM,CAACoB,KAAP,CAAc,SAAQV,IAAI,CAACY,SAAL,CAAeL,IAAf,CAAqB,EAA3C;;AACA,QAAI;AACF,UAAIb,MAAM,KAAKE,6BAAkBC,OAAjC,EAA0C;AACxCP,QAAAA,MAAM,CAACoB,KAAP,CAAa,4BAAb;AACA,eAAOb,+BAAQgB,IAAR,CAAaN,IAAb,EAAmBC,iBAAnB,CAAP;AACD;;AAED,UAAId,MAAM,KAAKE,6BAAkBG,OAAjC,EAA0C;AACxCT,QAAAA,MAAM,CAACoB,KAAP,CAAa,4BAAb;AACA,eAAOX,+BAAQc,IAAR,CAAaN,IAAb,EAAmBC,iBAAnB,CAAP;AACD;;AAED,UAAId,MAAM,KAAKE,6BAAkBI,IAAjC,EAAuC;AACrCV,QAAAA,MAAM,CAACoB,KAAP,CAAa,yBAAb;AACA,eAAOT,4BAAKY,IAAL,CAAUN,IAAV,EAAgBC,iBAAhB,CAAP;AACD;;AAED,YAAM,IAAIN,qBAAJ,CAAoBC,oBAAWC,sBAA/B,CAAN;AACD,KAjBD,CAiBE,OAAOC,GAAP,EAAY;AACZ,2BAASA,GAAT;;AACA,UAAIA,GAAG,YAAYH,qBAAnB,EAAoC;AAAE;AACpC,cAAMG,GAAN;AACD;;AACD,YAAM,IAAIH,qBAAJ,CAAoBC,oBAAWG,WAA/B,EAA4C,kCAA5C,CAAN;AACD;AACF;AACF","sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020-2021 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport httpStatus from 'http-status';\nimport {MARCXML, ISO2709, Json} from '@natlibfi/marc-record-serializers';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {Error as ConversionError} from '@natlibfi/melinda-commons';\nimport {conversionFormats} from '../constants';\nimport {logError} from '../utils';\n\nexport default function () {\n  const logger = createLogger();\n\n  return {serialize, unserialize};\n\n  function serialize(record, format) {\n    logger.verbose(`Serializing record to ${format}`);\n    try {\n      if (format === conversionFormats.MARCXML) {\n        return MARCXML.to(record);\n      }\n\n      if (format === conversionFormats.ISO2709) {\n        return ISO2709.to(record);\n      }\n\n      if (format === conversionFormats.JSON) {\n        return Json.to(record);\n      }\n\n      throw new ConversionError(httpStatus.UNSUPPORTED_MEDIA_TYPE);\n    } catch (err) {\n      logError(err);\n      if (err instanceof ConversionError) { // eslint-disable-line functional/no-conditional-statement\n        throw err;\n      }\n      throw new ConversionError(httpStatus.BAD_REQUEST, 'Error while serializing record');\n    }\n  }\n\n  function unserialize(data, format, validationOptions = {subfieldValues: false}) {\n    logger.verbose(`Unserializing record from ${format}`);\n    logger.silly(`Format: ${format}`);\n    logger.debug(`Validation options: ${JSON.stringify(validationOptions)}`);\n    logger.silly(`Data: ${JSON.stringify(data)}`);\n    try {\n      if (format === conversionFormats.MARCXML) {\n        logger.silly('Unserialize format marcxml');\n        return MARCXML.from(data, validationOptions);\n      }\n\n      if (format === conversionFormats.ISO2709) {\n        logger.silly('Unserialize format iso2709');\n        return ISO2709.from(data, validationOptions);\n      }\n\n      if (format === conversionFormats.JSON) {\n        logger.silly('Unserialize format json');\n        return Json.from(data, validationOptions);\n      }\n\n      throw new ConversionError(httpStatus.UNSUPPORTED_MEDIA_TYPE);\n    } catch (err) {\n      logError(err);\n      if (err instanceof ConversionError) { // eslint-disable-line functional/no-conditional-statement\n        throw err;\n      }\n      throw new ConversionError(httpStatus.BAD_REQUEST, 'Error while unserializing record');\n    }\n  }\n}\n"],"file":"conversion.js"}
+{"version":3,"sources":["../../src/helpers/conversion.js"],"names":["logger","serialize","unserialize","record","format","verbose","CONVERSION_FORMATS","MARCXML","to","ISO2709","JSON","Json","ALEPHSEQ","AlephSequential","ConversionError","httpStatus","UNSUPPORTED_MEDIA_TYPE","err","BAD_REQUEST","data","validationOptions","subfieldValues","silly","debug","stringify","from"],"mappings":";;;;;;;AA4BA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AAjCA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AASe,oBAAY;AACzB,QAAMA,MAAM,GAAG,0CAAf;AAEA,SAAO;AAACC,IAAAA,SAAD;AAAYC,IAAAA;AAAZ,GAAP,CAHyB,CAKzB;;AACA,WAASD,SAAT,CAAmBE,MAAnB,EAA2BC,MAA3B,EAAmC;AACjCJ,IAAAA,MAAM,CAACK,OAAP,CAAgB,yBAAwBD,MAAO,EAA/C;;AACA,QAAI;AACF,UAAIA,MAAM,KAAKE,8BAAmBC,OAAlC,EAA2C;AACzC,eAAOA,+BAAQC,EAAR,CAAWL,MAAX,CAAP;AACD;;AAED,UAAIC,MAAM,KAAKE,8BAAmBG,OAAlC,EAA2C;AACzC,eAAOA,+BAAQD,EAAR,CAAWL,MAAX,CAAP;AACD;;AAED,UAAIC,MAAM,KAAKE,8BAAmBI,IAAlC,EAAwC;AACtC,eAAOC,4BAAKH,EAAL,CAAQL,MAAR,CAAP;AACD;;AAED,UAAIC,MAAM,KAAKE,8BAAmBM,QAAlC,EAA4C;AAC1C,eAAOC,uCAAgBL,EAAhB,CAAmBL,MAAnB,CAAP;AACD;;AAED,YAAM,IAAIW,qBAAJ,CAAoBC,oBAAWC,sBAA/B,CAAN;AACD,KAlBD,CAkBE,OAAOC,GAAP,EAAY;AACZ,2BAASA,GAAT;;AACA,UAAIA,GAAG,YAAYH,qBAAnB,EAAoC;AAAE;AACpC,cAAMG,GAAN;AACD;;AACD,YAAM,IAAIH,qBAAJ,CAAoBC,oBAAWG,WAA/B,EAA4C,gCAA5C,CAAN;AACD;AACF,GAjCwB,CAmCzB;;;AACA,WAAShB,WAAT,CAAqBiB,IAArB,EAA2Bf,MAA3B,EAAmCgB,iBAAiB,GAAG;AAACC,IAAAA,cAAc,EAAE;AAAjB,GAAvD,EAAgF;AAC9ErB,IAAAA,MAAM,CAACK,OAAP,CAAgB,6BAA4BD,MAAO,EAAnD;AACAJ,IAAAA,MAAM,CAACsB,KAAP,CAAc,WAAUlB,MAAO,EAA/B;AACAJ,IAAAA,MAAM,CAACuB,KAAP,CAAc,uBAAsBb,IAAI,CAACc,SAAL,CAAeJ,iBAAf,CAAkC,EAAtE;AACApB,IAAAA,MAAM,CAACsB,KAAP,CAAc,SAAQZ,IAAI,CAACc,SAAL,CAAeL,IAAf,CAAqB,EAA3C;;AACA,QAAI;AACF,UAAIf,MAAM,KAAKE,8BAAmBC,OAAlC,EAA2C;AACzCP,QAAAA,MAAM,CAACsB,KAAP,CAAa,4BAAb;AACA,eAAOf,+BAAQkB,IAAR,CAAaN,IAAb,EAAmBC,iBAAnB,CAAP;AACD;;AAED,UAAIhB,MAAM,KAAKE,8BAAmBG,OAAlC,EAA2C;AACzCT,QAAAA,MAAM,CAACsB,KAAP,CAAa,4BAAb;AACA,eAAOb,+BAAQgB,IAAR,CAAaN,IAAb,EAAmBC,iBAAnB,CAAP;AACD;;AAED,UAAIhB,MAAM,KAAKE,8BAAmBI,IAAlC,EAAwC;AACtCV,QAAAA,MAAM,CAACsB,KAAP,CAAa,yBAAb;AACA,eAAOX,4BAAKc,IAAL,CAAUN,IAAV,EAAgBC,iBAAhB,CAAP;AACD;;AAED,UAAIhB,MAAM,KAAKE,8BAAmBM,QAAlC,EAA4C;AAC1CZ,QAAAA,MAAM,CAACsB,KAAP,CAAa,qCAAb;AACA,eAAOT,uCAAgBY,IAAhB,CAAqBN,IAArB,EAA2BC,iBAA3B,CAAP;AACD;;AAED,YAAM,IAAIN,qBAAJ,CAAoBC,oBAAWC,sBAA/B,CAAN;AACD,KAtBD,CAsBE,OAAOC,GAAP,EAAY;AACZ,2BAASA,GAAT;;AACA,UAAIA,GAAG,YAAYH,qBAAnB,EAAoC;AAAE;AACpC,cAAMG,GAAN;AACD;;AACD,YAAM,IAAIH,qBAAJ,CAAoBC,oBAAWG,WAA/B,EAA4C,kCAA5C,CAAN;AACD;AACF;AACF","sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020-2021 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport httpStatus from 'http-status';\nimport {MARCXML, ISO2709, Json, AlephSequential} from '@natlibfi/marc-record-serializers';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {Error as ConversionError} from '@natlibfi/melinda-commons';\nimport {CONVERSION_FORMATS} from '../constants';\nimport {logError} from '../utils';\n\nexport default function () {\n  const logger = createLogger();\n\n  return {serialize, unserialize};\n\n  // This uses format as a number form contants CONVERSION_FORMATS\n  function serialize(record, format) {\n    logger.verbose(`Serializing record to ${format}`);\n    try {\n      if (format === CONVERSION_FORMATS.MARCXML) {\n        return MARCXML.to(record);\n      }\n\n      if (format === CONVERSION_FORMATS.ISO2709) {\n        return ISO2709.to(record);\n      }\n\n      if (format === CONVERSION_FORMATS.JSON) {\n        return Json.to(record);\n      }\n\n      if (format === CONVERSION_FORMATS.ALEPHSEQ) {\n        return AlephSequential.to(record);\n      }\n\n      throw new ConversionError(httpStatus.UNSUPPORTED_MEDIA_TYPE);\n    } catch (err) {\n      logError(err);\n      if (err instanceof ConversionError) { // eslint-disable-line functional/no-conditional-statement\n        throw err;\n      }\n      throw new ConversionError(httpStatus.BAD_REQUEST, 'Error while serializing record');\n    }\n  }\n\n  // eslint-disable-next-line max-statements\n  function unserialize(data, format, validationOptions = {subfieldValues: false}) {\n    logger.verbose(`Unserializing record from ${format}`);\n    logger.silly(`Format: ${format}`);\n    logger.debug(`Validation options: ${JSON.stringify(validationOptions)}`);\n    logger.silly(`Data: ${JSON.stringify(data)}`);\n    try {\n      if (format === CONVERSION_FORMATS.MARCXML) {\n        logger.silly('Unserialize format marcxml');\n        return MARCXML.from(data, validationOptions);\n      }\n\n      if (format === CONVERSION_FORMATS.ISO2709) {\n        logger.silly('Unserialize format iso2709');\n        return ISO2709.from(data, validationOptions);\n      }\n\n      if (format === CONVERSION_FORMATS.JSON) {\n        logger.silly('Unserialize format json');\n        return Json.from(data, validationOptions);\n      }\n\n      if (format === CONVERSION_FORMATS.ALEPHSEQ) {\n        logger.silly('Unserialize format aleph sequential');\n        return AlephSequential.from(data, validationOptions);\n      }\n\n      throw new ConversionError(httpStatus.UNSUPPORTED_MEDIA_TYPE);\n    } catch (err) {\n      logError(err);\n      if (err instanceof ConversionError) { // eslint-disable-line functional/no-conditional-statement\n        throw err;\n      }\n      throw new ConversionError(httpStatus.BAD_REQUEST, 'Error while unserializing record');\n    }\n  }\n}\n"],"file":"conversion.js"}

package/dist/helpers/conversion.spec.js

@@ -62,6 +62,9 @@ describe('services/conversion', () => {
   const json = getFixture({
     components: ['json1']
   });
+  const alephSeq = getFixture({
+    components: ['aleph_sequential']
+  });
   describe('factory', () => {
     it('Should create the expected object', () => {
       const service = (0, _conversion.default)();
@@ -73,37 +76,45 @@ describe('services/conversion', () => {
       (0, _chai.expect)(conversionService.serialize).to.throw();
     });
     it('Should serialize to MARCXML', () => {
-      const data = conversionService.serialize(marcRecord, _constants.conversionFormats.MARCXML);
+      const data = conversionService.serialize(marcRecord, _constants.CONVERSION_FORMATS.MARCXML);
       (0, _chai.expect)(data).to.equal(marcXml);
     });
     it('Should serialize to ISO2709', () => {
-      const data = conversionService.serialize(marcRecord, _constants.conversionFormats.ISO2709);
+      const data = conversionService.serialize(marcRecord, _constants.CONVERSION_FORMATS.ISO2709);
       (0, _chai.expect)(data).to.equal(iso2709);
     });
     it('Should serialize to JSON', () => {
-      const data = conversionService.serialize(marcRecord, _constants.conversionFormats.JSON);
+      const data = conversionService.serialize(marcRecord, _constants.CONVERSION_FORMATS.JSON);
       (0, _chai.expect)(data).to.equal(json);
     });
+    it('Should serialize to aleph sequential', () => {
+      const data = conversionService.serialize(marcRecord, _constants.CONVERSION_FORMATS.ALEPHSEQ);
+      (0, _chai.expect)(data).to.equal(alephSeq);
+    });
   });
   describe('#unserialize', () => {
     it('Should throw because of unsupported format', () => {
       (0, _chai.expect)(conversionService.unserialize).to.throw();
     });
     it('Should unserialize from MARCXML', async () => {
-      const record = await conversionService.unserialize(marcXml, _constants.conversionFormats.MARCXML);
+      const record = await conversionService.unserialize(marcXml, _constants.CONVERSION_FORMATS.MARCXML);
       (0, _chai.expect)(record.equalsTo(marcRecord)).to.equal(true);
     });
     it('Should unserialize from ISO2709', () => {
-      const record = conversionService.unserialize(iso2709, _constants.conversionFormats.ISO2709);
+      const record = conversionService.unserialize(iso2709, _constants.CONVERSION_FORMATS.ISO2709);
       (0, _chai.expect)(record.equalsTo(marcRecord)).to.equal(true);
     });
     it('Should unserialize from JSON', () => {
-      const record = conversionService.unserialize(json, _constants.conversionFormats.JSON);
+      const record = conversionService.unserialize(json, _constants.CONVERSION_FORMATS.JSON);
+      (0, _chai.expect)(record.equalsTo(marcRecord)).to.equal(true);
+    });
+    it('Should unserialize from Aleph sequential', () => {
+      const record = conversionService.unserialize(alephSeq, _constants.CONVERSION_FORMATS.ALEPHSEQ);
       (0, _chai.expect)(record.equalsTo(marcRecord)).to.equal(true);
     });
     it('Should throw because the record could not be unserialized', () => {
       (0, _chai.expect)(() => {
-        conversionService.unserialize('', _constants.conversionFormats.JSON);
+        conversionService.unserialize('', _constants.CONVERSION_FORMATS.JSON);
       }).to.throw(_melindaCommons.Error);
     });
   });

package/dist/helpers/conversion.spec.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/helpers/conversion.spec.js"],"names":["describe","getFixture","root","__dirname","conversionService","marcRecord","MarcRecord","JSON","parse","components","subfieldValues","marcXml","iso2709","json","it","service","to","be","an","and","respondTo","serialize","throw","data","conversionFormats","MARCXML","equal","ISO2709","unserialize","record","equalsTo","ConversionError"],"mappings":";;AA4BA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AAjCA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AASAA,QAAQ,CAAC,qBAAD,EAAwB,MAAM;AACpC,QAAM;AAACC,IAAAA;AAAD,MAAe,qBAAe;AAACC,IAAAA,IAAI,EAAE,CACzCC,SADyC,EAEzC,IAFyC,EAGzC,IAHyC,EAIzC,eAJyC,EAKzC,YALyC;AAAP,GAAf,CAArB;AAOA,QAAMC,iBAAiB,GAAG,0BAA1B;AAEA,QAAMC,UAAU,GAAG,IAAIC,sBAAJ,CAAeC,IAAI,CAACC,KAAL,CAAWP,UAAU,CAAC;AAACQ,IAAAA,UAAU,EAAE,CAAC,OAAD;AAAb,GAAD,CAArB,CAAf,EAAgE;AAACC,IAAAA,cAAc,EAAE;AAAjB,GAAhE,CAAnB;AACA,QAAMC,OAAO,GAAGV,UAAU,CAAC;AAACQ,IAAAA,UAAU,EAAE,CAAC,UAAD;AAAb,GAAD,CAA1B;AACA,QAAMG,OAAO,GAAGX,UAAU,CAAC;AAACQ,IAAAA,UAAU,EAAE,CAAC,WAAD;AAAb,GAAD,CAA1B;AACA,QAAMI,IAAI,GAAGZ,UAAU,CAAC;AAACQ,IAAAA,UAAU,EAAE,CAAC,OAAD;AAAb,GAAD,CAAvB;AAEAT,EAAAA,QAAQ,CAAC,SAAD,EAAY,MAAM;AACxBc,IAAAA,EAAE,CAAC,mCAAD,EAAsC,MAAM;AAC5C,YAAMC,OAAO,GAAG,0BAAhB;AACA,wBAAOA,OAAP,EAAgBC,EAAhB,CAAmBC,EAAnB,CAAsBC,EAAtB,CAAyB,QAAzB,EAAmCC,GAAnC,CACGC,SADH,CACa,WADb,EAEGA,SAFH,CAEa,aAFb;AAGD,KALC,CAAF;AAMD,GAPO,CAAR;AASApB,EAAAA,QAAQ,CAAC,YAAD,EAAe,MAAM;AAC3Bc,IAAAA,EAAE,CAAC,4CAAD,EAA+C,MAAM;AACrD,wBAAOV,iBAAiB,CAACiB,SAAzB,EAAoCL,EAApC,CAAuCM,KAAvC;AACD,KAFC,CAAF;AAIAR,IAAAA,EAAE,CAAC,6BAAD,EAAgC,MAAM;AACtC,YAAMS,IAAI,GAAGnB,iBAAiB,CAACiB,SAAlB,CAA4BhB,UAA5B,EAAwCmB,6BAAkBC,OAA1D,CAAb;AACA,wBAAOF,IAAP,EAAaP,EAAb,CAAgBU,KAAhB,CAAsBf,OAAtB;AACD,KAHC,CAAF;AAKAG,IAAAA,EAAE,CAAC,6BAAD,EAAgC,MAAM;AACtC,YAAMS,IAAI,GAAGnB,iBAAiB,CAACiB,SAAlB,CAA4BhB,UAA5B,EAAwCmB,6BAAkBG,OAA1D,CAAb;AACA,wBAAOJ,IAAP,EAAaP,EAAb,CAAgBU,KAAhB,CAAsBd,OAAtB;AACD,KAHC,CAAF;AAKAE,IAAAA,EAAE,CAAC,0BAAD,EAA6B,MAAM;AACnC,YAAMS,IAAI,GAAGnB,iBAAiB,CAACiB,SAAlB,CAA4BhB,UAA5B,EAAwCmB,6BAAkBjB,IAA1D,CAAb;AACA,wBAAOgB,IAAP,EAAaP,EAAb,CAAgBU,KAAhB,CAAsBb,IAAtB;AACD,KAHC,CAAF;AAID,GAnBO,CAAR;AAqBAb,EAAAA,QAAQ,CAAC,cAAD,EAAiB,MAAM;AAC7Bc,IAAAA,EAAE,CAAC,4CAAD,EAA+C,MAAM;AACrD,wBAAOV,iBAAiB,CAACwB,WAAzB,EAAsCZ,EAAtC,CAAyCM,KAAzC;AACD,KAFC,CAAF;AAIAR,IAAAA,EAAE,CAAC,iCAAD,EAAoC,YAAY;AAChD,YAAMe,MAAM,GAAG,MAAMzB,iBAAiB,CAACwB,WAAlB,CAA8BjB,OAA9B,EAAuCa,6BAAkBC,OAAzD,CAArB;AACA,wBAAOI,MAAM,CAACC,QAAP,CAAgBzB,UAAhB,CAAP,EAAoCW,EAApC,CAAuCU,KAAvC,CAA6C,IAA7C;AACD,KAHC,CAAF;AAKAZ,IAAAA,EAAE,CAAC,iCAAD,EAAoC,MAAM;AAC1C,YAAMe,MAAM,GAAGzB,iBAAiB,CAACwB,WAAlB,CAA8BhB,OAA9B,EAAuCY,6BAAkBG,OAAzD,CAAf;AACA,wBAAOE,MAAM,CAACC,QAAP,CAAgBzB,UAAhB,CAAP,EAAoCW,EAApC,CAAuCU,KAAvC,CAA6C,IAA7C;AACD,KAHC,CAAF;AAKAZ,IAAAA,EAAE,CAAC,8BAAD,EAAiC,MAAM;AACvC,YAAMe,MAAM,GAAGzB,iBAAiB,CAACwB,WAAlB,CAA8Bf,IAA9B,EAAoCW,6BAAkBjB,IAAtD,CAAf;AACA,wBAAOsB,MAAM,CAACC,QAAP,CAAgBzB,UAAhB,CAAP,EAAoCW,EAApC,CAAuCU,KAAvC,CAA6C,IAA7C;AACD,KAHC,CAAF;AAKAZ,IAAAA,EAAE,CAAC,2DAAD,EAA8D,MAAM;AACpE,wBAAO,MAAM;AACXV,QAAAA,iBAAiB,CAACwB,WAAlB,CAA8B,EAA9B,EAAkCJ,6BAAkBjB,IAApD;AACD,OAFD,EAEGS,EAFH,CAEMM,KAFN,CAEYS,qBAFZ;AAGD,KAJC,CAAF;AAKD,GAzBO,CAAR;AA0BD,CAvEO,CAAR","sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020-2021 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport {expect} from 'chai';\nimport fixtureFactory from '@natlibfi/fixura';\nimport {MarcRecord} from '@natlibfi/marc-record';\nimport {Error as ConversionError} from '@natlibfi/melinda-commons';\nimport createConversionService from './conversion';\nimport {conversionFormats} from '../constants';\n\ndescribe('services/conversion', () => {\n  const {getFixture} = fixtureFactory({root: [\n    __dirname,\n    '..',\n    '..',\n    'test-fixtures',\n    'conversion'\n  ]});\n  const conversionService = createConversionService();\n\n  const marcRecord = new MarcRecord(JSON.parse(getFixture({components: ['json1']})), {subfieldValues: false});\n  const marcXml = getFixture({components: ['marcxml1']});\n  const iso2709 = getFixture({components: ['iso2709_1']});\n  const json = getFixture({components: ['json1']});\n\n  describe('factory', () => {\n    it('Should create the expected object', () => {\n      const service = createConversionService();\n      expect(service).to.be.an('object').and\n        .respondTo('serialize')\n        .respondTo('unserialize');\n    });\n  });\n\n  describe('#serialize', () => {\n    it('Should throw because of unsupported format', () => {\n      expect(conversionService.serialize).to.throw();\n    });\n\n    it('Should serialize to MARCXML', () => {\n      const data = conversionService.serialize(marcRecord, conversionFormats.MARCXML);\n      expect(data).to.equal(marcXml);\n    });\n\n    it('Should serialize to ISO2709', () => {\n      const data = conversionService.serialize(marcRecord, conversionFormats.ISO2709);\n      expect(data).to.equal(iso2709);\n    });\n\n    it('Should serialize to JSON', () => {\n      const data = conversionService.serialize(marcRecord, conversionFormats.JSON);\n      expect(data).to.equal(json);\n    });\n  });\n\n  describe('#unserialize', () => {\n    it('Should throw because of unsupported format', () => {\n      expect(conversionService.unserialize).to.throw();\n    });\n\n    it('Should unserialize from MARCXML', async () => {\n      const record = await conversionService.unserialize(marcXml, conversionFormats.MARCXML);\n      expect(record.equalsTo(marcRecord)).to.equal(true);\n    });\n\n    it('Should unserialize from ISO2709', () => {\n      const record = conversionService.unserialize(iso2709, conversionFormats.ISO2709);\n      expect(record.equalsTo(marcRecord)).to.equal(true);\n    });\n\n    it('Should unserialize from JSON', () => {\n      const record = conversionService.unserialize(json, conversionFormats.JSON);\n      expect(record.equalsTo(marcRecord)).to.equal(true);\n    });\n\n    it('Should throw because the record could not be unserialized', () => {\n      expect(() => {\n        conversionService.unserialize('', conversionFormats.JSON);\n      }).to.throw(ConversionError);\n    });\n  });\n});\n"],"file":"conversion.spec.js"}
+{"version":3,"sources":["../../src/helpers/conversion.spec.js"],"names":["describe","getFixture","root","__dirname","conversionService","marcRecord","MarcRecord","JSON","parse","components","subfieldValues","marcXml","iso2709","json","alephSeq","it","service","to","be","an","and","respondTo","serialize","throw","data","CONVERSION_FORMATS","MARCXML","equal","ISO2709","ALEPHSEQ","unserialize","record","equalsTo","ConversionError"],"mappings":";;AA4BA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AAjCA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AASAA,QAAQ,CAAC,qBAAD,EAAwB,MAAM;AACpC,QAAM;AAACC,IAAAA;AAAD,MAAe,qBAAe;AAACC,IAAAA,IAAI,EAAE,CACzCC,SADyC,EAEzC,IAFyC,EAGzC,IAHyC,EAIzC,eAJyC,EAKzC,YALyC;AAAP,GAAf,CAArB;AAOA,QAAMC,iBAAiB,GAAG,0BAA1B;AAEA,QAAMC,UAAU,GAAG,IAAIC,sBAAJ,CAAeC,IAAI,CAACC,KAAL,CAAWP,UAAU,CAAC;AAACQ,IAAAA,UAAU,EAAE,CAAC,OAAD;AAAb,GAAD,CAArB,CAAf,EAAgE;AAACC,IAAAA,cAAc,EAAE;AAAjB,GAAhE,CAAnB;AACA,QAAMC,OAAO,GAAGV,UAAU,CAAC;AAACQ,IAAAA,UAAU,EAAE,CAAC,UAAD;AAAb,GAAD,CAA1B;AACA,QAAMG,OAAO,GAAGX,UAAU,CAAC;AAACQ,IAAAA,UAAU,EAAE,CAAC,WAAD;AAAb,GAAD,CAA1B;AACA,QAAMI,IAAI,GAAGZ,UAAU,CAAC;AAACQ,IAAAA,UAAU,EAAE,CAAC,OAAD;AAAb,GAAD,CAAvB;AACA,QAAMK,QAAQ,GAAGb,UAAU,CAAC;AAACQ,IAAAA,UAAU,EAAE,CAAC,kBAAD;AAAb,GAAD,CAA3B;AAGAT,EAAAA,QAAQ,CAAC,SAAD,EAAY,MAAM;AACxBe,IAAAA,EAAE,CAAC,mCAAD,EAAsC,MAAM;AAC5C,YAAMC,OAAO,GAAG,0BAAhB;AACA,wBAAOA,OAAP,EAAgBC,EAAhB,CAAmBC,EAAnB,CAAsBC,EAAtB,CAAyB,QAAzB,EAAmCC,GAAnC,CACGC,SADH,CACa,WADb,EAEGA,SAFH,CAEa,aAFb;AAGD,KALC,CAAF;AAMD,GAPO,CAAR;AASArB,EAAAA,QAAQ,CAAC,YAAD,EAAe,MAAM;AAC3Be,IAAAA,EAAE,CAAC,4CAAD,EAA+C,MAAM;AACrD,wBAAOX,iBAAiB,CAACkB,SAAzB,EAAoCL,EAApC,CAAuCM,KAAvC;AACD,KAFC,CAAF;AAIAR,IAAAA,EAAE,CAAC,6BAAD,EAAgC,MAAM;AACtC,YAAMS,IAAI,GAAGpB,iBAAiB,CAACkB,SAAlB,CAA4BjB,UAA5B,EAAwCoB,8BAAmBC,OAA3D,CAAb;AACA,wBAAOF,IAAP,EAAaP,EAAb,CAAgBU,KAAhB,CAAsBhB,OAAtB;AACD,KAHC,CAAF;AAKAI,IAAAA,EAAE,CAAC,6BAAD,EAAgC,MAAM;AACtC,YAAMS,IAAI,GAAGpB,iBAAiB,CAACkB,SAAlB,CAA4BjB,UAA5B,EAAwCoB,8BAAmBG,OAA3D,CAAb;AACA,wBAAOJ,IAAP,EAAaP,EAAb,CAAgBU,KAAhB,CAAsBf,OAAtB;AACD,KAHC,CAAF;AAKAG,IAAAA,EAAE,CAAC,0BAAD,EAA6B,MAAM;AACnC,YAAMS,IAAI,GAAGpB,iBAAiB,CAACkB,SAAlB,CAA4BjB,UAA5B,EAAwCoB,8BAAmBlB,IAA3D,CAAb;AACA,wBAAOiB,IAAP,EAAaP,EAAb,CAAgBU,KAAhB,CAAsBd,IAAtB;AACD,KAHC,CAAF;AAKAE,IAAAA,EAAE,CAAC,sCAAD,EAAyC,MAAM;AAC/C,YAAMS,IAAI,GAAGpB,iBAAiB,CAACkB,SAAlB,CAA4BjB,UAA5B,EAAwCoB,8BAAmBI,QAA3D,CAAb;AACA,wBAAOL,IAAP,EAAaP,EAAb,CAAgBU,KAAhB,CAAsBb,QAAtB;AACD,KAHC,CAAF;AAKD,GAzBO,CAAR;AA2BAd,EAAAA,QAAQ,CAAC,cAAD,EAAiB,MAAM;AAC7Be,IAAAA,EAAE,CAAC,4CAAD,EAA+C,MAAM;AACrD,wBAAOX,iBAAiB,CAAC0B,WAAzB,EAAsCb,EAAtC,CAAyCM,KAAzC;AACD,KAFC,CAAF;AAIAR,IAAAA,EAAE,CAAC,iCAAD,EAAoC,YAAY;AAChD,YAAMgB,MAAM,GAAG,MAAM3B,iBAAiB,CAAC0B,WAAlB,CAA8BnB,OAA9B,EAAuCc,8BAAmBC,OAA1D,CAArB;AACA,wBAAOK,MAAM,CAACC,QAAP,CAAgB3B,UAAhB,CAAP,EAAoCY,EAApC,CAAuCU,KAAvC,CAA6C,IAA7C;AACD,KAHC,CAAF;AAKAZ,IAAAA,EAAE,CAAC,iCAAD,EAAoC,MAAM;AAC1C,YAAMgB,MAAM,GAAG3B,iBAAiB,CAAC0B,WAAlB,CAA8BlB,OAA9B,EAAuCa,8BAAmBG,OAA1D,CAAf;AACA,wBAAOG,MAAM,CAACC,QAAP,CAAgB3B,UAAhB,CAAP,EAAoCY,EAApC,CAAuCU,KAAvC,CAA6C,IAA7C;AACD,KAHC,CAAF;AAKAZ,IAAAA,EAAE,CAAC,8BAAD,EAAiC,MAAM;AACvC,YAAMgB,MAAM,GAAG3B,iBAAiB,CAAC0B,WAAlB,CAA8BjB,IAA9B,EAAoCY,8BAAmBlB,IAAvD,CAAf;AACA,wBAAOwB,MAAM,CAACC,QAAP,CAAgB3B,UAAhB,CAAP,EAAoCY,EAApC,CAAuCU,KAAvC,CAA6C,IAA7C;AACD,KAHC,CAAF;AAKAZ,IAAAA,EAAE,CAAC,0CAAD,EAA6C,MAAM;AACnD,YAAMgB,MAAM,GAAG3B,iBAAiB,CAAC0B,WAAlB,CAA8BhB,QAA9B,EAAwCW,8BAAmBI,QAA3D,CAAf;AACA,wBAAOE,MAAM,CAACC,QAAP,CAAgB3B,UAAhB,CAAP,EAAoCY,EAApC,CAAuCU,KAAvC,CAA6C,IAA7C;AACD,KAHC,CAAF;AAMAZ,IAAAA,EAAE,CAAC,2DAAD,EAA8D,MAAM;AACpE,wBAAO,MAAM;AACXX,QAAAA,iBAAiB,CAAC0B,WAAlB,CAA8B,EAA9B,EAAkCL,8BAAmBlB,IAArD;AACD,OAFD,EAEGU,EAFH,CAEMM,KAFN,CAEYU,qBAFZ;AAGD,KAJC,CAAF;AAKD,GA/BO,CAAR;AAgCD,CArFO,CAAR","sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020-2021 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport {expect} from 'chai';\nimport fixtureFactory from '@natlibfi/fixura';\nimport {MarcRecord} from '@natlibfi/marc-record';\nimport {Error as ConversionError} from '@natlibfi/melinda-commons';\nimport createConversionService from './conversion';\nimport {CONVERSION_FORMATS} from '../constants';\n\ndescribe('services/conversion', () => {\n  const {getFixture} = fixtureFactory({root: [\n    __dirname,\n    '..',\n    '..',\n    'test-fixtures',\n    'conversion'\n  ]});\n  const conversionService = createConversionService();\n\n  const marcRecord = new MarcRecord(JSON.parse(getFixture({components: ['json1']})), {subfieldValues: false});\n  const marcXml = getFixture({components: ['marcxml1']});\n  const iso2709 = getFixture({components: ['iso2709_1']});\n  const json = getFixture({components: ['json1']});\n  const alephSeq = getFixture({components: ['aleph_sequential']});\n\n\n  describe('factory', () => {\n    it('Should create the expected object', () => {\n      const service = createConversionService();\n      expect(service).to.be.an('object').and\n        .respondTo('serialize')\n        .respondTo('unserialize');\n    });\n  });\n\n  describe('#serialize', () => {\n    it('Should throw because of unsupported format', () => {\n      expect(conversionService.serialize).to.throw();\n    });\n\n    it('Should serialize to MARCXML', () => {\n      const data = conversionService.serialize(marcRecord, CONVERSION_FORMATS.MARCXML);\n      expect(data).to.equal(marcXml);\n    });\n\n    it('Should serialize to ISO2709', () => {\n      const data = conversionService.serialize(marcRecord, CONVERSION_FORMATS.ISO2709);\n      expect(data).to.equal(iso2709);\n    });\n\n    it('Should serialize to JSON', () => {\n      const data = conversionService.serialize(marcRecord, CONVERSION_FORMATS.JSON);\n      expect(data).to.equal(json);\n    });\n\n    it('Should serialize to aleph sequential', () => {\n      const data = conversionService.serialize(marcRecord, CONVERSION_FORMATS.ALEPHSEQ);\n      expect(data).to.equal(alephSeq);\n    });\n\n  });\n\n  describe('#unserialize', () => {\n    it('Should throw because of unsupported format', () => {\n      expect(conversionService.unserialize).to.throw();\n    });\n\n    it('Should unserialize from MARCXML', async () => {\n      const record = await conversionService.unserialize(marcXml, CONVERSION_FORMATS.MARCXML);\n      expect(record.equalsTo(marcRecord)).to.equal(true);\n    });\n\n    it('Should unserialize from ISO2709', () => {\n      const record = conversionService.unserialize(iso2709, CONVERSION_FORMATS.ISO2709);\n      expect(record.equalsTo(marcRecord)).to.equal(true);\n    });\n\n    it('Should unserialize from JSON', () => {\n      const record = conversionService.unserialize(json, CONVERSION_FORMATS.JSON);\n      expect(record.equalsTo(marcRecord)).to.equal(true);\n    });\n\n    it('Should unserialize from Aleph sequential', () => {\n      const record = conversionService.unserialize(alephSeq, CONVERSION_FORMATS.ALEPHSEQ);\n      expect(record.equalsTo(marcRecord)).to.equal(true);\n    });\n\n\n    it('Should throw because the record could not be unserialized', () => {\n      expect(() => {\n        conversionService.unserialize('', CONVERSION_FORMATS.JSON);\n      }).to.throw(ConversionError);\n    });\n  });\n});\n"],"file":"conversion.spec.js"}

package/dist/index.js

@@ -9,8 +9,18 @@ var _exportNames = {
   conversions: true,
   validations: true,
   format: true,
-  logError: true
+  logError: true,
+  createImportJobState: true,
+  createImportJobStateForQuery: true,
+  createRecordResponseItem: true,
+  addRecordResponseItem: true
 };
+Object.defineProperty(exports, "addRecordResponseItem", {
+  enumerable: true,
+  get: function () {
+    return _utils.addRecordResponseItem;
+  }
+});
 Object.defineProperty(exports, "amqpFactory", {
   enumerable: true,
   get: function () {
@@ -23,6 +33,24 @@ Object.defineProperty(exports, "conversions", {
     return _conversion.default;
   }
 });
+Object.defineProperty(exports, "createImportJobState", {
+  enumerable: true,
+  get: function () {
+    return _utils.createImportJobState;
+  }
+});
+Object.defineProperty(exports, "createImportJobStateForQuery", {
+  enumerable: true,
+  get: function () {
+    return _utils.createImportJobStateForQuery;
+  }
+});
+Object.defineProperty(exports, "createRecordResponseItem", {
+  enumerable: true,
+  get: function () {
+    return _utils.createRecordResponseItem;
+  }
+});
 exports.format = void 0;
 Object.defineProperty(exports, "logError", {
   enumerable: true,

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/index.js"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4BA;;AACA;;AACA;;AACA;;AACA;;;;AACA;;AAGA;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA","sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020-2021 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport mongoFactory from './mongo';\nimport amqpFactory from './amqp';\nimport conversions from './helpers/conversion';\nimport validations from './helpers/validation';\nimport * as format from './helpers/format';\nimport {logError} from './utils';\n\nexport {mongoFactory, amqpFactory, conversions, validations, format, logError};\nexport * from './constants';\n"],"file":"index.js"}
+{"version":3,"sources":["../src/index.js"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4BA;;AACA;;AACA;;AACA;;AACA;;;;AACA;;AAGA;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA","sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020-2021 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport mongoFactory from './mongo';\nimport amqpFactory from './amqp';\nimport conversions from './helpers/conversion';\nimport validations from './helpers/validation';\nimport * as format from './helpers/format';\nimport {logError, createImportJobState, createImportJobStateForQuery, createRecordResponseItem, addRecordResponseItem} from './utils';\n\nexport {mongoFactory, amqpFactory, conversions, validations, format, logError, createImportJobState, createImportJobStateForQuery, createRecordResponseItem, addRecordResponseItem};\nexport * from './constants';\n"],"file":"index.js"}

package/dist/mongo.js

@@ -57,6 +57,7 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
   "cataloger":"xxx0000",
   "oCatalogerIn":"xxx0000"
   "operation":"UPDATE",
+  "operations":["UPDATE", "CREATE"]
   "contentType":"application/json",
   "recordLoadParams": {
     "pActiveLibrary": "XXX00",
@@ -66,6 +67,10 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
     "pOldNew": "NEW"
   },
   "queueItemState":"PENDING_QUEUING",
+  "importJobState": {
+    "CREATE": "EMPTY",
+    "UPDATE": "EMPTY"
+  },
   "creationTime":"2020-01-01T00:00:00.000Z",
   "modificationTime":"2020-01-01T00:00:01.000Z",
   "handledIds": []
@@ -86,6 +91,7 @@ async function _default(MONGO_URI, collection) {
     createPrio,
     createBulk,
     checkAndSetState,
+    checkAndSetImportJobState,
     query,
     queryById,
     remove,
@@ -94,8 +100,11 @@ async function _default(MONGO_URI, collection) {
     getOne,
     getStream,
     setState,
+    setImportJobState,
     pushIds,
-    pushMessages
+    pushMessages,
+    setOperation,
+    setOperations
   };
 
   async function createPrio({
@@ -103,22 +112,24 @@ async function _default(MONGO_URI, collection) {
     cataloger,
     oCatalogerIn,
     operation,
-    noop = undefined,
-    unique = undefined,
-    prio = true
+    operationSettings
   }) {
     const time = (0, _moment.default)().toDate();
     const newQueueItem = {
       correlationId,
       cataloger,
       operation,
+      operations: [operation],
       oCatalogerIn,
       operationSettings: {
-        unique,
-        noop,
-        prio
+        prio: true,
+        ...operationSettings
       },
       queueItemState: _constants.QUEUE_ITEM_STATE.VALIDATOR.PENDING_VALIDATION,
+      importJobState: {
+        CREATE: _constants.IMPORT_JOB_STATE.EMPTY,
+        UPDATE: _constants.IMPORT_JOB_STATE.EMPTY
+      },
       creationTime: time,
       modificationTime: time,
       handledIds: [],
@@ -140,7 +151,7 @@ async function _default(MONGO_URI, collection) {
     }
   }
 
-  function createBulk({
+  async function createBulk({
     correlationId,
     cataloger,
     oCatalogerIn,
@@ -148,7 +159,7 @@ async function _default(MONGO_URI, collection) {
     contentType,
     recordLoadParams,
     stream,
-    prio = false
+    operationSettings
   }) {
     const time = (0, _moment.default)().toDate();
     const newQueueItem = {
@@ -156,37 +167,61 @@ async function _default(MONGO_URI, collection) {
       cataloger,
       oCatalogerIn,
       operation,
+      operations: [operation],
       operationSettings: {
-        prio
+        prio: false,
+        ...operationSettings
       },
       contentType,
       recordLoadParams,
       queueItemState: stream ? _constants.QUEUE_ITEM_STATE.VALIDATOR.UPLOADING : _constants.QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS,
+      importJobState: {
+        CREATE: _constants.IMPORT_JOB_STATE.EMPTY,
+        UPDATE: _constants.IMPORT_JOB_STATE.EMPTY
+      },
       creationTime: time,
       modificationTime: time,
       handledIds: [],
       rejectedIds: []
-    };
-
-    try {
-      // No await here, promises later
-      db.collection(collection).insertOne(newQueueItem);
-      logger.info(`New BULK queue item for ${operation} ${correlationId} has been made in ${collection}!`);
-    } catch (error) {
-      (0, _utils.logError)(error);
-      throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR);
+    }; // eslint-disable-next-line functional/no-conditional-statement
+
+    if (stream) {
+      try {
+        // No await here, promises later
+        db.collection(collection).insertOne(newQueueItem);
+        logger.info(`New BULK queue item for ${operation} ${correlationId} has been made in ${collection}!`);
+        return new Promise((resolve, reject) => {
+          const outputStream = gridFSBucket.openUploadStream(correlationId);
+          stream.on('error', reject).on('data', chunk => outputStream.write(chunk)).on('end', () => outputStream.end(undefined, undefined, () => {
+            resolve(correlationId);
+          }));
+        });
+      } catch (error) {
+        (0, _utils.logError)(error);
+        throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR);
+      }
     }
 
+    logger.debug(`No stream`); // eslint-disable-next-line functional/no-conditional-statement
+
     if (!stream) {
-      return correlationId;
-    }
+      try {
+        const result = await db.collection(collection).insertOne(newQueueItem);
+
+        if (result.acknowledged) {
+          logger.info(`New noStream BULK queue item for ${operation} ${correlationId} has been made in ${collection}`);
+          return {
+            correlationId,
+            queueItemState: _constants.QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS
+          };
+        }
 
-    return new Promise((resolve, reject) => {
-      const outputStream = gridFSBucket.openUploadStream(correlationId);
-      stream.on('error', reject).on('data', chunk => outputStream.write(chunk)).on('end', () => outputStream.end(undefined, undefined, () => {
-        resolve(correlationId);
-      }));
-    });
+        throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR);
+      } catch (error) {
+        (0, _utils.logError)(error);
+        throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR);
+      }
+    }
   } // Check state that the queueItem has not waited too long and set state
 
 
@@ -198,7 +233,9 @@ async function _default(MONGO_URI, collection) {
   }) {
     // checkTimeOut returns true, if queueItem is fresher than 1 minute and it's state is not ABORT/ERROR
     // otherwise it sets queueItem to state ABORT (408, 'Timeout')
-    const timeOut = await checkTimeOut(correlationId);
+    const timeOut = await checkTimeOut({
+      correlationId
+    });
 
     if (timeOut) {
       return setState({
@@ -209,6 +246,36 @@ async function _default(MONGO_URI, collection) {
       });
     }
 
+    return false;
+  } // Check state that the queueItem has not waited too long and set state
+
+
+  async function checkAndSetImportJobState({
+    correlationId,
+    operation,
+    importJobState,
+    errorMessage = '',
+    errorStatus = ''
+  }) {
+    // checkTimeOut returns true, if queueItem is fresher than 1 minute and it's state is not ABORT/ERROR
+    // otherwise it sets queueItem to state ABORT (408, 'Timeout')
+    logger.debug(`${correlationId}, ${importJobState}, ${operation}`);
+    const timeOut = await checkTimeOut({
+      correlationId,
+      operation,
+      importJobState
+    });
+
+    if (timeOut) {
+      return setImportJobState({
+        correlationId,
+        operation,
+        importJobState,
+        errorMessage,
+        errorStatus
+      });
+    }
+
     return false;
   }
 
@@ -222,16 +289,23 @@ async function _default(MONGO_URI, collection) {
     return result;
   }
 
-  async function queryById(correlationId, checkModTime) {
+  async function queryById({
+    correlationId,
+    checkModTime = false
+  }) {
     const result = await db.collection(collection).findOne({
       correlationId
     });
 
     if (checkModTime) {
-      const timeOut = await checkTimeOut(correlationId);
+      const timeOut = await checkTimeOut({
+        correlationId
+      });
 
       if (timeOut) {
-        return queryById(correlationId);
+        return queryById({
+          correlationId
+        });
       }
 
       return result;
@@ -244,15 +318,18 @@ async function _default(MONGO_URI, collection) {
   // If the state is already ABORT or ERROR return false
 
 
-  async function checkTimeOut(correlationId) {
+  async function checkTimeOut({
+    correlationId
+  }) {
     const {
       modificationTime,
-      queueitemState: oldState
+      queueitemState: oldState,
+      importJobState
     } = await db.collection(collection).findOne({
       correlationId
-    });
+    }); // should we check for DONE too?
 
-    if ([_constants.QUEUE_ITEM_STATE.ABORT, _constants.QUEUE_ITEM_STATE.ERROR].includes(oldState)) {
+    if ([_constants.QUEUE_ITEM_STATE.ABORT, _constants.QUEUE_ITEM_STATE.ERROR, _constants.QUEUE_ITEM_STATE.DONE].includes(oldState)) {
       logger.silly(`${correlationId} has already state: ${oldState}`);
       return false;
     }
@@ -261,6 +338,26 @@ async function _default(MONGO_URI, collection) {
     logger.silly(`timeOut @ ${timeoutTime}`);
 
     if (timeoutTime.isBefore()) {
+      const finalImportJobStates = [_constants.IMPORT_JOB_STATE.ABORT, _constants.IMPORT_JOB_STATE.DONE, _constants.IMPORT_JOB_STATE.EMPTY, _constants.IMPORT_JOB_STATE.ERROR];
+
+      if (!finalImportJobStates.includes(importJobState.CREATE)) {
+        // eslint-disable-line
+        await setImportJobState({
+          correlationId,
+          operation: 'CREATE',
+          importJobState: _constants.IMPORT_JOB_STATE.ABORT
+        });
+      }
+
+      if (!finalImportJobStates.includes(importJobState.UPDATE)) {
+        // eslint-disable-line
+        await setImportJobState({
+          correlationId,
+          operation: 'UPDATE',
+          importJobState: _constants.IMPORT_JOB_STATE.ABORT
+        });
+      }
+
       await setState({
         correlationId,
         state: _constants.QUEUE_ITEM_STATE.ABORT,
@@ -340,30 +437,43 @@ async function _default(MONGO_URI, collection) {
     }
 
     return true;
-  }
+  } // eslint-disable-next-line max-statements
+
 
   function getOne({
-    operation,
-    queueItemState
+    queueItemState,
+    importJobState = undefined
   }) {
-    const clean = {
+    logger.silly(`queueItemState: ${queueItemState}, importJobState: ${JSON.stringify(importJobState)}`);
+    const cleanQueueItemState = queueItemState ? {
       queueItemState: (0, _mongoSanitize.default)(queueItemState)
-    };
+    } : undefined;
 
     try {
-      if (operation === undefined) {
-        logger.silly(`Checking DB ${collection} for ${JSON.stringify(clean.queueItemState)}`);
-        return db.collection(collection).findOne({ ...clean
+      // Just queueItemState
+      if (queueItemState && importJobState === undefined) {
+        logger.silly(`Checking DB ${collection} for just ${JSON.stringify(cleanQueueItemState.queueItemState)}`);
+        return db.collection(collection).findOne({ ...cleanQueueItemState
+        });
+      } // importJobState
+
+
+      if (importJobState && queueItemState === undefined) {
+        logger.silly(`Checking DB ${collection} for ${JSON.stringify(importJobState)}`);
+        return db.collection(collection).findOne({ ...importJobState
+        });
+      } // importJobState and queueItemState
+
+
+      if (importJobState && queueItemState) {
+        logger.silly(`Checking DB ${collection} for ${queueItemState} and ${JSON.stringify(importJobState)}`);
+        return db.collection(collection).findOne({ ...cleanQueueItemState,
+          ...importJobState
         });
       }
 
-      const clean2 = {
-        operation: (0, _mongoSanitize.default)(operation)
-      };
-      logger.silly(`Checking DB ${collection} for ${clean.queueItemState} + ${clean2.operation}`);
-      return db.collection(collection).findOne({ ...clean,
-        ...clean2
-      });
+      logger.debug(`getOne not working!`);
+      throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR, 'Missing parameters, cannot get queueItem');
     } catch (error) {
       (0, _utils.logError)(error);
     }
@@ -466,5 +576,102 @@ async function _default(MONGO_URI, collection) {
       returnNewDocument: true
     });
   }
+
+  function setImportJobState({
+    correlationId,
+    operation,
+    importJobState
+  }) {
+    // should this also get importJobState as previously created object? {}
+    logger.info(`Setting queue-item importJobState: {${operation}: ${importJobState}} for ${correlationId}`);
+    const cleanCorrelationId = (0, _mongoSanitize.default)(correlationId);
+    const cleanImportJobState = (0, _mongoSanitize.default)(importJobState);
+
+    if (!(operation in _constants.OPERATIONS)) {
+      throw new _melindaCommons.Error('400', 'Invalid operation for import job state');
+    }
+
+    if (!(cleanImportJobState in _constants.IMPORT_JOB_STATE)) {
+      throw new _melindaCommons.Error('400', 'Invalid import job state');
+    }
+
+    const newJobState = operation === _constants.OPERATIONS.CREATE ? {
+      'importJobState.CREATE': cleanImportJobState
+    } : {
+      'importJobState.UPDATE': cleanImportJobState
+    };
+    return db.collection(collection).findOneAndUpdate({
+      correlationId: cleanCorrelationId
+    }, {
+      $set: { ...newJobState,
+        modificationTime: (0, _moment.default)().toDate()
+      }
+    }, {
+      projection: {
+        _id: 0
+      },
+      returnNewDocument: true
+    });
+  }
+
+  async function setOperation({
+    correlationId,
+    operation
+  }) {
+    const newOperation = operation;
+    const {
+      operation: oldOperation
+    } = await db.collection(collection).findOne({
+      correlationId
+    });
+    logger.info(`Setting queue-item operation from ${oldOperation} to ${newOperation} for ${correlationId} to ${collection}`);
+    const cleanCorrelationId = (0, _mongoSanitize.default)(correlationId);
+    const cleanNewOperation = (0, _mongoSanitize.default)(newOperation);
+    const result = await db.collection(collection).findOneAndUpdate({
+      correlationId: cleanCorrelationId
+    }, {
+      $set: {
+        operation: cleanNewOperation,
+        modificationTime: (0, _moment.default)().toDate()
+      }
+    }, {
+      projection: {
+        _id: 0
+      },
+      returnNewDocument: true
+    });
+    return result.ok; // logger.debug(JSON.stringify(result));
+  }
+
+  async function setOperations({
+    correlationId,
+    addOperation,
+    removeOperation = undefined
+  }) {
+    const cleanCorrelationId = (0, _mongoSanitize.default)(correlationId); //logger.info(`Setting queue-item operation from ${JSON.stringify(oldOperations)} by adding ${addOperation} and removing ${removeOperation} for ${correlationId} to ${collection}`);
+
+    const queueItem = await db.collection(collection).findOne({
+      correlationId: cleanCorrelationId
+    });
+    logger.debug(`We fould a queueItem: ${JSON.stringify(queueItem)}`);
+    const oldOperations = queueItem.operations;
+    logger.info(`Setting queue-item operation from ${JSON.stringify(oldOperations)} by adding ${addOperation} and removing ${removeOperation} for ${correlationId} to ${collection}`);
+    const operationsAfterRemove = oldOperations.filter(operation => operation !== removeOperation);
+    const operationsAfterRemoveAndAdd = [addOperation, ...operationsAfterRemove];
+    const result = await db.collection(collection).findOneAndUpdate({
+      correlationId: cleanCorrelationId
+    }, {
+      $set: {
+        operations: operationsAfterRemoveAndAdd,
+        modificationTime: (0, _moment.default)().toDate()
+      }
+    }, {
+      projection: {
+        _id: 0
+      },
+      returnNewDocument: true
+    });
+    return result.ok; // logger.debug(JSON.stringify(result));
+  }
 }
 //# sourceMappingURL=mongo.js.map

package/dist/mongo.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/mongo.js"],"names":["MONGO_URI","collection","logger","client","MongoClient","connect","useNewUrlParser","useUnifiedTopology","db","gridFSBucket","GridFSBucket","bucketName","createPrio","createBulk","checkAndSetState","query","queryById","remove","readContent","removeContent","getOne","getStream","setState","pushIds","pushMessages","correlationId","cataloger","oCatalogerIn","operation","noop","undefined","unique","prio","time","toDate","newQueueItem","operationSettings","queueItemState","QUEUE_ITEM_STATE","VALIDATOR","PENDING_VALIDATION","creationTime","modificationTime","handledIds","rejectedIds","result","insertOne","acknowledged","info","ApiError","httpStatus","INTERNAL_SERVER_ERROR","error","contentType","recordLoadParams","stream","UPLOADING","WAITING_FOR_RECORDS","Promise","resolve","reject","outputStream","openUploadStream","on","chunk","write","end","state","errorMessage","errorStatus","timeOut","checkTimeOut","params","find","projection","_id","toArray","debug","length","checkModTime","findOne","queueitemState","oldState","ABORT","ERROR","includes","silly","timeoutTime","add","isBefore","REQUEST_TIMEOUT","JSON","stringify","clean","noContent","deleteOne","err","MongoDriverError","message","indexOf","readStream","openDownloadStreamByName","NOT_FOUND","delete","clean2","verbose","updateOne","$set","$push","$each","messages","messageField","cleanMessageField","errorString","stateInQueueItemStates","Object","values","stateInQueueItemStatesValidator","stateInQueueItemStatesImporter","IMPORTER","BAD_REQUEST","updateValues","keys","forEach","key","findOneAndUpdate","returnNewDocument"],"mappings":";;;;;;;AA4BA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AAnCA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAWA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAEe,wBAAgBA,SAAhB,EAA2BC,UAA3B,EAAuC;AACpD,QAAMC,MAAM,GAAG,0CAAf,CADoD,CAGpD;;AACA,QAAMC,MAAM,GAAG,MAAMC,qBAAYC,OAAZ,CAAoBL,SAApB,EAA+B;AAACM,IAAAA,eAAe,EAAE,IAAlB;AAAwBC,IAAAA,kBAAkB,EAAE;AAA5C,GAA/B,CAArB;AACA,QAAMC,EAAE,GAAGL,MAAM,CAACK,EAAP,CAAU,UAAV,CAAX;AACA,QAAMC,YAAY,GAAG,IAAIC,qBAAJ,CAAiBF,EAAjB,EAAqB;AAACG,IAAAA,UAAU,EAAEV;AAAb,GAArB,CAArB;AAEA,SAAO;AAACW,IAAAA,UAAD;AAAaC,IAAAA,UAAb;AAAyBC,IAAAA,gBAAzB;AAA2CC,IAAAA,KAA3C;AAAkDC,IAAAA,SAAlD;AAA6DC,IAAAA,MAA7D;AAAqEC,IAAAA,WAArE;AAAkFC,IAAAA,aAAlF;AAAiGC,IAAAA,MAAjG;AAAyGC,IAAAA,SAAzG;AAAoHC,IAAAA,QAApH;AAA8HC,IAAAA,OAA9H;AAAuIC,IAAAA;AAAvI,GAAP;;AAEA,iBAAeZ,UAAf,CAA0B;AAACa,IAAAA,aAAD;AAAgBC,IAAAA,SAAhB;AAA2BC,IAAAA,YAA3B;AAAyCC,IAAAA,SAAzC;AAAoDC,IAAAA,IAAI,GAAGC,SAA3D;AAAsEC,IAAAA,MAAM,GAAGD,SAA/E;AAA0FE,IAAAA,IAAI,GAAG;AAAjG,GAA1B,EAAkI;AAChI,UAAMC,IAAI,GAAG,uBAASC,MAAT,EAAb;AACA,UAAMC,YAAY,GAAG;AACnBV,MAAAA,aADmB;AAEnBC,MAAAA,SAFmB;AAGnBE,MAAAA,SAHmB;AAInBD,MAAAA,YAJmB;AAKnBS,MAAAA,iBAAiB,EAAE;AAACL,QAAAA,MAAD;AAASF,QAAAA,IAAT;AAAeG,QAAAA;AAAf,OALA;AAMnBK,MAAAA,cAAc,EAAEC,4BAAiBC,SAAjB,CAA2BC,kBANxB;AAOnBC,MAAAA,YAAY,EAAER,IAPK;AAQnBS,MAAAA,gBAAgB,EAAET,IARC;AASnBU,MAAAA,UAAU,EAAE,EATO;AAUnBC,MAAAA,WAAW,EAAE;AAVM,KAArB;;AAYA,QAAI;AACF,YAAMC,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B6C,SAA1B,CAAoCX,YAApC,CAArB;;AACA,UAAIU,MAAM,CAACE,YAAX,EAAyB;AACvB7C,QAAAA,MAAM,CAAC8C,IAAP,CAAa,2BAA0BpB,SAAU,IAAGH,aAAc,qBAAoBxB,UAAW,EAAjG;AACA;AACD;;AACD,YAAM,IAAIgD,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD,KAPD,CAOE,OAAOC,KAAP,EAAc;AACd,2BAASA,KAAT;AACA,YAAM,IAAIH,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD;AACF;;AAED,WAAStC,UAAT,CAAoB;AAACY,IAAAA,aAAD;AAAgBC,IAAAA,SAAhB;AAA2BC,IAAAA,YAA3B;AAAyCC,IAAAA,SAAzC;AAAoDyB,IAAAA,WAApD;AAAiEC,IAAAA,gBAAjE;AAAmFC,IAAAA,MAAnF;AAA2FvB,IAAAA,IAAI,GAAG;AAAlG,GAApB,EAA8H;AAC5H,UAAMC,IAAI,GAAG,uBAASC,MAAT,EAAb;AACA,UAAMC,YAAY,GAAG;AACnBV,MAAAA,aADmB;AAEnBC,MAAAA,SAFmB;AAGnBC,MAAAA,YAHmB;AAInBC,MAAAA,SAJmB;AAKnBQ,MAAAA,iBAAiB,EAAE;AAACJ,QAAAA;AAAD,OALA;AAMnBqB,MAAAA,WANmB;AAOnBC,MAAAA,gBAPmB;AAQnBjB,MAAAA,cAAc,EAAEkB,MAAM,GAAGjB,4BAAiBC,SAAjB,CAA2BiB,SAA9B,GAA0ClB,4BAAiBC,SAAjB,CAA2BkB,mBARxE;AASnBhB,MAAAA,YAAY,EAAER,IATK;AAUnBS,MAAAA,gBAAgB,EAAET,IAVC;AAWnBU,MAAAA,UAAU,EAAE,EAXO;AAYnBC,MAAAA,WAAW,EAAE;AAZM,KAArB;;AAcA,QAAI;AACF;AACApC,MAAAA,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B6C,SAA1B,CAAoCX,YAApC;AACAjC,MAAAA,MAAM,CAAC8C,IAAP,CAAa,2BAA0BpB,SAAU,IAAGH,aAAc,qBAAoBxB,UAAW,GAAjG;AACD,KAJD,CAIE,OAAOmD,KAAP,EAAc;AACd,2BAASA,KAAT;AACA,YAAM,IAAIH,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD;;AAED,QAAI,CAACI,MAAL,EAAa;AACX,aAAO9B,aAAP;AACD;;AAED,WAAO,IAAIiC,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtC,YAAMC,YAAY,GAAGpD,YAAY,CAACqD,gBAAb,CAA8BrC,aAA9B,CAArB;AAEA8B,MAAAA,MAAM,CACHQ,EADH,CACM,OADN,EACeH,MADf,EAEGG,EAFH,CAEM,MAFN,EAEcC,KAAK,IAAIH,YAAY,CAACI,KAAb,CAAmBD,KAAnB,CAFvB,EAGGD,EAHH,CAGM,KAHN,EAGa,MAAMF,YAAY,CAACK,GAAb,CAAiBpC,SAAjB,EAA4BA,SAA5B,EAAuC,MAAM;AAC5D6B,QAAAA,OAAO,CAAClC,aAAD,CAAP;AACD,OAFgB,CAHnB;AAMD,KATM,CAAP;AAUD,GA5EmD,CA8EpD;;;AACA,iBAAeX,gBAAf,CAAgC;AAACW,IAAAA,aAAD;AAAgB0C,IAAAA,KAAhB;AAAuBC,IAAAA,YAAY,GAAGtC,SAAtC;AAAiDuC,IAAAA,WAAW,GAAGvC;AAA/D,GAAhC,EAA2G;AACzG;AACA;AACA,UAAMwC,OAAO,GAAG,MAAMC,YAAY,CAAC9C,aAAD,CAAlC;;AACA,QAAI6C,OAAJ,EAAa;AACX,aAAOhD,QAAQ,CAAC;AAACG,QAAAA,aAAD;AAAgB0C,QAAAA,KAAhB;AAAuBC,QAAAA,YAAvB;AAAqCC,QAAAA;AAArC,OAAD,CAAf;AACD;;AACD,WAAO,KAAP;AACD;;AAED,iBAAetD,KAAf,CAAqByD,MAArB,EAA6B;AAC3B,UAAM3B,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BwE,IAA1B,CAA+BD,MAA/B,EAAuC;AAACE,MAAAA,UAAU,EAAE;AAACC,QAAAA,GAAG,EAAE;AAAN;AAAb,KAAvC,EAClBC,OADkB,EAArB;AAEA1E,IAAAA,MAAM,CAAC2E,KAAP,CAAc,iBAAgBhC,MAAM,CAACiC,MAAP,GAAgB,CAAhB,GAAoB,QAApB,GAA+B,YAAa,EAA1E;AACA,WAAOjC,MAAP;AACD;;AAED,iBAAe7B,SAAf,CAAyBS,aAAzB,EAAwCsD,YAAxC,EAAsD;AACpD,UAAMlC,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B+E,OAA1B,CAAkC;AAACvD,MAAAA;AAAD,KAAlC,CAArB;;AACA,QAAIsD,YAAJ,EAAkB;AAChB,YAAMT,OAAO,GAAG,MAAMC,YAAY,CAAC9C,aAAD,CAAlC;;AACA,UAAI6C,OAAJ,EAAa;AACX,eAAOtD,SAAS,CAACS,aAAD,CAAhB;AACD;;AACD,aAAOoB,MAAP;AACD;;AAED,WAAOA,MAAP;AACD,GA3GmD,CA6GpD;AACA;AACA;AACA;;;AAEA,iBAAe0B,YAAf,CAA4B9C,aAA5B,EAA2C;AACzC,UAAM;AAACiB,MAAAA,gBAAD;AAAmBuC,MAAAA,cAAc,EAAEC;AAAnC,QAA+C,MAAM1E,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B+E,OAA1B,CAAkC;AAACvD,MAAAA;AAAD,KAAlC,CAA3D;;AAEA,QAAI,CAACa,4BAAiB6C,KAAlB,EAAyB7C,4BAAiB8C,KAA1C,EAAiDC,QAAjD,CAA0DH,QAA1D,CAAJ,EAAyE;AACvEhF,MAAAA,MAAM,CAACoF,KAAP,CAAc,GAAE7D,aAAc,uBAAsByD,QAAS,EAA7D;AACA,aAAO,KAAP;AACD;;AAED,UAAMK,WAAW,GAAG,qBAAO7C,gBAAP,EAAyB8C,GAAzB,CAA6B,CAA7B,EAAgC,GAAhC,CAApB;AACAtF,IAAAA,MAAM,CAACoF,KAAP,CAAc,aAAYC,WAAY,EAAtC;;AAEA,QAAIA,WAAW,CAACE,QAAZ,EAAJ,EAA4B;AAC1B,YAAMnE,QAAQ,CAAC;AAACG,QAAAA,aAAD;AAAgB0C,QAAAA,KAAK,EAAE7B,4BAAiB6C,KAAxC;AAA+Cd,QAAAA,WAAW,EAAEnB,oBAAWwC,eAAvE;AAAwFtB,QAAAA,YAAY,EAAG,cAAac,QAAS;AAA7H,OAAD,CAAd;AACA,aAAO,KAAP;AACD;;AAED,WAAO,IAAP;AACD;;AAED,iBAAejE,MAAf,CAAsBuD,MAAtB,EAA8B;AAC5BtE,IAAAA,MAAM,CAACoF,KAAP,CAAc,GAAEK,IAAI,CAACC,SAAL,CAAepB,MAAf,CAAuB,EAAvC;AACAtE,IAAAA,MAAM,CAAC8C,IAAP,CAAa,wBAAuB/C,UAAW,SAAQuE,MAAM,CAAC/C,aAAc,EAA5E;AACA,UAAMoE,KAAK,GAAG,4BAASrB,MAAM,CAAC/C,aAAhB,CAAd;AACAvB,IAAAA,MAAM,CAACoF,KAAP,CAAc,wBAAuBK,IAAI,CAACC,SAAL,CAAeC,KAAf,CAAsB,EAA3D;;AAEA,QAAI;AACF;AACA;AACA,YAAMC,SAAS,GAAG,MAAM3E,aAAa,CAACqD,MAAD,CAArC;;AACA,UAAIsB,SAAJ,EAAe;AACb,cAAMtF,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8F,SAA1B,CAAoC;AAACtE,UAAAA,aAAa,EAAEoE;AAAhB,SAApC,CAAN;AACA,eAAO,IAAP;AACD;AACF,KARD,CAQE,OAAOG,GAAP,EAAY;AACZ,UAAIA,GAAG,YAAYC,yBAAnB,EAAqC;AACnC,YAAID,GAAG,CAACE,OAAJ,CAAYC,OAAZ,CAAoB,uBAApB,MAAiD,CAAC,CAAtD,EAAyD;AACvDjG,UAAAA,MAAM,CAACoF,KAAP,CAAc,oDAAmDK,IAAI,CAACC,SAAL,CAAeC,KAAf,CAAsB,SAAQ5F,UAAW,EAA1G;AACA,gBAAMO,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8F,SAA1B,CAAoC;AAACtE,YAAAA,aAAa,EAAEoE;AAAhB,WAApC,CAAN;AACA,iBAAO,IAAP;AACD;;AACD3F,QAAAA,MAAM,CAACkD,KAAP,CAAa4C,GAAG,CAACE,OAAjB;AACA,cAAM,IAAIjD,qBAAJ,CAAaC,oBAAWC,qBAAxB,EAA+C6C,GAAG,CAACE,OAAnD,CAAN;AACD;;AACD,YAAM,IAAIjD,qBAAJ,CAAaC,oBAAWC,qBAAxB,EAA+C6C,GAAG,CAACE,OAAnD,CAAN;AACD;AACF;;AAED,iBAAehF,WAAf,CAA2BO,aAA3B,EAA0C;AACxCvB,IAAAA,MAAM,CAAC8C,IAAP,CAAa,sCAAqCvB,aAAc,OAAMxB,UAAW,EAAjF;AACA,UAAM4F,KAAK,GAAG,4BAASpE,aAAT,CAAd;AACA,UAAMoB,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B+E,OAA1B,CAAkC;AAACvD,MAAAA,aAAa,EAAEoE;AAAhB,KAAlC,CAArB,CAHwC,CAGwC;;AAEhF,QAAIhD,MAAJ,EAAY;AACV,aAAO;AACLQ,QAAAA,WAAW,EAAER,MAAM,CAACQ,WADf;AAEL+C,QAAAA,UAAU,EAAE3F,YAAY,CAAC4F,wBAAb,CAAsCR,KAAtC;AAFP,OAAP;AAID;;AAED,UAAM,IAAI5C,qBAAJ,CAAaC,oBAAWoD,SAAxB,CAAN;AACD;;AAED,iBAAenF,aAAf,CAA6BqD,MAA7B,EAAqC;AACnCtE,IAAAA,MAAM,CAAC8C,IAAP,CAAa,uCAAsCwB,MAAM,CAAC/C,aAAc,OAAMxB,UAAW,EAAzF;AACA,UAAM4F,KAAK,GAAG,4BAASrB,MAAM,CAAC/C,aAAhB,CAAd;AAEA,UAAMoB,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B+E,OAA1B,CAAkC;AAACvD,MAAAA,aAAa,EAAEoE;AAAhB,KAAlC,CAArB,CAJmC,CAI6C;;AAChF3F,IAAAA,MAAM,CAACoF,KAAP,CAAc,+BAA8BK,IAAI,CAACC,SAAL,CAAe/C,MAAf,CAAuB,EAAnE;;AAEA,QAAIA,MAAJ,EAAY;AACV,YAAMpC,YAAY,CAAC8F,MAAb,CAAoBV,KAApB,CAAN;AACA,aAAO,IAAP;AACD;;AAED,WAAO,IAAP;AACD;;AAED,WAASzE,MAAT,CAAgB;AAACQ,IAAAA,SAAD;AAAYS,IAAAA;AAAZ,GAAhB,EAA6C;AAC3C,UAAMwD,KAAK,GAAG;AAACxD,MAAAA,cAAc,EAAE,4BAASA,cAAT;AAAjB,KAAd;;AACA,QAAI;AACF,UAAIT,SAAS,KAAKE,SAAlB,EAA6B;AAC3B5B,QAAAA,MAAM,CAACoF,KAAP,CAAc,eAAcrF,UAAW,QAAO0F,IAAI,CAACC,SAAL,CAAeC,KAAK,CAACxD,cAArB,CAAqC,EAAnF;AACA,eAAO7B,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B+E,OAA1B,CAAkC,EAAC,GAAGa;AAAJ,SAAlC,CAAP;AACD;;AAED,YAAMW,MAAM,GAAG;AAAC5E,QAAAA,SAAS,EAAE,4BAASA,SAAT;AAAZ,OAAf;AACA1B,MAAAA,MAAM,CAACoF,KAAP,CAAc,eAAcrF,UAAW,QAAO4F,KAAK,CAACxD,cAAe,MAAKmE,MAAM,CAAC5E,SAAU,EAAzF;AACA,aAAOpB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B+E,OAA1B,CAAkC,EAAC,GAAGa,KAAJ;AAAW,WAAGW;AAAd,OAAlC,CAAP;AACD,KATD,CASE,OAAOpD,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,WAAS/B,SAAT,CAAmBI,aAAnB,EAAkC;AAChCvB,IAAAA,MAAM,CAAC8C,IAAP,CAAa,iCAAgCvB,aAAc,OAAMxB,UAAW,EAA5E;AACA,UAAM4F,KAAK,GAAG,4BAASpE,aAAT,CAAd;;AACA,QAAI;AACF;AACA,aAAOhB,YAAY,CAAC4F,wBAAb,CAAsCR,KAAtC,CAAP;AACD,KAHD,CAGE,OAAOzC,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAe7B,OAAf,CAAuB;AAACE,IAAAA,aAAD;AAAgBkB,IAAAA,UAAhB;AAA4BC,IAAAA;AAA5B,GAAvB,EAAiE;AAC/D1C,IAAAA,MAAM,CAACuG,OAAP,CAAgB,aAAY9D,UAAU,CAACmC,MAAO,sBAAqBlC,WAAW,CAACkC,MAAO,KAAIrD,aAAc,OAAMxB,UAAW,EAAzH;AACAC,IAAAA,MAAM,CAAC2E,KAAP,CAAc,QAAOlC,UAAU,CAACmC,MAAO,MAAKa,IAAI,CAACC,SAAL,CAAejD,UAAf,CAA2B,iBAAgBC,WAAW,CAACkC,MAAO,KAAIa,IAAI,CAACC,SAAL,CAAehD,WAAf,CAA4B,EAA1I;AACA,UAAMiD,KAAK,GAAG,4BAASpE,aAAT,CAAd;AACA,UAAMjB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0ByG,SAA1B,CAAoC;AACxCjF,MAAAA,aAAa,EAAEoE;AADyB,KAApC,EAEH;AACDc,MAAAA,IAAI,EAAE;AACJjE,QAAAA,gBAAgB,EAAE,uBAASR,MAAT;AADd,OADL;AAID0E,MAAAA,KAAK,EAAE;AACLjE,QAAAA,UAAU,EAAE;AAACkE,UAAAA,KAAK,EAAElE;AAAR,SADP;AAELC,QAAAA,WAAW,EAAE;AAACiE,UAAAA,KAAK,EAAEjE;AAAR;AAFR;AAJN,KAFG,CAAN;AAWD;;AAED,iBAAepB,YAAf,CAA4B;AAACC,IAAAA,aAAD;AAAgBqF,IAAAA,QAAhB;AAA0BC,IAAAA,YAAY,GAAG;AAAzC,GAA5B,EAAkF;AAChF7G,IAAAA,MAAM,CAACuG,OAAP,CAAgB,kBAAiBK,QAAQ,CAAChC,MAAO,QAAOiC,YAAa,IAAGtF,aAAc,OAAMxB,UAAW,EAAvG;AACAC,IAAAA,MAAM,CAACoF,KAAP,CAAc,aAAYwB,QAAQ,CAAChC,MAAO,MAAKa,IAAI,CAACC,SAAL,CAAekB,QAAf,CAAyB,GAAxE;AACA,UAAMjB,KAAK,GAAG,4BAASpE,aAAT,CAAd;AACA,UAAMuF,iBAAiB,GAAG,4BAASD,YAAT,CAA1B;AACA,UAAMvG,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0ByG,SAA1B,CAAoC;AACxCjF,MAAAA,aAAa,EAAEoE;AADyB,KAApC,EAEH;AACDc,MAAAA,IAAI,EAAE;AACJjE,QAAAA,gBAAgB,EAAE,uBAASR,MAAT;AADd,OADL;AAID0E,MAAAA,KAAK,EAAE;AACL,SAACI,iBAAD,GAAqB;AAACH,UAAAA,KAAK,EAAEC;AAAR;AADhB;AAJN,KAFG,CAAN;AAUD;;AAED,WAASxF,QAAT,CAAkB;AAACG,IAAAA,aAAD;AAAgB0C,IAAAA,KAAhB;AAAuBC,IAAAA,YAAY,GAAGtC,SAAtC;AAAiDuC,IAAAA,WAAW,GAAGvC;AAA/D,GAAlB,EAA6F;AAC3F,UAAMmF,WAAW,GAAG7C,YAAY,IAAIC,WAAhB,GAA+B,qBAAoBD,YAAY,IAAI,EAAG,qBAAoBC,WAAW,IAAI,EAAG,GAA5G,GAAiH,EAArI;AACAnE,IAAAA,MAAM,CAAC8C,IAAP,CAAa,4BAA2BmB,KAAM,QAAO1C,aAAc,GAAEwF,WAAY,OAAMhH,UAAW,EAAlG;AAEA,UAAMiH,sBAAsB,GAAGC,MAAM,CAACC,MAAP,CAAc9E,2BAAd,EAAgC6D,OAAhC,CAAwChC,KAAxC,IAAiD,CAAC,CAAjF;AACA,UAAMkD,+BAA+B,GAAGF,MAAM,CAACC,MAAP,CAAc9E,4BAAiBC,SAA/B,EAA0C4D,OAA1C,CAAkDhC,KAAlD,IAA2D,CAAC,CAApG;AACA,UAAMmD,8BAA8B,GAAGH,MAAM,CAACC,MAAP,CAAc9E,4BAAiBiF,QAA/B,EAAyCpB,OAAzC,CAAiDhC,KAAjD,IAA0D,CAAC,CAAlG;;AACA,QAAI,CAAC+C,sBAAD,IAA2B,CAACG,+BAA5B,IAA+D,CAACC,8BAApE,EAAoG;AAClG,YAAM,IAAIrE,qBAAJ,CAAaC,oBAAWsE,WAAxB,EAAqC,6BAArC,CAAN;AACD;;AAED,UAAM3B,KAAK,GAAG,4BAASpE,aAAT,CAAd;AACA,UAAMgG,YAAY,GAAG;AACnBpF,MAAAA,cAAc,EAAE8B,KADG;AAEnBzB,MAAAA,gBAAgB,EAAE,uBAASR,MAAT,EAFC;AAGnBkC,MAAAA,YAHmB;AAInBC,MAAAA;AAJmB,KAArB,CAZ2F,CAmB3F;AACA;;AACA8C,IAAAA,MAAM,CAACO,IAAP,CAAYD,YAAZ,EAA0BE,OAA1B,CAAkCC,GAAG,IAAIH,YAAY,CAACG,GAAD,CAAZ,KAAsB9F,SAAtB,IAAmC,OAAO2F,YAAY,CAACG,GAAD,CAA/F;AAEA,WAAOpH,EAAE,CAACP,UAAH,CAAcA,UAAd,EACJ4H,gBADI,CAEH;AAACpG,MAAAA,aAAa,EAAEoE;AAAhB,KAFG,EAGH;AAACc,MAAAA,IAAI,EAAEc;AAAP,KAHG,EAIH;AAAC/C,MAAAA,UAAU,EAAE;AAACC,QAAAA,GAAG,EAAE;AAAN,OAAb;AAAuBmD,MAAAA,iBAAiB,EAAE;AAA1C,KAJG,CAAP;AAMD;AACF","sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020-2021 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport {MongoClient, GridFSBucket, MongoDriverError} from 'mongodb';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {Error as ApiError} from '@natlibfi/melinda-commons';\nimport {QUEUE_ITEM_STATE} from './constants';\nimport {logError} from './utils.js';\nimport moment from 'moment';\nimport httpStatus from 'http-status';\nimport sanitize from 'mongo-sanitize';\n\n/* QueueItem:\n{\n  \"correlationId\":\"FOO\",\n  \"cataloger\":\"xxx0000\",\n  \"oCatalogerIn\":\"xxx0000\"\n  \"operation\":\"UPDATE\",\n  \"contentType\":\"application/json\",\n  \"recordLoadParams\": {\n    \"pActiveLibrary\": \"XXX00\",\n    \"pInputFile\": \"filename.seq\",\n    \"pRejectFile\": \"filename.rej\",\n    \"pLogFile\": \"filename.syslog\",\n    \"pOldNew\": \"NEW\"\n  },\n  \"queueItemState\":\"PENDING_QUEUING\",\n  \"creationTime\":\"2020-01-01T00:00:00.000Z\",\n  \"modificationTime\":\"2020-01-01T00:00:01.000Z\",\n  \"handledIds\": []\n}\n*/\n\nexport default async function (MONGO_URI, collection) {\n  const logger = createLogger();\n\n  // Connect to mongo (MONGO)\n  const client = await MongoClient.connect(MONGO_URI, {useNewUrlParser: true, useUnifiedTopology: true});\n  const db = client.db('rest-api');\n  const gridFSBucket = new GridFSBucket(db, {bucketName: collection});\n\n  return {createPrio, createBulk, checkAndSetState, query, queryById, remove, readContent, removeContent, getOne, getStream, setState, pushIds, pushMessages};\n\n  async function createPrio({correlationId, cataloger, oCatalogerIn, operation, noop = undefined, unique = undefined, prio = true}) {\n    const time = moment().toDate();\n    const newQueueItem = {\n      correlationId,\n      cataloger,\n      operation,\n      oCatalogerIn,\n      operationSettings: {unique, noop, prio},\n      queueItemState: QUEUE_ITEM_STATE.VALIDATOR.PENDING_VALIDATION,\n      creationTime: time,\n      modificationTime: time,\n      handledIds: [],\n      rejectedIds: []\n    };\n    try {\n      const result = await db.collection(collection).insertOne(newQueueItem);\n      if (result.acknowledged) {\n        logger.info(`New PRIO queue item for ${operation} ${correlationId} has been made in ${collection}`);\n        return;\n      }\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    } catch (error) {\n      logError(error);\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    }\n  }\n\n  function createBulk({correlationId, cataloger, oCatalogerIn, operation, contentType, recordLoadParams, stream, prio = false}) {\n    const time = moment().toDate();\n    const newQueueItem = {\n      correlationId,\n      cataloger,\n      oCatalogerIn,\n      operation,\n      operationSettings: {prio},\n      contentType,\n      recordLoadParams,\n      queueItemState: stream ? QUEUE_ITEM_STATE.VALIDATOR.UPLOADING : QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS,\n      creationTime: time,\n      modificationTime: time,\n      handledIds: [],\n      rejectedIds: []\n    };\n    try {\n      // No await here, promises later\n      db.collection(collection).insertOne(newQueueItem);\n      logger.info(`New BULK queue item for ${operation} ${correlationId} has been made in ${collection}!`);\n    } catch (error) {\n      logError(error);\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    }\n\n    if (!stream) {\n      return correlationId;\n    }\n\n    return new Promise((resolve, reject) => {\n      const outputStream = gridFSBucket.openUploadStream(correlationId);\n\n      stream\n        .on('error', reject)\n        .on('data', chunk => outputStream.write(chunk))\n        .on('end', () => outputStream.end(undefined, undefined, () => {\n          resolve(correlationId);\n        }));\n    });\n  }\n\n  // Check state that the queueItem has not waited too long and set state\n  async function checkAndSetState({correlationId, state, errorMessage = undefined, errorStatus = undefined}) {\n    // checkTimeOut returns true, if queueItem is fresher than 1 minute and it's state is not ABORT/ERROR\n    // otherwise it sets queueItem to state ABORT (408, 'Timeout')\n    const timeOut = await checkTimeOut(correlationId);\n    if (timeOut) {\n      return setState({correlationId, state, errorMessage, errorStatus});\n    }\n    return false;\n  }\n\n  async function query(params) {\n    const result = await db.collection(collection).find(params, {projection: {_id: 0}})\n      .toArray();\n    logger.debug(`Query result: ${result.length > 0 ? 'Found!' : 'Not found!'}`);\n    return result;\n  }\n\n  async function queryById(correlationId, checkModTime) {\n    const result = await db.collection(collection).findOne({correlationId});\n    if (checkModTime) {\n      const timeOut = await checkTimeOut(correlationId);\n      if (timeOut) {\n        return queryById(correlationId);\n      }\n      return result;\n    }\n\n    return result;\n  }\n\n  // Check that if the item has waited too long\n  // If the last modification time for the queueItem is older than 1 minute\n  // set state to ABORT and return false, otherwise return true\n  // If the state is already ABORT or ERROR return false\n\n  async function checkTimeOut(correlationId) {\n    const {modificationTime, queueitemState: oldState} = await db.collection(collection).findOne({correlationId});\n\n    if ([QUEUE_ITEM_STATE.ABORT, QUEUE_ITEM_STATE.ERROR].includes(oldState)) {\n      logger.silly(`${correlationId} has already state: ${oldState}`);\n      return false;\n    }\n\n    const timeoutTime = moment(modificationTime).add(1, 'm');\n    logger.silly(`timeOut @ ${timeoutTime}`);\n\n    if (timeoutTime.isBefore()) {\n      await setState({correlationId, state: QUEUE_ITEM_STATE.ABORT, errorStatus: httpStatus.REQUEST_TIMEOUT, errorMessage: `Timeout in ${oldState}`});\n      return false;\n    }\n\n    return true;\n  }\n\n  async function remove(params) {\n    logger.silly(`${JSON.stringify(params)}`);\n    logger.info(`Removing from Mongo (${collection}) id: ${params.correlationId}`);\n    const clean = sanitize(params.correlationId);\n    logger.silly(`mongo/remove: clean: ${JSON.stringify(clean)}`);\n\n    try {\n      //const metadataResult = await getFileMetadata({filename: clean});\n      //logger.debug(`mongo/remove: metadataResult: ${JSON.stringify(metadataResult)}`);\n      const noContent = await removeContent(params);\n      if (noContent) {\n        await db.collection(collection).deleteOne({correlationId: clean});\n        return true;\n      }\n    } catch (err) {\n      if (err instanceof MongoDriverError) {\n        if (err.message.indexOf('File not found for id') !== -1) {\n          logger.silly(`mongo/remove: File not found, removing queueItem ${JSON.stringify(clean)} from ${collection}`);\n          await db.collection(collection).deleteOne({correlationId: clean});\n          return true;\n        }\n        logger.error(err.message);\n        throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n      }\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n    }\n  }\n\n  async function readContent(correlationId) {\n    logger.info(`Reading content from mongo for id: ${correlationId} in ${collection}`);\n    const clean = sanitize(correlationId);\n    const result = await db.collection(collection).findOne({correlationId: clean}); // njsscan-ignore: node_nosqli_injection\n\n    if (result) {\n      return {\n        contentType: result.contentType,\n        readStream: gridFSBucket.openDownloadStreamByName(clean)\n      };\n    }\n\n    throw new ApiError(httpStatus.NOT_FOUND);\n  }\n\n  async function removeContent(params) {\n    logger.info(`Removing content from mongo for id: ${params.correlationId} in ${collection}`);\n    const clean = sanitize(params.correlationId);\n\n    const result = await db.collection(collection).findOne({correlationId: clean}); // njsscan-ignore: node_nosqli_injection\n    logger.silly(`mongo/removeContent: result ${JSON.stringify(result)}`);\n\n    if (result) {\n      await gridFSBucket.delete(clean);\n      return true;\n    }\n\n    return true;\n  }\n\n  function getOne({operation, queueItemState}) {\n    const clean = {queueItemState: sanitize(queueItemState)};\n    try {\n      if (operation === undefined) {\n        logger.silly(`Checking DB ${collection} for ${JSON.stringify(clean.queueItemState)}`);\n        return db.collection(collection).findOne({...clean});\n      }\n\n      const clean2 = {operation: sanitize(operation)};\n      logger.silly(`Checking DB ${collection} for ${clean.queueItemState} + ${clean2.operation}`);\n      return db.collection(collection).findOne({...clean, ...clean2});\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  function getStream(correlationId) {\n    logger.info(`Forming stream from mongo for ${correlationId} in ${collection}`);\n    const clean = sanitize(correlationId);\n    try {\n      // Return content stream\n      return gridFSBucket.openDownloadStreamByName(clean);\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function pushIds({correlationId, handledIds, rejectedIds}) {\n    logger.verbose(`Push ids (${handledIds.length}) and rejectedIds (${rejectedIds.length}) ${correlationId} to ${collection}`);\n    logger.debug(`ids (${handledIds.length}): ${JSON.stringify(handledIds)}, rejectedIds ${rejectedIds.length}: ${JSON.stringify(rejectedIds)}`);\n    const clean = sanitize(correlationId);\n    await db.collection(collection).updateOne({\n      correlationId: clean\n    }, {\n      $set: {\n        modificationTime: moment().toDate()\n      },\n      $push: {\n        handledIds: {$each: handledIds},\n        rejectedIds: {$each: rejectedIds}\n      }\n    });\n  }\n\n  async function pushMessages({correlationId, messages, messageField = 'messages'}) {\n    logger.verbose(`Push messages (${messages.length}) to ${messageField} ${correlationId} to ${collection}`);\n    logger.silly(`Messages (${messages.length}): ${JSON.stringify(messages)}}`);\n    const clean = sanitize(correlationId);\n    const cleanMessageField = sanitize(messageField);\n    await db.collection(collection).updateOne({\n      correlationId: clean\n    }, {\n      $set: {\n        modificationTime: moment().toDate()\n      },\n      $push: {\n        [cleanMessageField]: {$each: messages}\n      }\n    });\n  }\n\n  function setState({correlationId, state, errorMessage = undefined, errorStatus = undefined}) {\n    const errorString = errorMessage || errorStatus ? `, Error message: '${errorMessage || ''}', Error status: '${errorStatus || ''}'` : '';\n    logger.info(`Setting queue-item state ${state} for ${correlationId}${errorString} to ${collection}`);\n\n    const stateInQueueItemStates = Object.values(QUEUE_ITEM_STATE).indexOf(state) > -1;\n    const stateInQueueItemStatesValidator = Object.values(QUEUE_ITEM_STATE.VALIDATOR).indexOf(state) > -1;\n    const stateInQueueItemStatesImporter = Object.values(QUEUE_ITEM_STATE.IMPORTER).indexOf(state) > -1;\n    if (!stateInQueueItemStates && !stateInQueueItemStatesValidator && !stateInQueueItemStatesImporter) {\n      throw new ApiError(httpStatus.BAD_REQUEST, 'Trying to set invalid state');\n    }\n\n    const clean = sanitize(correlationId);\n    const updateValues = {\n      queueItemState: state,\n      modificationTime: moment().toDate(),\n      errorMessage,\n      errorStatus\n    };\n\n    // Do not update value that are undefined\n    // eslint-disable-next-line functional/immutable-data\n    Object.keys(updateValues).forEach(key => updateValues[key] === undefined && delete updateValues[key]);\n\n    return db.collection(collection)\n      .findOneAndUpdate(\n        {correlationId: clean},\n        {$set: updateValues},\n        {projection: {_id: 0}, returnNewDocument: true}\n      );\n  }\n}\n"],"file":"mongo.js"}
+{"version":3,"sources":["../src/mongo.js"],"names":["MONGO_URI","collection","logger","client","MongoClient","connect","useNewUrlParser","useUnifiedTopology","db","gridFSBucket","GridFSBucket","bucketName","createPrio","createBulk","checkAndSetState","checkAndSetImportJobState","query","queryById","remove","readContent","removeContent","getOne","getStream","setState","setImportJobState","pushIds","pushMessages","setOperation","setOperations","correlationId","cataloger","oCatalogerIn","operation","operationSettings","time","toDate","newQueueItem","operations","prio","queueItemState","QUEUE_ITEM_STATE","VALIDATOR","PENDING_VALIDATION","importJobState","CREATE","IMPORT_JOB_STATE","EMPTY","UPDATE","creationTime","modificationTime","handledIds","rejectedIds","result","insertOne","acknowledged","info","ApiError","httpStatus","INTERNAL_SERVER_ERROR","error","contentType","recordLoadParams","stream","UPLOADING","WAITING_FOR_RECORDS","Promise","resolve","reject","outputStream","openUploadStream","on","chunk","write","end","undefined","debug","state","errorMessage","errorStatus","timeOut","checkTimeOut","params","find","projection","_id","toArray","length","checkModTime","findOne","queueitemState","oldState","ABORT","ERROR","DONE","includes","silly","timeoutTime","add","isBefore","finalImportJobStates","REQUEST_TIMEOUT","JSON","stringify","clean","noContent","deleteOne","err","MongoDriverError","message","indexOf","readStream","openDownloadStreamByName","NOT_FOUND","delete","cleanQueueItemState","verbose","updateOne","$set","$push","$each","messages","messageField","cleanMessageField","errorString","stateInQueueItemStates","Object","values","stateInQueueItemStatesValidator","stateInQueueItemStatesImporter","IMPORTER","BAD_REQUEST","updateValues","keys","forEach","key","findOneAndUpdate","returnNewDocument","cleanCorrelationId","cleanImportJobState","OPERATIONS","newJobState","newOperation","oldOperation","cleanNewOperation","ok","addOperation","removeOperation","queueItem","oldOperations","operationsAfterRemove","filter","operationsAfterRemoveAndAdd"],"mappings":";;;;;;;AA4BA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AAnCA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAWA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAEe,wBAAgBA,SAAhB,EAA2BC,UAA3B,EAAuC;AACpD,QAAMC,MAAM,GAAG,0CAAf,CADoD,CAGpD;;AACA,QAAMC,MAAM,GAAG,MAAMC,qBAAYC,OAAZ,CAAoBL,SAApB,EAA+B;AAACM,IAAAA,eAAe,EAAE,IAAlB;AAAwBC,IAAAA,kBAAkB,EAAE;AAA5C,GAA/B,CAArB;AACA,QAAMC,EAAE,GAAGL,MAAM,CAACK,EAAP,CAAU,UAAV,CAAX;AACA,QAAMC,YAAY,GAAG,IAAIC,qBAAJ,CAAiBF,EAAjB,EAAqB;AAACG,IAAAA,UAAU,EAAEV;AAAb,GAArB,CAArB;AAEA,SAAO;AAACW,IAAAA,UAAD;AAAaC,IAAAA,UAAb;AAAyBC,IAAAA,gBAAzB;AAA2CC,IAAAA,yBAA3C;AAAsEC,IAAAA,KAAtE;AAA6EC,IAAAA,SAA7E;AAAwFC,IAAAA,MAAxF;AAAgGC,IAAAA,WAAhG;AAA6GC,IAAAA,aAA7G;AAA4HC,IAAAA,MAA5H;AAAoIC,IAAAA,SAApI;AAA+IC,IAAAA,QAA/I;AAAyJC,IAAAA,iBAAzJ;AAA4KC,IAAAA,OAA5K;AAAqLC,IAAAA,YAArL;AAAmMC,IAAAA,YAAnM;AAAiNC,IAAAA;AAAjN,GAAP;;AAEA,iBAAehB,UAAf,CAA0B;AAACiB,IAAAA,aAAD;AAAgBC,IAAAA,SAAhB;AAA2BC,IAAAA,YAA3B;AAAyCC,IAAAA,SAAzC;AAAoDC,IAAAA;AAApD,GAA1B,EAAkG;AAChG,UAAMC,IAAI,GAAG,uBAASC,MAAT,EAAb;AACA,UAAMC,YAAY,GAAG;AACnBP,MAAAA,aADmB;AAEnBC,MAAAA,SAFmB;AAGnBE,MAAAA,SAHmB;AAInBK,MAAAA,UAAU,EAAE,CAACL,SAAD,CAJO;AAKnBD,MAAAA,YALmB;AAMnBE,MAAAA,iBAAiB,EAAE;AACjBK,QAAAA,IAAI,EAAE,IADW;AAEjB,WAAGL;AAFc,OANA;AAUnBM,MAAAA,cAAc,EAAEC,4BAAiBC,SAAjB,CAA2BC,kBAVxB;AAWnBC,MAAAA,cAAc,EAAE;AACdC,QAAAA,MAAM,EAAEC,4BAAiBC,KADX;AAEdC,QAAAA,MAAM,EAAEF,4BAAiBC;AAFX,OAXG;AAenBE,MAAAA,YAAY,EAAEd,IAfK;AAgBnBe,MAAAA,gBAAgB,EAAEf,IAhBC;AAiBnBgB,MAAAA,UAAU,EAAE,EAjBO;AAkBnBC,MAAAA,WAAW,EAAE;AAlBM,KAArB;;AAoBA,QAAI;AACF,YAAMC,MAAM,GAAG,MAAM5C,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BoD,SAA1B,CAAoCjB,YAApC,CAArB;;AACA,UAAIgB,MAAM,CAACE,YAAX,EAAyB;AACvBpD,QAAAA,MAAM,CAACqD,IAAP,CAAa,2BAA0BvB,SAAU,IAAGH,aAAc,qBAAoB5B,UAAW,EAAjG;AACA;AACD;;AACD,YAAM,IAAIuD,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD,KAPD,CAOE,OAAOC,KAAP,EAAc;AACd,2BAASA,KAAT;AACA,YAAM,IAAIH,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD;AACF;;AAED,iBAAe7C,UAAf,CAA0B;AAACgB,IAAAA,aAAD;AAAgBC,IAAAA,SAAhB;AAA2BC,IAAAA,YAA3B;AAAyCC,IAAAA,SAAzC;AAAoD4B,IAAAA,WAApD;AAAiEC,IAAAA,gBAAjE;AAAmFC,IAAAA,MAAnF;AAA2F7B,IAAAA;AAA3F,GAA1B,EAAyI;AACvI,UAAMC,IAAI,GAAG,uBAASC,MAAT,EAAb;AACA,UAAMC,YAAY,GAAG;AACnBP,MAAAA,aADmB;AAEnBC,MAAAA,SAFmB;AAGnBC,MAAAA,YAHmB;AAInBC,MAAAA,SAJmB;AAKnBK,MAAAA,UAAU,EAAE,CAACL,SAAD,CALO;AAMnBC,MAAAA,iBAAiB,EAAE;AACjBK,QAAAA,IAAI,EAAE,KADW;AAEjB,WAAGL;AAFc,OANA;AAUnB2B,MAAAA,WAVmB;AAWnBC,MAAAA,gBAXmB;AAYnBtB,MAAAA,cAAc,EAAEuB,MAAM,GAAGtB,4BAAiBC,SAAjB,CAA2BsB,SAA9B,GAA0CvB,4BAAiBC,SAAjB,CAA2BuB,mBAZxE;AAanBrB,MAAAA,cAAc,EAAE;AACdC,QAAAA,MAAM,EAAEC,4BAAiBC,KADX;AAEdC,QAAAA,MAAM,EAAEF,4BAAiBC;AAFX,OAbG;AAiBnBE,MAAAA,YAAY,EAAEd,IAjBK;AAkBnBe,MAAAA,gBAAgB,EAAEf,IAlBC;AAmBnBgB,MAAAA,UAAU,EAAE,EAnBO;AAoBnBC,MAAAA,WAAW,EAAE;AApBM,KAArB,CAFuI,CAyBvI;;AACA,QAAIW,MAAJ,EAAY;AACV,UAAI;AACF;AACAtD,QAAAA,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BoD,SAA1B,CAAoCjB,YAApC;AACAlC,QAAAA,MAAM,CAACqD,IAAP,CAAa,2BAA0BvB,SAAU,IAAGH,aAAc,qBAAoB5B,UAAW,GAAjG;AACA,eAAO,IAAIgE,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtC,gBAAMC,YAAY,GAAG3D,YAAY,CAAC4D,gBAAb,CAA8BxC,aAA9B,CAArB;AAEAiC,UAAAA,MAAM,CACHQ,EADH,CACM,OADN,EACeH,MADf,EAEGG,EAFH,CAEM,MAFN,EAEcC,KAAK,IAAIH,YAAY,CAACI,KAAb,CAAmBD,KAAnB,CAFvB,EAGGD,EAHH,CAGM,KAHN,EAGa,MAAMF,YAAY,CAACK,GAAb,CAAiBC,SAAjB,EAA4BA,SAA5B,EAAuC,MAAM;AAC5DR,YAAAA,OAAO,CAACrC,aAAD,CAAP;AACD,WAFgB,CAHnB;AAMD,SATM,CAAP;AAUD,OAdD,CAcE,OAAO8B,KAAP,EAAc;AACd,6BAASA,KAAT;AACA,cAAM,IAAIH,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD;AACF;;AAEDxD,IAAAA,MAAM,CAACyE,KAAP,CAAc,WAAd,EA/CuI,CAgDvI;;AACA,QAAI,CAACb,MAAL,EAAa;AACX,UAAI;AACF,cAAMV,MAAM,GAAG,MAAM5C,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BoD,SAA1B,CAAoCjB,YAApC,CAArB;;AACA,YAAIgB,MAAM,CAACE,YAAX,EAAyB;AACvBpD,UAAAA,MAAM,CAACqD,IAAP,CAAa,oCAAmCvB,SAAU,IAAGH,aAAc,qBAAoB5B,UAAW,EAA1G;AACA,iBAAO;AAAC4B,YAAAA,aAAD;AAAgBU,YAAAA,cAAc,EAAEC,4BAAiBC,SAAjB,CAA2BuB;AAA3D,WAAP;AACD;;AACD,cAAM,IAAIR,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD,OAPD,CAOE,OAAOC,KAAP,EAAc;AACd,6BAASA,KAAT;AACA,cAAM,IAAIH,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD;AACF;AACF,GA3GmD,CA6GpD;;;AACA,iBAAe5C,gBAAf,CAAgC;AAACe,IAAAA,aAAD;AAAgB+C,IAAAA,KAAhB;AAAuBC,IAAAA,YAAY,GAAGH,SAAtC;AAAiDI,IAAAA,WAAW,GAAGJ;AAA/D,GAAhC,EAA2G;AACzG;AACA;AACA,UAAMK,OAAO,GAAG,MAAMC,YAAY,CAAC;AAACnD,MAAAA;AAAD,KAAD,CAAlC;;AACA,QAAIkD,OAAJ,EAAa;AACX,aAAOxD,QAAQ,CAAC;AAACM,QAAAA,aAAD;AAAgB+C,QAAAA,KAAhB;AAAuBC,QAAAA,YAAvB;AAAqCC,QAAAA;AAArC,OAAD,CAAf;AACD;;AACD,WAAO,KAAP;AACD,GAtHmD,CAwHpD;;;AACA,iBAAe/D,yBAAf,CAAyC;AAACc,IAAAA,aAAD;AAAgBG,IAAAA,SAAhB;AAA2BW,IAAAA,cAA3B;AAA2CkC,IAAAA,YAAY,GAAG,EAA1D;AAA8DC,IAAAA,WAAW,GAAG;AAA5E,GAAzC,EAA0H;AACxH;AACA;AACA5E,IAAAA,MAAM,CAACyE,KAAP,CAAc,GAAE9C,aAAc,KAAIc,cAAe,KAAIX,SAAU,EAA/D;AACA,UAAM+C,OAAO,GAAG,MAAMC,YAAY,CAAC;AAACnD,MAAAA,aAAD;AAAgBG,MAAAA,SAAhB;AAA2BW,MAAAA;AAA3B,KAAD,CAAlC;;AACA,QAAIoC,OAAJ,EAAa;AACX,aAAOvD,iBAAiB,CAAC;AAACK,QAAAA,aAAD;AAAgBG,QAAAA,SAAhB;AAA2BW,QAAAA,cAA3B;AAA2CkC,QAAAA,YAA3C;AAAyDC,QAAAA;AAAzD,OAAD,CAAxB;AACD;;AACD,WAAO,KAAP;AACD;;AAED,iBAAe9D,KAAf,CAAqBiE,MAArB,EAA6B;AAC3B,UAAM7B,MAAM,GAAG,MAAM5C,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BiF,IAA1B,CAA+BD,MAA/B,EAAuC;AAACE,MAAAA,UAAU,EAAE;AAACC,QAAAA,GAAG,EAAE;AAAN;AAAb,KAAvC,EAClBC,OADkB,EAArB;AAEAnF,IAAAA,MAAM,CAACyE,KAAP,CAAc,iBAAgBvB,MAAM,CAACkC,MAAP,GAAgB,CAAhB,GAAoB,QAApB,GAA+B,YAAa,EAA1E;AACA,WAAOlC,MAAP;AACD;;AAED,iBAAenC,SAAf,CAAyB;AAACY,IAAAA,aAAD;AAAgB0D,IAAAA,YAAY,GAAG;AAA/B,GAAzB,EAAgE;AAC9D,UAAMnC,MAAM,GAAG,MAAM5C,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BuF,OAA1B,CAAkC;AAAC3D,MAAAA;AAAD,KAAlC,CAArB;;AACA,QAAI0D,YAAJ,EAAkB;AAChB,YAAMR,OAAO,GAAG,MAAMC,YAAY,CAAC;AAACnD,QAAAA;AAAD,OAAD,CAAlC;;AACA,UAAIkD,OAAJ,EAAa;AACX,eAAO9D,SAAS,CAAC;AAACY,UAAAA;AAAD,SAAD,CAAhB;AACD;;AACD,aAAOuB,MAAP;AACD;;AAED,WAAOA,MAAP;AACD,GAtJmD,CAwJpD;AACA;AACA;AACA;;;AAEA,iBAAe4B,YAAf,CAA4B;AAACnD,IAAAA;AAAD,GAA5B,EAA6C;AAC3C,UAAM;AAACoB,MAAAA,gBAAD;AAAmBwC,MAAAA,cAAc,EAAEC,QAAnC;AAA6C/C,MAAAA;AAA7C,QAA+D,MAAMnC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BuF,OAA1B,CAAkC;AAAC3D,MAAAA;AAAD,KAAlC,CAA3E,CAD2C,CAG3C;;AACA,QAAI,CAACW,4BAAiBmD,KAAlB,EAAyBnD,4BAAiBoD,KAA1C,EAAiDpD,4BAAiBqD,IAAlE,EAAwEC,QAAxE,CAAiFJ,QAAjF,CAAJ,EAAgG;AAC9FxF,MAAAA,MAAM,CAAC6F,KAAP,CAAc,GAAElE,aAAc,uBAAsB6D,QAAS,EAA7D;AACA,aAAO,KAAP;AACD;;AAED,UAAMM,WAAW,GAAG,qBAAO/C,gBAAP,EAAyBgD,GAAzB,CAA6B,CAA7B,EAAgC,GAAhC,CAApB;AACA/F,IAAAA,MAAM,CAAC6F,KAAP,CAAc,aAAYC,WAAY,EAAtC;;AAEA,QAAIA,WAAW,CAACE,QAAZ,EAAJ,EAA4B;AAC1B,YAAMC,oBAAoB,GAAG,CAACtD,4BAAiB8C,KAAlB,EAAyB9C,4BAAiBgD,IAA1C,EAAgDhD,4BAAiBC,KAAjE,EAAwED,4BAAiB+C,KAAzF,CAA7B;;AACA,UAAI,CAACO,oBAAoB,CAACL,QAArB,CAA8BnD,cAAc,CAACC,MAA7C,CAAL,EAA2D;AAAE;AAC3D,cAAMpB,iBAAiB,CAAC;AAACK,UAAAA,aAAD;AAAgBG,UAAAA,SAAS,EAAE,QAA3B;AAAqCW,UAAAA,cAAc,EAAEE,4BAAiB8C;AAAtE,SAAD,CAAvB;AACD;;AAED,UAAI,CAACQ,oBAAoB,CAACL,QAArB,CAA8BnD,cAAc,CAACI,MAA7C,CAAL,EAA2D;AAAE;AAC3D,cAAMvB,iBAAiB,CAAC;AAACK,UAAAA,aAAD;AAAgBG,UAAAA,SAAS,EAAE,QAA3B;AAAqCW,UAAAA,cAAc,EAAEE,4BAAiB8C;AAAtE,SAAD,CAAvB;AACD;;AAED,YAAMpE,QAAQ,CAAC;AAACM,QAAAA,aAAD;AAAgB+C,QAAAA,KAAK,EAAEpC,4BAAiBmD,KAAxC;AAA+Cb,QAAAA,WAAW,EAAErB,oBAAW2C,eAAvE;AAAwFvB,QAAAA,YAAY,EAAG,cAAaa,QAAS;AAA7H,OAAD,CAAd;AAEA,aAAO,KAAP;AACD;;AAED,WAAO,IAAP;AACD;;AAED,iBAAexE,MAAf,CAAsB+D,MAAtB,EAA8B;AAC5B/E,IAAAA,MAAM,CAAC6F,KAAP,CAAc,GAAEM,IAAI,CAACC,SAAL,CAAerB,MAAf,CAAuB,EAAvC;AACA/E,IAAAA,MAAM,CAACqD,IAAP,CAAa,wBAAuBtD,UAAW,SAAQgF,MAAM,CAACpD,aAAc,EAA5E;AACA,UAAM0E,KAAK,GAAG,4BAAStB,MAAM,CAACpD,aAAhB,CAAd;AACA3B,IAAAA,MAAM,CAAC6F,KAAP,CAAc,wBAAuBM,IAAI,CAACC,SAAL,CAAeC,KAAf,CAAsB,EAA3D;;AAEA,QAAI;AACF;AACA;AACA,YAAMC,SAAS,GAAG,MAAMpF,aAAa,CAAC6D,MAAD,CAArC;;AACA,UAAIuB,SAAJ,EAAe;AACb,cAAMhG,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BwG,SAA1B,CAAoC;AAAC5E,UAAAA,aAAa,EAAE0E;AAAhB,SAApC,CAAN;AACA,eAAO,IAAP;AACD;AACF,KARD,CAQE,OAAOG,GAAP,EAAY;AACZ,UAAIA,GAAG,YAAYC,yBAAnB,EAAqC;AACnC,YAAID,GAAG,CAACE,OAAJ,CAAYC,OAAZ,CAAoB,uBAApB,MAAiD,CAAC,CAAtD,EAAyD;AACvD3G,UAAAA,MAAM,CAAC6F,KAAP,CAAc,oDAAmDM,IAAI,CAACC,SAAL,CAAeC,KAAf,CAAsB,SAAQtG,UAAW,EAA1G;AACA,gBAAMO,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BwG,SAA1B,CAAoC;AAAC5E,YAAAA,aAAa,EAAE0E;AAAhB,WAApC,CAAN;AACA,iBAAO,IAAP;AACD;;AACDrG,QAAAA,MAAM,CAACyD,KAAP,CAAa+C,GAAG,CAACE,OAAjB;AACA,cAAM,IAAIpD,qBAAJ,CAAaC,oBAAWC,qBAAxB,EAA+CgD,GAAG,CAACE,OAAnD,CAAN;AACD;;AACD,YAAM,IAAIpD,qBAAJ,CAAaC,oBAAWC,qBAAxB,EAA+CgD,GAAG,CAACE,OAAnD,CAAN;AACD;AACF;;AAED,iBAAezF,WAAf,CAA2BU,aAA3B,EAA0C;AACxC3B,IAAAA,MAAM,CAACqD,IAAP,CAAa,sCAAqC1B,aAAc,OAAM5B,UAAW,EAAjF;AACA,UAAMsG,KAAK,GAAG,4BAAS1E,aAAT,CAAd;AACA,UAAMuB,MAAM,GAAG,MAAM5C,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BuF,OAA1B,CAAkC;AAAC3D,MAAAA,aAAa,EAAE0E;AAAhB,KAAlC,CAArB,CAHwC,CAGwC;;AAEhF,QAAInD,MAAJ,EAAY;AACV,aAAO;AACLQ,QAAAA,WAAW,EAAER,MAAM,CAACQ,WADf;AAELkD,QAAAA,UAAU,EAAErG,YAAY,CAACsG,wBAAb,CAAsCR,KAAtC;AAFP,OAAP;AAID;;AAED,UAAM,IAAI/C,qBAAJ,CAAaC,oBAAWuD,SAAxB,CAAN;AACD;;AAED,iBAAe5F,aAAf,CAA6B6D,MAA7B,EAAqC;AACnC/E,IAAAA,MAAM,CAACqD,IAAP,CAAa,uCAAsC0B,MAAM,CAACpD,aAAc,OAAM5B,UAAW,EAAzF;AACA,UAAMsG,KAAK,GAAG,4BAAStB,MAAM,CAACpD,aAAhB,CAAd;AAEA,UAAMuB,MAAM,GAAG,MAAM5C,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BuF,OAA1B,CAAkC;AAAC3D,MAAAA,aAAa,EAAE0E;AAAhB,KAAlC,CAArB,CAJmC,CAI6C;;AAChFrG,IAAAA,MAAM,CAAC6F,KAAP,CAAc,+BAA8BM,IAAI,CAACC,SAAL,CAAelD,MAAf,CAAuB,EAAnE;;AAEA,QAAIA,MAAJ,EAAY;AACV,YAAM3C,YAAY,CAACwG,MAAb,CAAoBV,KAApB,CAAN;AACA,aAAO,IAAP;AACD;;AAED,WAAO,IAAP;AACD,GAnPmD,CAqPpD;;;AACA,WAASlF,MAAT,CAAgB;AAACkB,IAAAA,cAAD;AAAiBI,IAAAA,cAAc,GAAG+B;AAAlC,GAAhB,EAA8D;AAE5DxE,IAAAA,MAAM,CAAC6F,KAAP,CAAc,mBAAkBxD,cAAe,qBAAoB8D,IAAI,CAACC,SAAL,CAAe3D,cAAf,CAA+B,EAAlG;AACA,UAAMuE,mBAAmB,GAAG3E,cAAc,GAAG;AAACA,MAAAA,cAAc,EAAE,4BAASA,cAAT;AAAjB,KAAH,GAAgDmC,SAA1F;;AAEA,QAAI;AAEF;AACA,UAAInC,cAAc,IAAII,cAAc,KAAK+B,SAAzC,EAAoD;AAClDxE,QAAAA,MAAM,CAAC6F,KAAP,CAAc,eAAc9F,UAAW,aAAYoG,IAAI,CAACC,SAAL,CAAeY,mBAAmB,CAAC3E,cAAnC,CAAmD,EAAtG;AACA,eAAO/B,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BuF,OAA1B,CAAkC,EAAC,GAAG0B;AAAJ,SAAlC,CAAP;AACD,OANC,CAQF;;;AACA,UAAIvE,cAAc,IAAIJ,cAAc,KAAKmC,SAAzC,EAAoD;AAClDxE,QAAAA,MAAM,CAAC6F,KAAP,CAAc,eAAc9F,UAAW,QAAOoG,IAAI,CAACC,SAAL,CAAe3D,cAAf,CAA+B,EAA7E;AACA,eAAOnC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BuF,OAA1B,CAAkC,EAAC,GAAG7C;AAAJ,SAAlC,CAAP;AACD,OAZC,CAcF;;;AACA,UAAIA,cAAc,IAAIJ,cAAtB,EAAsC;AACpCrC,QAAAA,MAAM,CAAC6F,KAAP,CAAc,eAAc9F,UAAW,QAAOsC,cAAe,QAAO8D,IAAI,CAACC,SAAL,CAAe3D,cAAf,CAA+B,EAAnG;AACA,eAAOnC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BuF,OAA1B,CAAkC,EAAC,GAAG0B,mBAAJ;AAAyB,aAAGvE;AAA5B,SAAlC,CAAP;AACD;;AAEDzC,MAAAA,MAAM,CAACyE,KAAP,CAAc,qBAAd;AACA,YAAM,IAAInB,qBAAJ,CAAaC,oBAAWC,qBAAxB,EAA+C,0CAA/C,CAAN;AAED,KAvBD,CAuBE,OAAOC,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,WAASrC,SAAT,CAAmBO,aAAnB,EAAkC;AAChC3B,IAAAA,MAAM,CAACqD,IAAP,CAAa,iCAAgC1B,aAAc,OAAM5B,UAAW,EAA5E;AACA,UAAMsG,KAAK,GAAG,4BAAS1E,aAAT,CAAd;;AACA,QAAI;AACF;AACA,aAAOpB,YAAY,CAACsG,wBAAb,CAAsCR,KAAtC,CAAP;AACD,KAHD,CAGE,OAAO5C,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAelC,OAAf,CAAuB;AAACI,IAAAA,aAAD;AAAgBqB,IAAAA,UAAhB;AAA4BC,IAAAA;AAA5B,GAAvB,EAAiE;AAC/DjD,IAAAA,MAAM,CAACiH,OAAP,CAAgB,aAAYjE,UAAU,CAACoC,MAAO,sBAAqBnC,WAAW,CAACmC,MAAO,KAAIzD,aAAc,OAAM5B,UAAW,EAAzH;AACAC,IAAAA,MAAM,CAACyE,KAAP,CAAc,QAAOzB,UAAU,CAACoC,MAAO,MAAKe,IAAI,CAACC,SAAL,CAAepD,UAAf,CAA2B,iBAAgBC,WAAW,CAACmC,MAAO,KAAIe,IAAI,CAACC,SAAL,CAAenD,WAAf,CAA4B,EAA1I;AACA,UAAMoD,KAAK,GAAG,4BAAS1E,aAAT,CAAd;AACA,UAAMrB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BmH,SAA1B,CAAoC;AACxCvF,MAAAA,aAAa,EAAE0E;AADyB,KAApC,EAEH;AACDc,MAAAA,IAAI,EAAE;AACJpE,QAAAA,gBAAgB,EAAE,uBAASd,MAAT;AADd,OADL;AAIDmF,MAAAA,KAAK,EAAE;AACLpE,QAAAA,UAAU,EAAE;AAACqE,UAAAA,KAAK,EAAErE;AAAR,SADP;AAELC,QAAAA,WAAW,EAAE;AAACoE,UAAAA,KAAK,EAAEpE;AAAR;AAFR;AAJN,KAFG,CAAN;AAWD;;AAED,iBAAezB,YAAf,CAA4B;AAACG,IAAAA,aAAD;AAAgB2F,IAAAA,QAAhB;AAA0BC,IAAAA,YAAY,GAAG;AAAzC,GAA5B,EAAkF;AAChFvH,IAAAA,MAAM,CAACiH,OAAP,CAAgB,kBAAiBK,QAAQ,CAAClC,MAAO,QAAOmC,YAAa,IAAG5F,aAAc,OAAM5B,UAAW,EAAvG;AACAC,IAAAA,MAAM,CAAC6F,KAAP,CAAc,aAAYyB,QAAQ,CAAClC,MAAO,MAAKe,IAAI,CAACC,SAAL,CAAekB,QAAf,CAAyB,GAAxE;AACA,UAAMjB,KAAK,GAAG,4BAAS1E,aAAT,CAAd;AACA,UAAM6F,iBAAiB,GAAG,4BAASD,YAAT,CAA1B;AACA,UAAMjH,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BmH,SAA1B,CAAoC;AACxCvF,MAAAA,aAAa,EAAE0E;AADyB,KAApC,EAEH;AACDc,MAAAA,IAAI,EAAE;AACJpE,QAAAA,gBAAgB,EAAE,uBAASd,MAAT;AADd,OADL;AAIDmF,MAAAA,KAAK,EAAE;AACL,SAACI,iBAAD,GAAqB;AAACH,UAAAA,KAAK,EAAEC;AAAR;AADhB;AAJN,KAFG,CAAN;AAUD;;AAED,WAASjG,QAAT,CAAkB;AAACM,IAAAA,aAAD;AAAgB+C,IAAAA,KAAhB;AAAuBC,IAAAA,YAAY,GAAGH,SAAtC;AAAiDI,IAAAA,WAAW,GAAGJ;AAA/D,GAAlB,EAA6F;AAC3F,UAAMiD,WAAW,GAAG9C,YAAY,IAAIC,WAAhB,GAA+B,qBAAoBD,YAAY,IAAI,EAAG,qBAAoBC,WAAW,IAAI,EAAG,GAA5G,GAAiH,EAArI;AACA5E,IAAAA,MAAM,CAACqD,IAAP,CAAa,4BAA2BqB,KAAM,QAAO/C,aAAc,GAAE8F,WAAY,OAAM1H,UAAW,EAAlG;AAEA,UAAM2H,sBAAsB,GAAGC,MAAM,CAACC,MAAP,CAActF,2BAAd,EAAgCqE,OAAhC,CAAwCjC,KAAxC,IAAiD,CAAC,CAAjF;AACA,UAAMmD,+BAA+B,GAAGF,MAAM,CAACC,MAAP,CAActF,4BAAiBC,SAA/B,EAA0CoE,OAA1C,CAAkDjC,KAAlD,IAA2D,CAAC,CAApG;AACA,UAAMoD,8BAA8B,GAAGH,MAAM,CAACC,MAAP,CAActF,4BAAiByF,QAA/B,EAAyCpB,OAAzC,CAAiDjC,KAAjD,IAA0D,CAAC,CAAlG;;AACA,QAAI,CAACgD,sBAAD,IAA2B,CAACG,+BAA5B,IAA+D,CAACC,8BAApE,EAAoG;AAClG,YAAM,IAAIxE,qBAAJ,CAAaC,oBAAWyE,WAAxB,EAAqC,6BAArC,CAAN;AACD;;AAED,UAAM3B,KAAK,GAAG,4BAAS1E,aAAT,CAAd;AACA,UAAMsG,YAAY,GAAG;AACnB5F,MAAAA,cAAc,EAAEqC,KADG;AAEnB3B,MAAAA,gBAAgB,EAAE,uBAASd,MAAT,EAFC;AAGnB0C,MAAAA,YAHmB;AAInBC,MAAAA;AAJmB,KAArB,CAZ2F,CAmB3F;AACA;;AACA+C,IAAAA,MAAM,CAACO,IAAP,CAAYD,YAAZ,EAA0BE,OAA1B,CAAkCC,GAAG,IAAIH,YAAY,CAACG,GAAD,CAAZ,KAAsB5D,SAAtB,IAAmC,OAAOyD,YAAY,CAACG,GAAD,CAA/F;AAEA,WAAO9H,EAAE,CAACP,UAAH,CAAcA,UAAd,EACJsI,gBADI,CAEH;AAAC1G,MAAAA,aAAa,EAAE0E;AAAhB,KAFG,EAGH;AAACc,MAAAA,IAAI,EAAEc;AAAP,KAHG,EAIH;AAAChD,MAAAA,UAAU,EAAE;AAACC,QAAAA,GAAG,EAAE;AAAN,OAAb;AAAuBoD,MAAAA,iBAAiB,EAAE;AAA1C,KAJG,CAAP;AAMD;;AAED,WAAShH,iBAAT,CAA2B;AAACK,IAAAA,aAAD;AAAgBG,IAAAA,SAAhB;AAA2BW,IAAAA;AAA3B,GAA3B,EAAuE;AAErE;AACAzC,IAAAA,MAAM,CAACqD,IAAP,CAAa,uCAAsCvB,SAAU,KAAIW,cAAe,SAAQd,aAAc,EAAtG;AACA,UAAM4G,kBAAkB,GAAG,4BAAS5G,aAAT,CAA3B;AACA,UAAM6G,mBAAmB,GAAG,4BAAS/F,cAAT,CAA5B;;AAEA,QAAI,EAAEX,SAAS,IAAI2G,qBAAf,CAAJ,EAAgC;AAC9B,YAAM,IAAInF,qBAAJ,CAAa,KAAb,EAAoB,wCAApB,CAAN;AACD;;AAED,QAAI,EAAEkF,mBAAmB,IAAI7F,2BAAzB,CAAJ,EAAgD;AAC9C,YAAM,IAAIW,qBAAJ,CAAa,KAAb,EAAoB,0BAApB,CAAN;AACD;;AAED,UAAMoF,WAAW,GAAG5G,SAAS,KAAK2G,sBAAW/F,MAAzB,GAAkC;AAAC,+BAAyB8F;AAA1B,KAAlC,GAAmF;AAAC,+BAAyBA;AAA1B,KAAvG;AACA,WAAOlI,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BsI,gBAA1B,CAA2C;AAChD1G,MAAAA,aAAa,EAAE4G;AADiC,KAA3C,EAEJ;AACDpB,MAAAA,IAAI,EAAE,EACJ,GAAGuB,WADC;AAEJ3F,QAAAA,gBAAgB,EAAE,uBAASd,MAAT;AAFd;AADL,KAFI,EAOJ;AAACgD,MAAAA,UAAU,EAAE;AAACC,QAAAA,GAAG,EAAE;AAAN,OAAb;AAAuBoD,MAAAA,iBAAiB,EAAE;AAA1C,KAPI,CAAP;AAQD;;AAED,iBAAe7G,YAAf,CAA4B;AAACE,IAAAA,aAAD;AAAgBG,IAAAA;AAAhB,GAA5B,EAAwD;AACtD,UAAM6G,YAAY,GAAG7G,SAArB;AACA,UAAM;AAACA,MAAAA,SAAS,EAAE8G;AAAZ,QAA4B,MAAMtI,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BuF,OAA1B,CAAkC;AAAC3D,MAAAA;AAAD,KAAlC,CAAxC;AACA3B,IAAAA,MAAM,CAACqD,IAAP,CAAa,qCAAoCuF,YAAa,OAAMD,YAAa,QAAOhH,aAAc,OAAM5B,UAAW,EAAvH;AACA,UAAMwI,kBAAkB,GAAG,4BAAS5G,aAAT,CAA3B;AACA,UAAMkH,iBAAiB,GAAG,4BAASF,YAAT,CAA1B;AAEA,UAAMzF,MAAM,GAAG,MAAM5C,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BsI,gBAA1B,CAA2C;AAC9D1G,MAAAA,aAAa,EAAE4G;AAD+C,KAA3C,EAElB;AACDpB,MAAAA,IAAI,EAAE;AACJrF,QAAAA,SAAS,EAAE+G,iBADP;AAEJ9F,QAAAA,gBAAgB,EAAE,uBAASd,MAAT;AAFd;AADL,KAFkB,EAOlB;AAACgD,MAAAA,UAAU,EAAE;AAACC,QAAAA,GAAG,EAAE;AAAN,OAAb;AAAuBoD,MAAAA,iBAAiB,EAAE;AAA1C,KAPkB,CAArB;AASA,WAAOpF,MAAM,CAAC4F,EAAd,CAhBsD,CAiBtD;AACD;;AAED,iBAAepH,aAAf,CAA6B;AAACC,IAAAA,aAAD;AAAgBoH,IAAAA,YAAhB;AAA8BC,IAAAA,eAAe,GAAGxE;AAAhD,GAA7B,EAAyF;AACvF,UAAM+D,kBAAkB,GAAG,4BAAS5G,aAAT,CAA3B,CADuF,CAGvF;;AAEA,UAAMsH,SAAS,GAAG,MAAM3I,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BuF,OAA1B,CAAkC;AAAC3D,MAAAA,aAAa,EAAE4G;AAAhB,KAAlC,CAAxB;AACAvI,IAAAA,MAAM,CAACyE,KAAP,CAAc,yBAAwB0B,IAAI,CAACC,SAAL,CAAe6C,SAAf,CAA0B,EAAhE;AAEA,UAAMC,aAAa,GAAGD,SAAS,CAAC9G,UAAhC;AAEAnC,IAAAA,MAAM,CAACqD,IAAP,CAAa,qCAAoC8C,IAAI,CAACC,SAAL,CAAe8C,aAAf,CAA8B,cAAaH,YAAa,iBAAgBC,eAAgB,QAAOrH,aAAc,OAAM5B,UAAW,EAA/K;AAEA,UAAMoJ,qBAAqB,GAAGD,aAAa,CAACE,MAAd,CAAqBtH,SAAS,IAAIA,SAAS,KAAKkH,eAAhD,CAA9B;AACA,UAAMK,2BAA2B,GAAG,CAACN,YAAD,EAAe,GAAGI,qBAAlB,CAApC;AAEA,UAAMjG,MAAM,GAAG,MAAM5C,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BsI,gBAA1B,CAA2C;AAC9D1G,MAAAA,aAAa,EAAE4G;AAD+C,KAA3C,EAElB;AACDpB,MAAAA,IAAI,EAAE;AACJhF,QAAAA,UAAU,EAAEkH,2BADR;AAEJtG,QAAAA,gBAAgB,EAAE,uBAASd,MAAT;AAFd;AADL,KAFkB,EAOlB;AAACgD,MAAAA,UAAU,EAAE;AAACC,QAAAA,GAAG,EAAE;AAAN,OAAb;AAAuBoD,MAAAA,iBAAiB,EAAE;AAA1C,KAPkB,CAArB;AASA,WAAOpF,MAAM,CAAC4F,EAAd,CAxBuF,CAyBvF;AACD;AACF","sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020-2021 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport {MongoClient, GridFSBucket, MongoDriverError} from 'mongodb';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {Error as ApiError} from '@natlibfi/melinda-commons';\nimport {QUEUE_ITEM_STATE, IMPORT_JOB_STATE, OPERATIONS} from './constants';\nimport {logError} from './utils.js';\nimport moment from 'moment';\nimport httpStatus from 'http-status';\nimport sanitize from 'mongo-sanitize';\n\n/* QueueItem:\n{\n  \"correlationId\":\"FOO\",\n  \"cataloger\":\"xxx0000\",\n  \"oCatalogerIn\":\"xxx0000\"\n  \"operation\":\"UPDATE\",\n  \"operations\":[\"UPDATE\", \"CREATE\"]\n  \"contentType\":\"application/json\",\n  \"recordLoadParams\": {\n    \"pActiveLibrary\": \"XXX00\",\n    \"pInputFile\": \"filename.seq\",\n    \"pRejectFile\": \"filename.rej\",\n    \"pLogFile\": \"filename.syslog\",\n    \"pOldNew\": \"NEW\"\n  },\n  \"queueItemState\":\"PENDING_QUEUING\",\n  \"importJobState\": {\n    \"CREATE\": \"EMPTY\",\n    \"UPDATE\": \"EMPTY\"\n  },\n  \"creationTime\":\"2020-01-01T00:00:00.000Z\",\n  \"modificationTime\":\"2020-01-01T00:00:01.000Z\",\n  \"handledIds\": []\n}\n*/\n\nexport default async function (MONGO_URI, collection) {\n  const logger = createLogger();\n\n  // Connect to mongo (MONGO)\n  const client = await MongoClient.connect(MONGO_URI, {useNewUrlParser: true, useUnifiedTopology: true});\n  const db = client.db('rest-api');\n  const gridFSBucket = new GridFSBucket(db, {bucketName: collection});\n\n  return {createPrio, createBulk, checkAndSetState, checkAndSetImportJobState, query, queryById, remove, readContent, removeContent, getOne, getStream, setState, setImportJobState, pushIds, pushMessages, setOperation, setOperations};\n\n  async function createPrio({correlationId, cataloger, oCatalogerIn, operation, operationSettings}) {\n    const time = moment().toDate();\n    const newQueueItem = {\n      correlationId,\n      cataloger,\n      operation,\n      operations: [operation],\n      oCatalogerIn,\n      operationSettings: {\n        prio: true,\n        ...operationSettings\n      },\n      queueItemState: QUEUE_ITEM_STATE.VALIDATOR.PENDING_VALIDATION,\n      importJobState: {\n        CREATE: IMPORT_JOB_STATE.EMPTY,\n        UPDATE: IMPORT_JOB_STATE.EMPTY\n      },\n      creationTime: time,\n      modificationTime: time,\n      handledIds: [],\n      rejectedIds: []\n    };\n    try {\n      const result = await db.collection(collection).insertOne(newQueueItem);\n      if (result.acknowledged) {\n        logger.info(`New PRIO queue item for ${operation} ${correlationId} has been made in ${collection}`);\n        return;\n      }\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    } catch (error) {\n      logError(error);\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    }\n  }\n\n  async function createBulk({correlationId, cataloger, oCatalogerIn, operation, contentType, recordLoadParams, stream, operationSettings}) {\n    const time = moment().toDate();\n    const newQueueItem = {\n      correlationId,\n      cataloger,\n      oCatalogerIn,\n      operation,\n      operations: [operation],\n      operationSettings: {\n        prio: false,\n        ...operationSettings\n      },\n      contentType,\n      recordLoadParams,\n      queueItemState: stream ? QUEUE_ITEM_STATE.VALIDATOR.UPLOADING : QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS,\n      importJobState: {\n        CREATE: IMPORT_JOB_STATE.EMPTY,\n        UPDATE: IMPORT_JOB_STATE.EMPTY\n      },\n      creationTime: time,\n      modificationTime: time,\n      handledIds: [],\n      rejectedIds: []\n    };\n\n    // eslint-disable-next-line functional/no-conditional-statement\n    if (stream) {\n      try {\n        // No await here, promises later\n        db.collection(collection).insertOne(newQueueItem);\n        logger.info(`New BULK queue item for ${operation} ${correlationId} has been made in ${collection}!`);\n        return new Promise((resolve, reject) => {\n          const outputStream = gridFSBucket.openUploadStream(correlationId);\n\n          stream\n            .on('error', reject)\n            .on('data', chunk => outputStream.write(chunk))\n            .on('end', () => outputStream.end(undefined, undefined, () => {\n              resolve(correlationId);\n            }));\n        });\n      } catch (error) {\n        logError(error);\n        throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n      }\n    }\n\n    logger.debug(`No stream`);\n    // eslint-disable-next-line functional/no-conditional-statement\n    if (!stream) {\n      try {\n        const result = await db.collection(collection).insertOne(newQueueItem);\n        if (result.acknowledged) {\n          logger.info(`New noStream BULK queue item for ${operation} ${correlationId} has been made in ${collection}`);\n          return {correlationId, queueItemState: QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS};\n        }\n        throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n      } catch (error) {\n        logError(error);\n        throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n      }\n    }\n  }\n\n  // Check state that the queueItem has not waited too long and set state\n  async function checkAndSetState({correlationId, state, errorMessage = undefined, errorStatus = undefined}) {\n    // checkTimeOut returns true, if queueItem is fresher than 1 minute and it's state is not ABORT/ERROR\n    // otherwise it sets queueItem to state ABORT (408, 'Timeout')\n    const timeOut = await checkTimeOut({correlationId});\n    if (timeOut) {\n      return setState({correlationId, state, errorMessage, errorStatus});\n    }\n    return false;\n  }\n\n  // Check state that the queueItem has not waited too long and set state\n  async function checkAndSetImportJobState({correlationId, operation, importJobState, errorMessage = '', errorStatus = ''}) {\n    // checkTimeOut returns true, if queueItem is fresher than 1 minute and it's state is not ABORT/ERROR\n    // otherwise it sets queueItem to state ABORT (408, 'Timeout')\n    logger.debug(`${correlationId}, ${importJobState}, ${operation}`);\n    const timeOut = await checkTimeOut({correlationId, operation, importJobState});\n    if (timeOut) {\n      return setImportJobState({correlationId, operation, importJobState, errorMessage, errorStatus});\n    }\n    return false;\n  }\n\n  async function query(params) {\n    const result = await db.collection(collection).find(params, {projection: {_id: 0}})\n      .toArray();\n    logger.debug(`Query result: ${result.length > 0 ? 'Found!' : 'Not found!'}`);\n    return result;\n  }\n\n  async function queryById({correlationId, checkModTime = false}) {\n    const result = await db.collection(collection).findOne({correlationId});\n    if (checkModTime) {\n      const timeOut = await checkTimeOut({correlationId});\n      if (timeOut) {\n        return queryById({correlationId});\n      }\n      return result;\n    }\n\n    return result;\n  }\n\n  // Check that if the item has waited too long\n  // If the last modification time for the queueItem is older than 1 minute\n  // set state to ABORT and return false, otherwise return true\n  // If the state is already ABORT or ERROR return false\n\n  async function checkTimeOut({correlationId}) {\n    const {modificationTime, queueitemState: oldState, importJobState} = await db.collection(collection).findOne({correlationId});\n\n    // should we check for DONE too?\n    if ([QUEUE_ITEM_STATE.ABORT, QUEUE_ITEM_STATE.ERROR, QUEUE_ITEM_STATE.DONE].includes(oldState)) {\n      logger.silly(`${correlationId} has already state: ${oldState}`);\n      return false;\n    }\n\n    const timeoutTime = moment(modificationTime).add(1, 'm');\n    logger.silly(`timeOut @ ${timeoutTime}`);\n\n    if (timeoutTime.isBefore()) {\n      const finalImportJobStates = [IMPORT_JOB_STATE.ABORT, IMPORT_JOB_STATE.DONE, IMPORT_JOB_STATE.EMPTY, IMPORT_JOB_STATE.ERROR];\n      if (!finalImportJobStates.includes(importJobState.CREATE)) { // eslint-disable-line\n        await setImportJobState({correlationId, operation: 'CREATE', importJobState: IMPORT_JOB_STATE.ABORT});\n      }\n\n      if (!finalImportJobStates.includes(importJobState.UPDATE)) { // eslint-disable-line\n        await setImportJobState({correlationId, operation: 'UPDATE', importJobState: IMPORT_JOB_STATE.ABORT});\n      }\n\n      await setState({correlationId, state: QUEUE_ITEM_STATE.ABORT, errorStatus: httpStatus.REQUEST_TIMEOUT, errorMessage: `Timeout in ${oldState}`});\n\n      return false;\n    }\n\n    return true;\n  }\n\n  async function remove(params) {\n    logger.silly(`${JSON.stringify(params)}`);\n    logger.info(`Removing from Mongo (${collection}) id: ${params.correlationId}`);\n    const clean = sanitize(params.correlationId);\n    logger.silly(`mongo/remove: clean: ${JSON.stringify(clean)}`);\n\n    try {\n      //const metadataResult = await getFileMetadata({filename: clean});\n      //logger.debug(`mongo/remove: metadataResult: ${JSON.stringify(metadataResult)}`);\n      const noContent = await removeContent(params);\n      if (noContent) {\n        await db.collection(collection).deleteOne({correlationId: clean});\n        return true;\n      }\n    } catch (err) {\n      if (err instanceof MongoDriverError) {\n        if (err.message.indexOf('File not found for id') !== -1) {\n          logger.silly(`mongo/remove: File not found, removing queueItem ${JSON.stringify(clean)} from ${collection}`);\n          await db.collection(collection).deleteOne({correlationId: clean});\n          return true;\n        }\n        logger.error(err.message);\n        throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n      }\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n    }\n  }\n\n  async function readContent(correlationId) {\n    logger.info(`Reading content from mongo for id: ${correlationId} in ${collection}`);\n    const clean = sanitize(correlationId);\n    const result = await db.collection(collection).findOne({correlationId: clean}); // njsscan-ignore: node_nosqli_injection\n\n    if (result) {\n      return {\n        contentType: result.contentType,\n        readStream: gridFSBucket.openDownloadStreamByName(clean)\n      };\n    }\n\n    throw new ApiError(httpStatus.NOT_FOUND);\n  }\n\n  async function removeContent(params) {\n    logger.info(`Removing content from mongo for id: ${params.correlationId} in ${collection}`);\n    const clean = sanitize(params.correlationId);\n\n    const result = await db.collection(collection).findOne({correlationId: clean}); // njsscan-ignore: node_nosqli_injection\n    logger.silly(`mongo/removeContent: result ${JSON.stringify(result)}`);\n\n    if (result) {\n      await gridFSBucket.delete(clean);\n      return true;\n    }\n\n    return true;\n  }\n\n  // eslint-disable-next-line max-statements\n  function getOne({queueItemState, importJobState = undefined}) {\n\n    logger.silly(`queueItemState: ${queueItemState}, importJobState: ${JSON.stringify(importJobState)}`);\n    const cleanQueueItemState = queueItemState ? {queueItemState: sanitize(queueItemState)} : undefined;\n\n    try {\n\n      // Just queueItemState\n      if (queueItemState && importJobState === undefined) {\n        logger.silly(`Checking DB ${collection} for just ${JSON.stringify(cleanQueueItemState.queueItemState)}`);\n        return db.collection(collection).findOne({...cleanQueueItemState});\n      }\n\n      // importJobState\n      if (importJobState && queueItemState === undefined) {\n        logger.silly(`Checking DB ${collection} for ${JSON.stringify(importJobState)}`);\n        return db.collection(collection).findOne({...importJobState});\n      }\n\n      // importJobState and queueItemState\n      if (importJobState && queueItemState) {\n        logger.silly(`Checking DB ${collection} for ${queueItemState} and ${JSON.stringify(importJobState)}`);\n        return db.collection(collection).findOne({...cleanQueueItemState, ...importJobState});\n      }\n\n      logger.debug(`getOne not working!`);\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, 'Missing parameters, cannot get queueItem');\n\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  function getStream(correlationId) {\n    logger.info(`Forming stream from mongo for ${correlationId} in ${collection}`);\n    const clean = sanitize(correlationId);\n    try {\n      // Return content stream\n      return gridFSBucket.openDownloadStreamByName(clean);\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function pushIds({correlationId, handledIds, rejectedIds}) {\n    logger.verbose(`Push ids (${handledIds.length}) and rejectedIds (${rejectedIds.length}) ${correlationId} to ${collection}`);\n    logger.debug(`ids (${handledIds.length}): ${JSON.stringify(handledIds)}, rejectedIds ${rejectedIds.length}: ${JSON.stringify(rejectedIds)}`);\n    const clean = sanitize(correlationId);\n    await db.collection(collection).updateOne({\n      correlationId: clean\n    }, {\n      $set: {\n        modificationTime: moment().toDate()\n      },\n      $push: {\n        handledIds: {$each: handledIds},\n        rejectedIds: {$each: rejectedIds}\n      }\n    });\n  }\n\n  async function pushMessages({correlationId, messages, messageField = 'messages'}) {\n    logger.verbose(`Push messages (${messages.length}) to ${messageField} ${correlationId} to ${collection}`);\n    logger.silly(`Messages (${messages.length}): ${JSON.stringify(messages)}}`);\n    const clean = sanitize(correlationId);\n    const cleanMessageField = sanitize(messageField);\n    await db.collection(collection).updateOne({\n      correlationId: clean\n    }, {\n      $set: {\n        modificationTime: moment().toDate()\n      },\n      $push: {\n        [cleanMessageField]: {$each: messages}\n      }\n    });\n  }\n\n  function setState({correlationId, state, errorMessage = undefined, errorStatus = undefined}) {\n    const errorString = errorMessage || errorStatus ? `, Error message: '${errorMessage || ''}', Error status: '${errorStatus || ''}'` : '';\n    logger.info(`Setting queue-item state ${state} for ${correlationId}${errorString} to ${collection}`);\n\n    const stateInQueueItemStates = Object.values(QUEUE_ITEM_STATE).indexOf(state) > -1;\n    const stateInQueueItemStatesValidator = Object.values(QUEUE_ITEM_STATE.VALIDATOR).indexOf(state) > -1;\n    const stateInQueueItemStatesImporter = Object.values(QUEUE_ITEM_STATE.IMPORTER).indexOf(state) > -1;\n    if (!stateInQueueItemStates && !stateInQueueItemStatesValidator && !stateInQueueItemStatesImporter) {\n      throw new ApiError(httpStatus.BAD_REQUEST, 'Trying to set invalid state');\n    }\n\n    const clean = sanitize(correlationId);\n    const updateValues = {\n      queueItemState: state,\n      modificationTime: moment().toDate(),\n      errorMessage,\n      errorStatus\n    };\n\n    // Do not update value that are undefined\n    // eslint-disable-next-line functional/immutable-data\n    Object.keys(updateValues).forEach(key => updateValues[key] === undefined && delete updateValues[key]);\n\n    return db.collection(collection)\n      .findOneAndUpdate(\n        {correlationId: clean},\n        {$set: updateValues},\n        {projection: {_id: 0}, returnNewDocument: true}\n      );\n  }\n\n  function setImportJobState({correlationId, operation, importJobState}) {\n\n    // should this also get importJobState as previously created object? {}\n    logger.info(`Setting queue-item importJobState: {${operation}: ${importJobState}} for ${correlationId}`);\n    const cleanCorrelationId = sanitize(correlationId);\n    const cleanImportJobState = sanitize(importJobState);\n\n    if (!(operation in OPERATIONS)) {\n      throw new ApiError('400', 'Invalid operation for import job state');\n    }\n\n    if (!(cleanImportJobState in IMPORT_JOB_STATE)) {\n      throw new ApiError('400', 'Invalid import job state');\n    }\n\n    const newJobState = operation === OPERATIONS.CREATE ? {'importJobState.CREATE': cleanImportJobState} : {'importJobState.UPDATE': cleanImportJobState};\n    return db.collection(collection).findOneAndUpdate({\n      correlationId: cleanCorrelationId\n    }, {\n      $set: {\n        ...newJobState,\n        modificationTime: moment().toDate()\n      }\n    }, {projection: {_id: 0}, returnNewDocument: true});\n  }\n\n  async function setOperation({correlationId, operation}) {\n    const newOperation = operation;\n    const {operation: oldOperation} = await db.collection(collection).findOne({correlationId});\n    logger.info(`Setting queue-item operation from ${oldOperation} to ${newOperation} for ${correlationId} to ${collection}`);\n    const cleanCorrelationId = sanitize(correlationId);\n    const cleanNewOperation = sanitize(newOperation);\n\n    const result = await db.collection(collection).findOneAndUpdate({\n      correlationId: cleanCorrelationId\n    }, {\n      $set: {\n        operation: cleanNewOperation,\n        modificationTime: moment().toDate()\n      }\n    }, {projection: {_id: 0}, returnNewDocument: true});\n\n    return result.ok;\n    // logger.debug(JSON.stringify(result));\n  }\n\n  async function setOperations({correlationId, addOperation, removeOperation = undefined}) {\n    const cleanCorrelationId = sanitize(correlationId);\n\n    //logger.info(`Setting queue-item operation from ${JSON.stringify(oldOperations)} by adding ${addOperation} and removing ${removeOperation} for ${correlationId} to ${collection}`);\n\n    const queueItem = await db.collection(collection).findOne({correlationId: cleanCorrelationId});\n    logger.debug(`We fould a queueItem: ${JSON.stringify(queueItem)}`);\n\n    const oldOperations = queueItem.operations;\n\n    logger.info(`Setting queue-item operation from ${JSON.stringify(oldOperations)} by adding ${addOperation} and removing ${removeOperation} for ${correlationId} to ${collection}`);\n\n    const operationsAfterRemove = oldOperations.filter(operation => operation !== removeOperation);\n    const operationsAfterRemoveAndAdd = [addOperation, ...operationsAfterRemove];\n\n    const result = await db.collection(collection).findOneAndUpdate({\n      correlationId: cleanCorrelationId\n    }, {\n      $set: {\n        operations: operationsAfterRemoveAndAdd,\n        modificationTime: moment().toDate()\n      }\n    }, {projection: {_id: 0}, returnNewDocument: true});\n\n    return result.ok;\n    // logger.debug(JSON.stringify(result));\n  }\n}\n"],"file":"mongo.js"}

package/dist/utils.js

@@ -3,12 +3,21 @@
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
+exports.addRecordResponseItem = addRecordResponseItem;
+exports.createImportJobState = createImportJobState;
+exports.createRecordResponseItem = createRecordResponseItem;
 exports.logError = logError;
 
 var _melindaBackendCommons = require("@natlibfi/melinda-backend-commons");
 
 var _melindaCommons = require("@natlibfi/melinda-commons");
 
+var _constants = require("./constants");
+
+var _httpStatus = _interopRequireDefault(require("http-status"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
 /**
 *
 * @licstart  The following is the entire license notice for the JavaScript code in this file.
@@ -51,4 +60,127 @@ function logError(err) {
 
   logger.error(err.stack === undefined ? err : err.stack);
 }
+
+function createImportJobState(operation, state, queryImportJobState = false) {
+  if (!(state in _constants.IMPORT_JOB_STATE)) {
+    throw new Error('Invalid IMPORT_JOB_STATE');
+  }
+
+  if (!(operation in _constants.OPERATIONS)) {
+    throw new Error('Invalid operation');
+  }
+
+  if (queryImportJobState) {
+    const importJobStateWithOperation = `importJobState.${operation}`;
+    const importJobStateForQuery = {
+      [importJobStateWithOperation]: state
+    };
+    return importJobStateForQuery;
+  }
+
+  return {
+    [operation]: state
+  };
+}
+
+function createRecordResponseItem({
+  responsePayload,
+  responseStatus,
+  recordMetadata,
+  id
+}) {
+  const recordResponseStatusAndMessage = getRecordResponseStatusAndMessage(responseStatus, responsePayload);
+  const recordResponseItem = {
+    melindaId: id || '000000000',
+    recordMetadata: responsePayload.recordMetadata || recordMetadata || undefined,
+    ...recordResponseStatusAndMessage
+  };
+  return recordResponseItem;
+}
+
+function getRecordResponseStatusAndMessage(responseStatus, responsePayload) {
+  logger.debug(`Response status: ${responseStatus} responsePayload: ${JSON.stringify(responsePayload)}`);
+  const responseStatusName = _httpStatus.default[`${responseStatus}_NAME`];
+  logger.debug(`Response status name: ${responseStatusName}`);
+  const message = getMessageFromResponsePayload(responsePayload);
+  logger.debug(`Response message: ${message}`); // Non-http statuses
+
+  if (['UPDATED', 'CREATED', 'INVALID', 'ERROR'].includes(responseStatus)) {
+    return {
+      status: responseStatus,
+      message
+    };
+  }
+
+  if (['UNKNOWN'].includes(responseStatus)) {
+    const possibleIds = responsePayload.ids || [];
+    return {
+      status: responseStatus,
+      message,
+      possibleIds
+    };
+  } // Duplicates and other CONFLICT statuses
+
+
+  if ([_httpStatus.default.CONFLICT, 'CONFLICT'].includes(responseStatus)) {
+    if (/^Duplicates in database/u.test(message)) {
+      const duplicateIds = responsePayload.ids || [];
+      return {
+        status: 'DUPLICATE',
+        message,
+        duplicateIds
+      };
+    }
+
+    return {
+      status: 'CONFLICT',
+      message
+    };
+  }
+
+  if ([_httpStatus.default.UNPROCESSABLE_ENTITY, 'UNPROCESSABLE_ENTITY'].includes(responseStatus)) {
+    return {
+      status: 'UNPROCESSABLE_ENTITY',
+      message
+    };
+  }
+
+  if ([_httpStatus.default.NOT_FOUND, 'NOT_FOUND'].includes(responseStatus)) {
+    return {
+      status: 'NOT_FOUND',
+      message
+    };
+  } // Otherwise
+
+
+  return {
+    status: 'ERROR',
+    message
+  };
+}
+
+function getMessageFromResponsePayload(responsePayload) {
+  if (responsePayload.message) {
+    return responsePayload.message;
+  }
+
+  if (typeof responsePayload === 'string' || responsePayload instanceof String) {
+    return responsePayload;
+  }
+
+  return '';
+}
+
+async function addRecordResponseItem({
+  recordResponseItem,
+  correlationId,
+  mongoOperator
+}) {
+  await mongoOperator.pushMessages({
+    correlationId,
+    messages: [recordResponseItem],
+    messageField: 'records'
+  });
+  return true;
+}
 //# sourceMappingURL=utils.js.map

package/dist/utils.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/utils.js"],"names":["logger","logError","err","ApiError","error","JSON","stringify","stack","undefined"],"mappings":";;;;;;;AA4BA;;AACA;;AA7BA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAKA,MAAMA,MAAM,GAAG,0CAAf;;AAEO,SAASC,QAAT,CAAkBC,GAAlB,EAAuB;AAC5B,MAAIA,GAAG,YAAYC,qBAAnB,EAA6B;AAC3BH,IAAAA,MAAM,CAACI,KAAP,CAAaC,IAAI,CAACC,SAAL,CAAeJ,GAAf,CAAb;AACA;AACD;;AAED,MAAIA,GAAG,KAAK,QAAZ,EAAsB;AACpBF,IAAAA,MAAM,CAACI,KAAP,CAAaF,GAAb;AACA;AACD;;AAEDF,EAAAA,MAAM,CAACI,KAAP,CAAaF,GAAG,CAACK,KAAJ,KAAcC,SAAd,GAA0BN,GAA1B,GAAgCA,GAAG,CAACK,KAAjD;AACD","sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020-2021 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {Error as ApiError} from '@natlibfi/melinda-commons';\n\nconst logger = createLogger();\n\nexport function logError(err) {\n  if (err instanceof ApiError) {\n    logger.error(JSON.stringify(err));\n    return;\n  }\n\n  if (err === 'SIGINT') {\n    logger.error(err);\n    return;\n  }\n\n  logger.error(err.stack === undefined ? err : err.stack);\n}\n"],"file":"utils.js"}
+{"version":3,"sources":["../src/utils.js"],"names":["logger","logError","err","ApiError","error","JSON","stringify","stack","undefined","createImportJobState","operation","state","queryImportJobState","IMPORT_JOB_STATE","Error","OPERATIONS","importJobStateWithOperation","importJobStateForQuery","createRecordResponseItem","responsePayload","responseStatus","recordMetadata","id","recordResponseStatusAndMessage","getRecordResponseStatusAndMessage","recordResponseItem","melindaId","debug","responseStatusName","httpStatus","message","getMessageFromResponsePayload","includes","status","possibleIds","ids","CONFLICT","test","duplicateIds","UNPROCESSABLE_ENTITY","NOT_FOUND","String","addRecordResponseItem","correlationId","mongoOperator","pushMessages","messages","messageField"],"mappings":";;;;;;;;;;AA4BA;;AACA;;AACA;;AACA;;;;AA/BA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAOA,MAAMA,MAAM,GAAG,0CAAf;;AAEO,SAASC,QAAT,CAAkBC,GAAlB,EAAuB;AAC5B,MAAIA,GAAG,YAAYC,qBAAnB,EAA6B;AAC3BH,IAAAA,MAAM,CAACI,KAAP,CAAaC,IAAI,CAACC,SAAL,CAAeJ,GAAf,CAAb;AACA;AACD;;AAED,MAAIA,GAAG,KAAK,QAAZ,EAAsB;AACpBF,IAAAA,MAAM,CAACI,KAAP,CAAaF,GAAb;AACA;AACD;;AAEDF,EAAAA,MAAM,CAACI,KAAP,CAAaF,GAAG,CAACK,KAAJ,KAAcC,SAAd,GAA0BN,GAA1B,GAAgCA,GAAG,CAACK,KAAjD;AACD;;AAEM,SAASE,oBAAT,CAA8BC,SAA9B,EAAyCC,KAAzC,EAAgDC,mBAAmB,GAAG,KAAtE,EAA6E;AAClF,MAAI,EAAED,KAAK,IAAIE,2BAAX,CAAJ,EAAkC;AAChC,UAAM,IAAIC,KAAJ,CAAU,0BAAV,CAAN;AACD;;AAED,MAAI,EAAEJ,SAAS,IAAIK,qBAAf,CAAJ,EAAgC;AAC9B,UAAM,IAAID,KAAJ,CAAU,mBAAV,CAAN;AACD;;AAED,MAAIF,mBAAJ,EAAyB;AACvB,UAAMI,2BAA2B,GAAI,kBAAiBN,SAAU,EAAhE;AACA,UAAMO,sBAAsB,GAAG;AAAC,OAACD,2BAAD,GAA+BL;AAAhC,KAA/B;AAEA,WAAOM,sBAAP;AACD;;AAED,SAAO;AAAC,KAACP,SAAD,GAAaC;AAAd,GAAP;AACD;;AAEM,SAASO,wBAAT,CAAkC;AAACC,EAAAA,eAAD;AAAkBC,EAAAA,cAAlB;AAAkCC,EAAAA,cAAlC;AAAkDC,EAAAA;AAAlD,CAAlC,EAAyF;AAC9F,QAAMC,8BAA8B,GAAGC,iCAAiC,CAACJ,cAAD,EAAiBD,eAAjB,CAAxE;AACA,QAAMM,kBAAkB,GAAG;AACzBC,IAAAA,SAAS,EAAEJ,EAAE,IAAI,WADQ;AAEzBD,IAAAA,cAAc,EAAEF,eAAe,CAACE,cAAhB,IAAkCA,cAAlC,IAAoDb,SAF3C;AAGzB,OAAGe;AAHsB,GAA3B;AAKA,SAAOE,kBAAP;AACD;;AAED,SAASD,iCAAT,CAA2CJ,cAA3C,EAA2DD,eAA3D,EAA4E;AAE1EnB,EAAAA,MAAM,CAAC2B,KAAP,CAAc,oBAAmBP,cAAe,qBAAoBf,IAAI,CAACC,SAAL,CAAea,eAAf,CAAgC,EAApG;AACA,QAAMS,kBAAkB,GAAGC,oBAAY,GAAET,cAAe,OAA7B,CAA3B;AACApB,EAAAA,MAAM,CAAC2B,KAAP,CAAc,yBAAwBC,kBAAmB,EAAzD;AAEA,QAAME,OAAO,GAAGC,6BAA6B,CAACZ,eAAD,CAA7C;AAEAnB,EAAAA,MAAM,CAAC2B,KAAP,CAAc,qBAAoBG,OAAQ,EAA1C,EAR0E,CAU1E;;AACA,MAAI,CAAC,SAAD,EAAY,SAAZ,EAAuB,SAAvB,EAAkC,OAAlC,EAA2CE,QAA3C,CAAoDZ,cAApD,CAAJ,EAAyE;AACvE,WAAO;AAACa,MAAAA,MAAM,EAAEb,cAAT;AAAyBU,MAAAA;AAAzB,KAAP;AACD;;AAED,MAAI,CAAC,SAAD,EAAYE,QAAZ,CAAqBZ,cAArB,CAAJ,EAA0C;AACxC,UAAMc,WAAW,GAAGf,eAAe,CAACgB,GAAhB,IAAuB,EAA3C;AACA,WAAO;AAACF,MAAAA,MAAM,EAAEb,cAAT;AAAyBU,MAAAA,OAAzB;AAAkCI,MAAAA;AAAlC,KAAP;AACD,GAlByE,CAoB1E;;;AACA,MAAI,CAACL,oBAAWO,QAAZ,EAAsB,UAAtB,EAAkCJ,QAAlC,CAA2CZ,cAA3C,CAAJ,EAAgE;AAC9D,QAAK,0BAAD,CAA6BiB,IAA7B,CAAkCP,OAAlC,CAAJ,EAAgD;AAC9C,YAAMQ,YAAY,GAAGnB,eAAe,CAACgB,GAAhB,IAAuB,EAA5C;AACA,aAAO;AAACF,QAAAA,MAAM,EAAE,WAAT;AAAsBH,QAAAA,OAAtB;AAA+BQ,QAAAA;AAA/B,OAAP;AACD;;AACD,WAAO;AAACL,MAAAA,MAAM,EAAE,UAAT;AAAqBH,MAAAA;AAArB,KAAP;AACD;;AAED,MAAI,CAACD,oBAAWU,oBAAZ,EAAkC,sBAAlC,EAA0DP,QAA1D,CAAmEZ,cAAnE,CAAJ,EAAwF;AACtF,WAAO;AAACa,MAAAA,MAAM,EAAE,sBAAT;AAAiCH,MAAAA;AAAjC,KAAP;AACD;;AAED,MAAI,CAACD,oBAAWW,SAAZ,EAAuB,WAAvB,EAAoCR,QAApC,CAA6CZ,cAA7C,CAAJ,EAAkE;AAChE,WAAO;AAACa,MAAAA,MAAM,EAAE,WAAT;AAAsBH,MAAAA;AAAtB,KAAP;AACD,GAnCyE,CAqC1E;;;AACA,SAAO;AAACG,IAAAA,MAAM,EAAE,OAAT;AAAkBH,IAAAA;AAAlB,GAAP;AACD;;AAED,SAASC,6BAAT,CAAuCZ,eAAvC,EAAwD;AACtD,MAAIA,eAAe,CAACW,OAApB,EAA6B;AAC3B,WAAOX,eAAe,CAACW,OAAvB;AACD;;AACD,MAAI,OAAOX,eAAP,KAA2B,QAA3B,IAAuCA,eAAe,YAAYsB,MAAtE,EAA8E;AAC5E,WAAOtB,eAAP;AACD;;AACD,SAAO,EAAP;AACD;;AAEM,eAAeuB,qBAAf,CAAqC;AAACjB,EAAAA,kBAAD;AAAqBkB,EAAAA,aAArB;AAAoCC,EAAAA;AAApC,CAArC,EAAyF;AAC9F,QAAMA,aAAa,CAACC,YAAd,CAA2B;AAACF,IAAAA,aAAD;AAAgBG,IAAAA,QAAQ,EAAE,CAACrB,kBAAD,CAA1B;AAAgDsB,IAAAA,YAAY,EAAE;AAA9D,GAA3B,CAAN;AACA,SAAO,IAAP;AACD","sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020-2021 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {Error as ApiError} from '@natlibfi/melinda-commons';\nimport {IMPORT_JOB_STATE, OPERATIONS} from './constants';\nimport httpStatus from 'http-status';\n\nconst logger = createLogger();\n\nexport function logError(err) {\n  if (err instanceof ApiError) {\n    logger.error(JSON.stringify(err));\n    return;\n  }\n\n  if (err === 'SIGINT') {\n    logger.error(err);\n    return;\n  }\n\n  logger.error(err.stack === undefined ? err : err.stack);\n}\n\nexport function createImportJobState(operation, state, queryImportJobState = false) {\n  if (!(state in IMPORT_JOB_STATE)) {\n    throw new Error('Invalid IMPORT_JOB_STATE');\n  }\n\n  if (!(operation in OPERATIONS)) {\n    throw new Error('Invalid operation');\n  }\n\n  if (queryImportJobState) {\n    const importJobStateWithOperation = `importJobState.${operation}`;\n    const importJobStateForQuery = {[importJobStateWithOperation]: state};\n\n    return importJobStateForQuery;\n  }\n\n  return {[operation]: state};\n}\n\nexport function createRecordResponseItem({responsePayload, responseStatus, recordMetadata, id}) {\n  const recordResponseStatusAndMessage = getRecordResponseStatusAndMessage(responseStatus, responsePayload);\n  const recordResponseItem = {\n    melindaId: id || '000000000',\n    recordMetadata: responsePayload.recordMetadata || recordMetadata || undefined,\n    ...recordResponseStatusAndMessage\n  };\n  return recordResponseItem;\n}\n\nfunction getRecordResponseStatusAndMessage(responseStatus, responsePayload) {\n\n  logger.debug(`Response status: ${responseStatus} responsePayload: ${JSON.stringify(responsePayload)}`);\n  const responseStatusName = httpStatus[`${responseStatus}_NAME`];\n  logger.debug(`Response status name: ${responseStatusName}`);\n\n  const message = getMessageFromResponsePayload(responsePayload);\n\n  logger.debug(`Response message: ${message}`);\n\n  // Non-http statuses\n  if (['UPDATED', 'CREATED', 'INVALID', 'ERROR'].includes(responseStatus)) {\n    return {status: responseStatus, message};\n  }\n\n  if (['UNKNOWN'].includes(responseStatus)) {\n    const possibleIds = responsePayload.ids || [];\n    return {status: responseStatus, message, possibleIds};\n  }\n\n  // Duplicates and other CONFLICT statuses\n  if ([httpStatus.CONFLICT, 'CONFLICT'].includes(responseStatus)) {\n    if ((/^Duplicates in database/u).test(message)) {\n      const duplicateIds = responsePayload.ids || [];\n      return {status: 'DUPLICATE', message, duplicateIds};\n    }\n    return {status: 'CONFLICT', message};\n  }\n\n  if ([httpStatus.UNPROCESSABLE_ENTITY, 'UNPROCESSABLE_ENTITY'].includes(responseStatus)) {\n    return {status: 'UNPROCESSABLE_ENTITY', message};\n  }\n\n  if ([httpStatus.NOT_FOUND, 'NOT_FOUND'].includes(responseStatus)) {\n    return {status: 'NOT_FOUND', message};\n  }\n\n  // Otherwise\n  return {status: 'ERROR', message};\n}\n\nfunction getMessageFromResponsePayload(responsePayload) {\n  if (responsePayload.message) {\n    return responsePayload.message;\n  }\n  if (typeof responsePayload === 'string' || responsePayload instanceof String) {\n    return responsePayload;\n  }\n  return '';\n}\n\nexport async function addRecordResponseItem({recordResponseItem, correlationId, mongoOperator}) {\n  await mongoOperator.pushMessages({correlationId, messages: [recordResponseItem], messageField: 'records'});\n  return true;\n}\n\n"],"file":"utils.js"}

package/package.json

@@ -14,7 +14,7 @@
 		"url": "git@github.com:natlibfi/melinda-rest-api-commons.git"
 	},
 	"license": "AGPL-3.0+",
-	"version": "2.2.0-alpha.1",
+	"version": "2.2.0-alpha.10",
 	"main": "./dist/index.js",
 	"engines": {
 		"node": ">=14"
@@ -33,7 +33,6 @@
 		"build": "babel src --source-maps --copy-files --delete-dir-on-start --out-dir=dist"
 	},
 	"dependencies": {
-		"@natlibfi/fixura": "^2.2.1",
 		"@natlibfi/marc-record": "^7.0.0",
 		"@natlibfi/marc-record-serializers": "^8.1.0",
 		"@natlibfi/marc-record-validate": "^7.0.1",
@@ -42,26 +41,26 @@
 		"@natlibfi/melinda-commons": "^12.0.3",
 		"amqplib": ">=0.8.0 <1.0.0",
 		"http-status": "^1.5.0",
-		"moment": "^2.29.1",
+		"moment": "^2.29.2",
 		"mongo-sanitize": "^1.1.0",
-		"mongodb": "^4.1.3"
+		"mongodb": "^4.5.0"
 	},
 	"devDependencies": {
-		"@babel/cli": "^7.15.7",
-		"@babel/core": "^7.15.8",
+		"@babel/cli": "^7.17.6",
+		"@babel/core": "^7.17.8",
+		"@babel/eslint-parser": "^7.17.0",
 		"@babel/plugin-syntax-dynamic-import": "^7.8.3",
-		"@babel/preset-env": "^7.15.8",
+		"@babel/preset-env": "^7.16.11",
 		"@natlibfi/eslint-config-melinda-backend": "^2.0.0",
-		"@babel/eslint-parser": "^7.15.8",
+		"@natlibfi/fixura": "^2.2.1",
 		"babel-plugin-dynamic-import-node": "^2.3.3",
 		"babel-plugin-istanbul": "^6.1.1",
 		"babel-plugin-rewire": "^1.2.0",
-		"chai": "^4.3.4",
+		"chai": "^4.3.6",
 		"cross-env": "^7.0.3",
-		"eslint": "^8.10.0",
-		"eslint-plugin-import": "^2.25.2",
-		"mocha": "^9.2.1",
-		"nodemon": "^2.0.14",
+		"eslint": "^8.12.0",
+		"mocha": "^9.2.2",
+		"nodemon": "^2.0.15",
 		"nyc": "^15.1.0"
 	},
 	"eslintConfig": {