@natlibfi/melinda-rest-api-commons 2.1.2 → 2.2.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/constants.js +2 -1
- package/dist/constants.js.map +1 -1
- package/dist/mongo.js +28 -12
- package/dist/mongo.js.map +1 -1
- package/package.json +9 -9
package/dist/constants.js
CHANGED
|
@@ -38,8 +38,9 @@ const QUEUE_ITEM_STATE = {
|
|
|
38
38
|
PENDING_QUEUING: 'PENDING_QUEUING',
|
|
39
39
|
PENDING_VALIDATION: 'PENDING_VALIDATION',
|
|
40
40
|
QUEUING_IN_PROGRESS: 'QUEUING_IN_PROGRESS',
|
|
41
|
+
UPLOADING: 'UPLOADING',
|
|
41
42
|
VALIDATING: 'VALIDATING',
|
|
42
|
-
|
|
43
|
+
WAITING_FOR_RECORDS: 'WAITING_FOR_RECORDS'
|
|
43
44
|
},
|
|
44
45
|
IMPORTER: {
|
|
45
46
|
IMPORTING: 'IMPORTING',
|
package/dist/constants.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/constants.js"],"names":["QUEUE_ITEM_STATE","VALIDATOR","PENDING_QUEUING","PENDING_VALIDATION","QUEUING_IN_PROGRESS","VALIDATING","
|
|
1
|
+
{"version":3,"sources":["../src/constants.js"],"names":["QUEUE_ITEM_STATE","VALIDATOR","PENDING_QUEUING","PENDING_VALIDATION","QUEUING_IN_PROGRESS","UPLOADING","VALIDATING","WAITING_FOR_RECORDS","IMPORTER","IMPORTING","IN_PROCESS","IN_QUEUE","DONE","ERROR","ABORT","OPERATIONS","CREATE","UPDATE","CHUNK_SIZE","conversionFormats","MARCXML","ISO2709","JSON"],"mappings":";;;;;;;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAEA;AAEO,MAAMA,gBAAgB,GAAG;AAC9BC,EAAAA,SAAS,EAAE;AACTC,IAAAA,eAAe,EAAE,iBADR;AAETC,IAAAA,kBAAkB,EAAE,oBAFX;AAGTC,IAAAA,mBAAmB,EAAE,qBAHZ;AAITC,IAAAA,SAAS,EAAE,WAJF;AAKTC,IAAAA,UAAU,EAAE,YALH;AAMTC,IAAAA,mBAAmB,EAAE;AANZ,GADmB;AAS9BC,EAAAA,QAAQ,EAAE;AACRC,IAAAA,SAAS,EAAE,WADH;AAERC,IAAAA,UAAU,EAAE,YAFJ;AAGRC,IAAAA,QAAQ,EAAE;AAHF,GAToB;AAc9BC,EAAAA,IAAI,EAAE,MAdwB;AAe9BC,EAAAA,KAAK,EAAE,OAfuB;AAgB9BC,EAAAA,KAAK,EAAE;AAhBuB,CAAzB;;AAmBA,MAAMC,UAAU,GAAG;AACxBC,EAAAA,MAAM,EAAE,QADgB;AAExBC,EAAAA,MAAM,EAAE;AAFgB,CAAnB;;AAKA,MAAMC,UAAU,GAAG,GAAnB;;AAEA,MAAMC,iBAAiB,GAAG;AAC/BC,EAAAA,OAAO,EAAE,CADsB;AAE/BC,EAAAA,OAAO,EAAE,CAFsB;AAG/BC,EAAAA,IAAI,EAAE;AAHyB,CAA1B","sourcesContent":["/**\n*\n* @licstart The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020-2021 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program. If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\n// QueueItemState flows: see README.md\n\nexport const QUEUE_ITEM_STATE = {\n VALIDATOR: {\n PENDING_QUEUING: 'PENDING_QUEUING',\n PENDING_VALIDATION: 'PENDING_VALIDATION',\n QUEUING_IN_PROGRESS: 'QUEUING_IN_PROGRESS',\n UPLOADING: 'UPLOADING',\n VALIDATING: 'VALIDATING',\n WAITING_FOR_RECORDS: 'WAITING_FOR_RECORDS'\n },\n IMPORTER: {\n IMPORTING: 'IMPORTING',\n IN_PROCESS: 'IN_PROCESS',\n IN_QUEUE: 'IN_QUEUE'\n },\n DONE: 'DONE',\n ERROR: 'ERROR',\n ABORT: 'ABORT'\n};\n\nexport const OPERATIONS = {\n CREATE: 'CREATE',\n UPDATE: 'UPDATE'\n};\n\nexport const CHUNK_SIZE = 100;\n\nexport const conversionFormats = {\n MARCXML: 1,\n ISO2709: 2,\n JSON: 3\n};\n"],"file":"constants.js"}
|
package/dist/mongo.js
CHANGED
|
@@ -161,7 +161,7 @@ async function _default(MONGO_URI, collection) {
|
|
|
161
161
|
},
|
|
162
162
|
contentType,
|
|
163
163
|
recordLoadParams,
|
|
164
|
-
queueItemState: _constants.QUEUE_ITEM_STATE.VALIDATOR.UPLOADING,
|
|
164
|
+
queueItemState: stream ? _constants.QUEUE_ITEM_STATE.VALIDATOR.UPLOADING : _constants.QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS,
|
|
165
165
|
creationTime: time,
|
|
166
166
|
modificationTime: time,
|
|
167
167
|
handledIds: [],
|
|
@@ -177,6 +177,10 @@ async function _default(MONGO_URI, collection) {
|
|
|
177
177
|
throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR);
|
|
178
178
|
}
|
|
179
179
|
|
|
180
|
+
if (!stream) {
|
|
181
|
+
return correlationId;
|
|
182
|
+
}
|
|
183
|
+
|
|
180
184
|
return new Promise((resolve, reject) => {
|
|
181
185
|
const outputStream = gridFSBucket.openUploadStream(correlationId);
|
|
182
186
|
stream.on('error', reject).on('data', chunk => outputStream.write(chunk)).on('end', () => outputStream.end(undefined, undefined, () => {
|
|
@@ -189,8 +193,8 @@ async function _default(MONGO_URI, collection) {
|
|
|
189
193
|
async function checkAndSetState({
|
|
190
194
|
correlationId,
|
|
191
195
|
state,
|
|
192
|
-
errorMessage =
|
|
193
|
-
errorStatus =
|
|
196
|
+
errorMessage = undefined,
|
|
197
|
+
errorStatus = undefined
|
|
194
198
|
}) {
|
|
195
199
|
// checkTimeOut returns true, if queueItem is fresher than 1 minute and it's state is not ABORT/ERROR
|
|
196
200
|
// otherwise it sets queueItem to state ABORT (408, 'Timeout')
|
|
@@ -428,21 +432,33 @@ async function _default(MONGO_URI, collection) {
|
|
|
428
432
|
function setState({
|
|
429
433
|
correlationId,
|
|
430
434
|
state,
|
|
431
|
-
errorMessage =
|
|
432
|
-
errorStatus =
|
|
435
|
+
errorMessage = undefined,
|
|
436
|
+
errorStatus = undefined
|
|
433
437
|
}) {
|
|
434
|
-
const errorString = errorMessage || errorStatus ? `, Error message: '${errorMessage}', Error status: '${errorStatus}'` : '';
|
|
438
|
+
const errorString = errorMessage || errorStatus ? `, Error message: '${errorMessage || ''}', Error status: '${errorStatus || ''}'` : '';
|
|
435
439
|
logger.info(`Setting queue-item state ${state} for ${correlationId}${errorString} to ${collection}`);
|
|
440
|
+
const stateInQueueItemStates = Object.values(_constants.QUEUE_ITEM_STATE).indexOf(state) > -1;
|
|
441
|
+
const stateInQueueItemStatesValidator = Object.values(_constants.QUEUE_ITEM_STATE.VALIDATOR).indexOf(state) > -1;
|
|
442
|
+
const stateInQueueItemStatesImporter = Object.values(_constants.QUEUE_ITEM_STATE.IMPORTER).indexOf(state) > -1;
|
|
443
|
+
|
|
444
|
+
if (!stateInQueueItemStates && !stateInQueueItemStatesValidator && !stateInQueueItemStatesImporter) {
|
|
445
|
+
throw new _melindaCommons.Error(_httpStatus.default.BAD_REQUEST, 'Trying to set invalid state');
|
|
446
|
+
}
|
|
447
|
+
|
|
436
448
|
const clean = (0, _mongoSanitize.default)(correlationId);
|
|
449
|
+
const updateValues = {
|
|
450
|
+
queueItemState: state,
|
|
451
|
+
modificationTime: (0, _moment.default)().toDate(),
|
|
452
|
+
errorMessage,
|
|
453
|
+
errorStatus
|
|
454
|
+
}; // Do not update value that are undefined
|
|
455
|
+
// eslint-disable-next-line functional/immutable-data
|
|
456
|
+
|
|
457
|
+
Object.keys(updateValues).forEach(key => updateValues[key] === undefined && delete updateValues[key]);
|
|
437
458
|
return db.collection(collection).findOneAndUpdate({
|
|
438
459
|
correlationId: clean
|
|
439
460
|
}, {
|
|
440
|
-
$set:
|
|
441
|
-
queueItemState: state,
|
|
442
|
-
modificationTime: (0, _moment.default)().toDate(),
|
|
443
|
-
errorMessage,
|
|
444
|
-
errorStatus
|
|
445
|
-
}
|
|
461
|
+
$set: updateValues
|
|
446
462
|
}, {
|
|
447
463
|
projection: {
|
|
448
464
|
_id: 0
|
package/dist/mongo.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/mongo.js"],"names":["MONGO_URI","collection","logger","client","MongoClient","connect","useNewUrlParser","useUnifiedTopology","db","gridFSBucket","GridFSBucket","bucketName","createPrio","createBulk","checkAndSetState","query","queryById","remove","readContent","removeContent","getOne","getStream","setState","pushIds","pushMessages","correlationId","cataloger","oCatalogerIn","operation","noop","undefined","unique","prio","time","toDate","newQueueItem","operationSettings","queueItemState","QUEUE_ITEM_STATE","VALIDATOR","PENDING_VALIDATION","creationTime","modificationTime","handledIds","rejectedIds","result","insertOne","acknowledged","info","ApiError","httpStatus","INTERNAL_SERVER_ERROR","error","contentType","recordLoadParams","stream","UPLOADING","Promise","resolve","reject","outputStream","openUploadStream","on","chunk","write","end","state","errorMessage","errorStatus","timeOut","checkTimeOut","params","find","projection","_id","toArray","debug","length","checkModTime","findOne","queueitemState","oldState","ABORT","ERROR","includes","silly","timeoutTime","add","isBefore","REQUEST_TIMEOUT","JSON","stringify","clean","noContent","deleteOne","err","MongoDriverError","message","indexOf","readStream","openDownloadStreamByName","NOT_FOUND","delete","clean2","verbose","updateOne","$set","$push","$each","messages","messageField","cleanMessageField","errorString","findOneAndUpdate","returnNewDocument"],"mappings":";;;;;;;AA4BA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AAnCA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAWA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAEe,wBAAgBA,SAAhB,EAA2BC,UAA3B,EAAuC;AACpD,QAAMC,MAAM,GAAG,0CAAf,CADoD,CAGpD;;AACA,QAAMC,MAAM,GAAG,MAAMC,qBAAYC,OAAZ,CAAoBL,SAApB,EAA+B;AAACM,IAAAA,eAAe,EAAE,IAAlB;AAAwBC,IAAAA,kBAAkB,EAAE;AAA5C,GAA/B,CAArB;AACA,QAAMC,EAAE,GAAGL,MAAM,CAACK,EAAP,CAAU,UAAV,CAAX;AACA,QAAMC,YAAY,GAAG,IAAIC,qBAAJ,CAAiBF,EAAjB,EAAqB;AAACG,IAAAA,UAAU,EAAEV;AAAb,GAArB,CAArB;AAEA,SAAO;AAACW,IAAAA,UAAD;AAAaC,IAAAA,UAAb;AAAyBC,IAAAA,gBAAzB;AAA2CC,IAAAA,KAA3C;AAAkDC,IAAAA,SAAlD;AAA6DC,IAAAA,MAA7D;AAAqEC,IAAAA,WAArE;AAAkFC,IAAAA,aAAlF;AAAiGC,IAAAA,MAAjG;AAAyGC,IAAAA,SAAzG;AAAoHC,IAAAA,QAApH;AAA8HC,IAAAA,OAA9H;AAAuIC,IAAAA;AAAvI,GAAP;;AAEA,iBAAeZ,UAAf,CAA0B;AAACa,IAAAA,aAAD;AAAgBC,IAAAA,SAAhB;AAA2BC,IAAAA,YAA3B;AAAyCC,IAAAA,SAAzC;AAAoDC,IAAAA,IAAI,GAAGC,SAA3D;AAAsEC,IAAAA,MAAM,GAAGD,SAA/E;AAA0FE,IAAAA,IAAI,GAAG;AAAjG,GAA1B,EAAkI;AAChI,UAAMC,IAAI,GAAG,uBAASC,MAAT,EAAb;AACA,UAAMC,YAAY,GAAG;AACnBV,MAAAA,aADmB;AAEnBC,MAAAA,SAFmB;AAGnBE,MAAAA,SAHmB;AAInBD,MAAAA,YAJmB;AAKnBS,MAAAA,iBAAiB,EAAE;AAACL,QAAAA,MAAD;AAASF,QAAAA,IAAT;AAAeG,QAAAA;AAAf,OALA;AAMnBK,MAAAA,cAAc,EAAEC,4BAAiBC,SAAjB,CAA2BC,kBANxB;AAOnBC,MAAAA,YAAY,EAAER,IAPK;AAQnBS,MAAAA,gBAAgB,EAAET,IARC;AASnBU,MAAAA,UAAU,EAAE,EATO;AAUnBC,MAAAA,WAAW,EAAE;AAVM,KAArB;;AAYA,QAAI;AACF,YAAMC,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B6C,SAA1B,CAAoCX,YAApC,CAArB;;AACA,UAAIU,MAAM,CAACE,YAAX,EAAyB;AACvB7C,QAAAA,MAAM,CAAC8C,IAAP,CAAa,2BAA0BpB,SAAU,IAAGH,aAAc,qBAAoBxB,UAAW,EAAjG;AACA;AACD;;AACD,YAAM,IAAIgD,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD,KAPD,CAOE,OAAOC,KAAP,EAAc;AACd,2BAASA,KAAT;AACA,YAAM,IAAIH,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD;AACF;;AAED,WAAStC,UAAT,CAAoB;AAACY,IAAAA,aAAD;AAAgBC,IAAAA,SAAhB;AAA2BC,IAAAA,YAA3B;AAAyCC,IAAAA,SAAzC;AAAoDyB,IAAAA,WAApD;AAAiEC,IAAAA,gBAAjE;AAAmFC,IAAAA,MAAnF;AAA2FvB,IAAAA,IAAI,GAAG;AAAlG,GAApB,EAA8H;AAC5H,UAAMC,IAAI,GAAG,uBAASC,MAAT,EAAb;AACA,UAAMC,YAAY,GAAG;AACnBV,MAAAA,aADmB;AAEnBC,MAAAA,SAFmB;AAGnBC,MAAAA,YAHmB;AAInBC,MAAAA,SAJmB;AAKnBQ,MAAAA,iBAAiB,EAAE;AAACJ,QAAAA;AAAD,OALA;AAMnBqB,MAAAA,WANmB;AAOnBC,MAAAA,gBAPmB;AAQnBjB,MAAAA,cAAc,EAAEC,4BAAiBC,SAAjB,CAA2BiB,SARxB;AASnBf,MAAAA,YAAY,EAAER,IATK;AAUnBS,MAAAA,gBAAgB,EAAET,IAVC;AAWnBU,MAAAA,UAAU,EAAE,EAXO;AAYnBC,MAAAA,WAAW,EAAE;AAZM,KAArB;;AAcA,QAAI;AACF;AACApC,MAAAA,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B6C,SAA1B,CAAoCX,YAApC;AACAjC,MAAAA,MAAM,CAAC8C,IAAP,CAAa,2BAA0BpB,SAAU,IAAGH,aAAc,qBAAoBxB,UAAW,GAAjG;AACD,KAJD,CAIE,OAAOmD,KAAP,EAAc;AACd,2BAASA,KAAT;AACA,YAAM,IAAIH,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD;;AAED,WAAO,IAAIM,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtC,YAAMC,YAAY,GAAGnD,YAAY,CAACoD,gBAAb,CAA8BpC,aAA9B,CAArB;AAEA8B,MAAAA,MAAM,CACHO,EADH,CACM,OADN,EACeH,MADf,EAEGG,EAFH,CAEM,MAFN,EAEcC,KAAK,IAAIH,YAAY,CAACI,KAAb,CAAmBD,KAAnB,CAFvB,EAGGD,EAHH,CAGM,KAHN,EAGa,MAAMF,YAAY,CAACK,GAAb,CAAiBnC,SAAjB,EAA4BA,SAA5B,EAAuC,MAAM;AAC5D4B,QAAAA,OAAO,CAACjC,aAAD,CAAP;AACD,OAFgB,CAHnB;AAMD,KATM,CAAP;AAUD,GAxEmD,CA0EpD;;;AACA,iBAAeX,gBAAf,CAAgC;AAACW,IAAAA,aAAD;AAAgByC,IAAAA,KAAhB;AAAuBC,IAAAA,YAAY,GAAG,EAAtC;AAA0CC,IAAAA,WAAW,GAAG;AAAxD,GAAhC,EAA6F;AAC3F;AACA;AACA,UAAMC,OAAO,GAAG,MAAMC,YAAY,CAAC7C,aAAD,CAAlC;;AACA,QAAI4C,OAAJ,EAAa;AACX,aAAO/C,QAAQ,CAAC;AAACG,QAAAA,aAAD;AAAgByC,QAAAA,KAAhB;AAAuBC,QAAAA,YAAvB;AAAqCC,QAAAA;AAArC,OAAD,CAAf;AACD;;AACD,WAAO,KAAP;AACD;;AAED,iBAAerD,KAAf,CAAqBwD,MAArB,EAA6B;AAC3B,UAAM1B,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BuE,IAA1B,CAA+BD,MAA/B,EAAuC;AAACE,MAAAA,UAAU,EAAE;AAACC,QAAAA,GAAG,EAAE;AAAN;AAAb,KAAvC,EAClBC,OADkB,EAArB;AAEAzE,IAAAA,MAAM,CAAC0E,KAAP,CAAc,iBAAgB/B,MAAM,CAACgC,MAAP,GAAgB,CAAhB,GAAoB,QAApB,GAA+B,YAAa,EAA1E;AACA,WAAOhC,MAAP;AACD;;AAED,iBAAe7B,SAAf,CAAyBS,aAAzB,EAAwCqD,YAAxC,EAAsD;AACpD,UAAMjC,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC;AAACtD,MAAAA;AAAD,KAAlC,CAArB;;AACA,QAAIqD,YAAJ,EAAkB;AAChB,YAAMT,OAAO,GAAG,MAAMC,YAAY,CAAC7C,aAAD,CAAlC;;AACA,UAAI4C,OAAJ,EAAa;AACX,eAAOrD,SAAS,CAACS,aAAD,CAAhB;AACD;;AACD,aAAOoB,MAAP;AACD;;AAED,WAAOA,MAAP;AACD,GAvGmD,CAyGpD;AACA;AACA;AACA;;;AAEA,iBAAeyB,YAAf,CAA4B7C,aAA5B,EAA2C;AACzC,UAAM;AAACiB,MAAAA,gBAAD;AAAmBsC,MAAAA,cAAc,EAAEC;AAAnC,QAA+C,MAAMzE,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC;AAACtD,MAAAA;AAAD,KAAlC,CAA3D;;AAEA,QAAI,CAACa,4BAAiB4C,KAAlB,EAAyB5C,4BAAiB6C,KAA1C,EAAiDC,QAAjD,CAA0DH,QAA1D,CAAJ,EAAyE;AACvE/E,MAAAA,MAAM,CAACmF,KAAP,CAAc,GAAE5D,aAAc,uBAAsBwD,QAAS,EAA7D;AACA,aAAO,KAAP;AACD;;AAED,UAAMK,WAAW,GAAG,qBAAO5C,gBAAP,EAAyB6C,GAAzB,CAA6B,CAA7B,EAAgC,GAAhC,CAApB;AACArF,IAAAA,MAAM,CAACmF,KAAP,CAAc,aAAYC,WAAY,EAAtC;;AAEA,QAAIA,WAAW,CAACE,QAAZ,EAAJ,EAA4B;AAC1B,YAAMlE,QAAQ,CAAC;AAACG,QAAAA,aAAD;AAAgByC,QAAAA,KAAK,EAAE5B,4BAAiB4C,KAAxC;AAA+Cd,QAAAA,WAAW,EAAElB,oBAAWuC,eAAvE;AAAwFtB,QAAAA,YAAY,EAAG,cAAac,QAAS;AAA7H,OAAD,CAAd;AACA,aAAO,KAAP;AACD;;AAED,WAAO,IAAP;AACD;;AAED,iBAAehE,MAAf,CAAsBsD,MAAtB,EAA8B;AAC5BrE,IAAAA,MAAM,CAACmF,KAAP,CAAc,GAAEK,IAAI,CAACC,SAAL,CAAepB,MAAf,CAAuB,EAAvC;AACArE,IAAAA,MAAM,CAAC8C,IAAP,CAAa,wBAAuB/C,UAAW,SAAQsE,MAAM,CAAC9C,aAAc,EAA5E;AACA,UAAMmE,KAAK,GAAG,4BAASrB,MAAM,CAAC9C,aAAhB,CAAd;AACAvB,IAAAA,MAAM,CAACmF,KAAP,CAAc,wBAAuBK,IAAI,CAACC,SAAL,CAAeC,KAAf,CAAsB,EAA3D;;AAEA,QAAI;AACF;AACA;AACA,YAAMC,SAAS,GAAG,MAAM1E,aAAa,CAACoD,MAAD,CAArC;;AACA,UAAIsB,SAAJ,EAAe;AACb,cAAMrF,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B6F,SAA1B,CAAoC;AAACrE,UAAAA,aAAa,EAAEmE;AAAhB,SAApC,CAAN;AACA,eAAO,IAAP;AACD;AACF,KARD,CAQE,OAAOG,GAAP,EAAY;AACZ,UAAIA,GAAG,YAAYC,yBAAnB,EAAqC;AACnC,YAAID,GAAG,CAACE,OAAJ,CAAYC,OAAZ,CAAoB,uBAApB,MAAiD,CAAC,CAAtD,EAAyD;AACvDhG,UAAAA,MAAM,CAACmF,KAAP,CAAc,oDAAmDK,IAAI,CAACC,SAAL,CAAeC,KAAf,CAAsB,SAAQ3F,UAAW,EAA1G;AACA,gBAAMO,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B6F,SAA1B,CAAoC;AAACrE,YAAAA,aAAa,EAAEmE;AAAhB,WAApC,CAAN;AACA,iBAAO,IAAP;AACD;;AACD1F,QAAAA,MAAM,CAACkD,KAAP,CAAa2C,GAAG,CAACE,OAAjB;AACA,cAAM,IAAIhD,qBAAJ,CAAaC,oBAAWC,qBAAxB,EAA+C4C,GAAG,CAACE,OAAnD,CAAN;AACD;;AACD,YAAM,IAAIhD,qBAAJ,CAAaC,oBAAWC,qBAAxB,EAA+C4C,GAAG,CAACE,OAAnD,CAAN;AACD;AACF;;AAED,iBAAe/E,WAAf,CAA2BO,aAA3B,EAA0C;AACxCvB,IAAAA,MAAM,CAAC8C,IAAP,CAAa,sCAAqCvB,aAAc,OAAMxB,UAAW,EAAjF;AACA,UAAM2F,KAAK,GAAG,4BAASnE,aAAT,CAAd;AACA,UAAMoB,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC;AAACtD,MAAAA,aAAa,EAAEmE;AAAhB,KAAlC,CAArB,CAHwC,CAGwC;;AAEhF,QAAI/C,MAAJ,EAAY;AACV,aAAO;AACLQ,QAAAA,WAAW,EAAER,MAAM,CAACQ,WADf;AAEL8C,QAAAA,UAAU,EAAE1F,YAAY,CAAC2F,wBAAb,CAAsCR,KAAtC;AAFP,OAAP;AAID;;AAED,UAAM,IAAI3C,qBAAJ,CAAaC,oBAAWmD,SAAxB,CAAN;AACD;;AAED,iBAAelF,aAAf,CAA6BoD,MAA7B,EAAqC;AACnCrE,IAAAA,MAAM,CAAC8C,IAAP,CAAa,uCAAsCuB,MAAM,CAAC9C,aAAc,OAAMxB,UAAW,EAAzF;AACA,UAAM2F,KAAK,GAAG,4BAASrB,MAAM,CAAC9C,aAAhB,CAAd;AAEA,UAAMoB,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC;AAACtD,MAAAA,aAAa,EAAEmE;AAAhB,KAAlC,CAArB,CAJmC,CAI6C;;AAChF1F,IAAAA,MAAM,CAACmF,KAAP,CAAc,+BAA8BK,IAAI,CAACC,SAAL,CAAe9C,MAAf,CAAuB,EAAnE;;AAEA,QAAIA,MAAJ,EAAY;AACV,YAAMpC,YAAY,CAAC6F,MAAb,CAAoBV,KAApB,CAAN;AACA,aAAO,IAAP;AACD;;AAED,WAAO,IAAP;AACD;;AAED,WAASxE,MAAT,CAAgB;AAACQ,IAAAA,SAAD;AAAYS,IAAAA;AAAZ,GAAhB,EAA6C;AAC3C,UAAMuD,KAAK,GAAG;AAACvD,MAAAA,cAAc,EAAE,4BAASA,cAAT;AAAjB,KAAd;;AACA,QAAI;AACF,UAAIT,SAAS,KAAKE,SAAlB,EAA6B;AAC3B5B,QAAAA,MAAM,CAACmF,KAAP,CAAc,eAAcpF,UAAW,QAAOyF,IAAI,CAACC,SAAL,CAAeC,KAAK,CAACvD,cAArB,CAAqC,EAAnF;AACA,eAAO7B,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC,EAAC,GAAGa;AAAJ,SAAlC,CAAP;AACD;;AAED,YAAMW,MAAM,GAAG;AAAC3E,QAAAA,SAAS,EAAE,4BAASA,SAAT;AAAZ,OAAf;AACA1B,MAAAA,MAAM,CAACmF,KAAP,CAAc,eAAcpF,UAAW,QAAO2F,KAAK,CAACvD,cAAe,MAAKkE,MAAM,CAAC3E,SAAU,EAAzF;AACA,aAAOpB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC,EAAC,GAAGa,KAAJ;AAAW,WAAGW;AAAd,OAAlC,CAAP;AACD,KATD,CASE,OAAOnD,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,WAAS/B,SAAT,CAAmBI,aAAnB,EAAkC;AAChCvB,IAAAA,MAAM,CAAC8C,IAAP,CAAa,iCAAgCvB,aAAc,OAAMxB,UAAW,EAA5E;AACA,UAAM2F,KAAK,GAAG,4BAASnE,aAAT,CAAd;;AACA,QAAI;AACF;AACA,aAAOhB,YAAY,CAAC2F,wBAAb,CAAsCR,KAAtC,CAAP;AACD,KAHD,CAGE,OAAOxC,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAe7B,OAAf,CAAuB;AAACE,IAAAA,aAAD;AAAgBkB,IAAAA,UAAhB;AAA4BC,IAAAA;AAA5B,GAAvB,EAAiE;AAC/D1C,IAAAA,MAAM,CAACsG,OAAP,CAAgB,aAAY7D,UAAU,CAACkC,MAAO,sBAAqBjC,WAAW,CAACiC,MAAO,KAAIpD,aAAc,OAAMxB,UAAW,EAAzH;AACAC,IAAAA,MAAM,CAAC0E,KAAP,CAAc,QAAOjC,UAAU,CAACkC,MAAO,MAAKa,IAAI,CAACC,SAAL,CAAehD,UAAf,CAA2B,iBAAgBC,WAAW,CAACiC,MAAO,KAAIa,IAAI,CAACC,SAAL,CAAe/C,WAAf,CAA4B,EAA1I;AACA,UAAMgD,KAAK,GAAG,4BAASnE,aAAT,CAAd;AACA,UAAMjB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BwG,SAA1B,CAAoC;AACxChF,MAAAA,aAAa,EAAEmE;AADyB,KAApC,EAEH;AACDc,MAAAA,IAAI,EAAE;AACJhE,QAAAA,gBAAgB,EAAE,uBAASR,MAAT;AADd,OADL;AAIDyE,MAAAA,KAAK,EAAE;AACLhE,QAAAA,UAAU,EAAE;AAACiE,UAAAA,KAAK,EAAEjE;AAAR,SADP;AAELC,QAAAA,WAAW,EAAE;AAACgE,UAAAA,KAAK,EAAEhE;AAAR;AAFR;AAJN,KAFG,CAAN;AAWD;;AAED,iBAAepB,YAAf,CAA4B;AAACC,IAAAA,aAAD;AAAgBoF,IAAAA,QAAhB;AAA0BC,IAAAA,YAAY,GAAG;AAAzC,GAA5B,EAAkF;AAChF5G,IAAAA,MAAM,CAACsG,OAAP,CAAgB,kBAAiBK,QAAQ,CAAChC,MAAO,QAAOiC,YAAa,IAAGrF,aAAc,OAAMxB,UAAW,EAAvG;AACAC,IAAAA,MAAM,CAACmF,KAAP,CAAc,aAAYwB,QAAQ,CAAChC,MAAO,MAAKa,IAAI,CAACC,SAAL,CAAekB,QAAf,CAAyB,GAAxE;AACA,UAAMjB,KAAK,GAAG,4BAASnE,aAAT,CAAd;AACA,UAAMsF,iBAAiB,GAAG,4BAASD,YAAT,CAA1B;AACA,UAAMtG,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BwG,SAA1B,CAAoC;AACxChF,MAAAA,aAAa,EAAEmE;AADyB,KAApC,EAEH;AACDc,MAAAA,IAAI,EAAE;AACJhE,QAAAA,gBAAgB,EAAE,uBAASR,MAAT;AADd,OADL;AAIDyE,MAAAA,KAAK,EAAE;AACL,SAACI,iBAAD,GAAqB;AAACH,UAAAA,KAAK,EAAEC;AAAR;AADhB;AAJN,KAFG,CAAN;AAUD;;AAED,WAASvF,QAAT,CAAkB;AAACG,IAAAA,aAAD;AAAgByC,IAAAA,KAAhB;AAAuBC,IAAAA,YAAY,GAAG,EAAtC;AAA0CC,IAAAA,WAAW,GAAG;AAAxD,GAAlB,EAA+E;AAC7E,UAAM4C,WAAW,GAAG7C,YAAY,IAAIC,WAAhB,GAA+B,qBAAoBD,YAAa,qBAAoBC,WAAY,GAAhG,GAAqG,EAAzH;AACAlE,IAAAA,MAAM,CAAC8C,IAAP,CAAa,4BAA2BkB,KAAM,QAAOzC,aAAc,GAAEuF,WAAY,OAAM/G,UAAW,EAAlG;AACA,UAAM2F,KAAK,GAAG,4BAASnE,aAAT,CAAd;AACA,WAAOjB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BgH,gBAA1B,CAA2C;AAChDxF,MAAAA,aAAa,EAAEmE;AADiC,KAA3C,EAEJ;AACDc,MAAAA,IAAI,EAAE;AACJrE,QAAAA,cAAc,EAAE6B,KADZ;AAEJxB,QAAAA,gBAAgB,EAAE,uBAASR,MAAT,EAFd;AAGJiC,QAAAA,YAHI;AAIJC,QAAAA;AAJI;AADL,KAFI,EASJ;AAACK,MAAAA,UAAU,EAAE;AAACC,QAAAA,GAAG,EAAE;AAAN,OAAb;AAAuBwC,MAAAA,iBAAiB,EAAE;AAA1C,KATI,CAAP;AAUD;AACF","sourcesContent":["/**\n*\n* @licstart The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020-2021 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program. If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport {MongoClient, GridFSBucket, MongoDriverError} from 'mongodb';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {Error as ApiError} from '@natlibfi/melinda-commons';\nimport {QUEUE_ITEM_STATE} from './constants';\nimport {logError} from './utils.js';\nimport moment from 'moment';\nimport httpStatus from 'http-status';\nimport sanitize from 'mongo-sanitize';\n\n/* QueueItem:\n{\n \"correlationId\":\"FOO\",\n \"cataloger\":\"xxx0000\",\n \"oCatalogerIn\":\"xxx0000\"\n \"operation\":\"UPDATE\",\n \"contentType\":\"application/json\",\n \"recordLoadParams\": {\n \"pActiveLibrary\": \"XXX00\",\n \"pInputFile\": \"filename.seq\",\n \"pRejectFile\": \"filename.rej\",\n \"pLogFile\": \"filename.syslog\",\n \"pOldNew\": \"NEW\"\n },\n \"queueItemState\":\"PENDING_QUEUING\",\n \"creationTime\":\"2020-01-01T00:00:00.000Z\",\n \"modificationTime\":\"2020-01-01T00:00:01.000Z\",\n \"handledIds\": []\n}\n*/\n\nexport default async function (MONGO_URI, collection) {\n const logger = createLogger();\n\n // Connect to mongo (MONGO)\n const client = await MongoClient.connect(MONGO_URI, {useNewUrlParser: true, useUnifiedTopology: true});\n const db = client.db('rest-api');\n const gridFSBucket = new GridFSBucket(db, {bucketName: collection});\n\n return {createPrio, createBulk, checkAndSetState, query, queryById, remove, readContent, removeContent, getOne, getStream, setState, pushIds, pushMessages};\n\n async function createPrio({correlationId, cataloger, oCatalogerIn, operation, noop = undefined, unique = undefined, prio = true}) {\n const time = moment().toDate();\n const newQueueItem = {\n correlationId,\n cataloger,\n operation,\n oCatalogerIn,\n operationSettings: {unique, noop, prio},\n queueItemState: QUEUE_ITEM_STATE.VALIDATOR.PENDING_VALIDATION,\n creationTime: time,\n modificationTime: time,\n handledIds: [],\n rejectedIds: []\n };\n try {\n const result = await db.collection(collection).insertOne(newQueueItem);\n if (result.acknowledged) {\n logger.info(`New PRIO queue item for ${operation} ${correlationId} has been made in ${collection}`);\n return;\n }\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n } catch (error) {\n logError(error);\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n }\n }\n\n function createBulk({correlationId, cataloger, oCatalogerIn, operation, contentType, recordLoadParams, stream, prio = false}) {\n const time = moment().toDate();\n const newQueueItem = {\n correlationId,\n cataloger,\n oCatalogerIn,\n operation,\n operationSettings: {prio},\n contentType,\n recordLoadParams,\n queueItemState: QUEUE_ITEM_STATE.VALIDATOR.UPLOADING,\n creationTime: time,\n modificationTime: time,\n handledIds: [],\n rejectedIds: []\n };\n try {\n // No await here, promises later\n db.collection(collection).insertOne(newQueueItem);\n logger.info(`New BULK queue item for ${operation} ${correlationId} has been made in ${collection}!`);\n } catch (error) {\n logError(error);\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n }\n\n return new Promise((resolve, reject) => {\n const outputStream = gridFSBucket.openUploadStream(correlationId);\n\n stream\n .on('error', reject)\n .on('data', chunk => outputStream.write(chunk))\n .on('end', () => outputStream.end(undefined, undefined, () => {\n resolve(correlationId);\n }));\n });\n }\n\n // Check state that the queueItem has not waited too long and set state\n async function checkAndSetState({correlationId, state, errorMessage = '', errorStatus = ''}) {\n // checkTimeOut returns true, if queueItem is fresher than 1 minute and it's state is not ABORT/ERROR\n // otherwise it sets queueItem to state ABORT (408, 'Timeout')\n const timeOut = await checkTimeOut(correlationId);\n if (timeOut) {\n return setState({correlationId, state, errorMessage, errorStatus});\n }\n return false;\n }\n\n async function query(params) {\n const result = await db.collection(collection).find(params, {projection: {_id: 0}})\n .toArray();\n logger.debug(`Query result: ${result.length > 0 ? 'Found!' : 'Not found!'}`);\n return result;\n }\n\n async function queryById(correlationId, checkModTime) {\n const result = await db.collection(collection).findOne({correlationId});\n if (checkModTime) {\n const timeOut = await checkTimeOut(correlationId);\n if (timeOut) {\n return queryById(correlationId);\n }\n return result;\n }\n\n return result;\n }\n\n // Check that if the item has waited too long\n // If the last modification time for the queueItem is older than 1 minute\n // set state to ABORT and return false, otherwise return true\n // If the state is already ABORT or ERROR return false\n\n async function checkTimeOut(correlationId) {\n const {modificationTime, queueitemState: oldState} = await db.collection(collection).findOne({correlationId});\n\n if ([QUEUE_ITEM_STATE.ABORT, QUEUE_ITEM_STATE.ERROR].includes(oldState)) {\n logger.silly(`${correlationId} has already state: ${oldState}`);\n return false;\n }\n\n const timeoutTime = moment(modificationTime).add(1, 'm');\n logger.silly(`timeOut @ ${timeoutTime}`);\n\n if (timeoutTime.isBefore()) {\n await setState({correlationId, state: QUEUE_ITEM_STATE.ABORT, errorStatus: httpStatus.REQUEST_TIMEOUT, errorMessage: `Timeout in ${oldState}`});\n return false;\n }\n\n return true;\n }\n\n async function remove(params) {\n logger.silly(`${JSON.stringify(params)}`);\n logger.info(`Removing from Mongo (${collection}) id: ${params.correlationId}`);\n const clean = sanitize(params.correlationId);\n logger.silly(`mongo/remove: clean: ${JSON.stringify(clean)}`);\n\n try {\n //const metadataResult = await getFileMetadata({filename: clean});\n //logger.debug(`mongo/remove: metadataResult: ${JSON.stringify(metadataResult)}`);\n const noContent = await removeContent(params);\n if (noContent) {\n await db.collection(collection).deleteOne({correlationId: clean});\n return true;\n }\n } catch (err) {\n if (err instanceof MongoDriverError) {\n if (err.message.indexOf('File not found for id') !== -1) {\n logger.silly(`mongo/remove: File not found, removing queueItem ${JSON.stringify(clean)} from ${collection}`);\n await db.collection(collection).deleteOne({correlationId: clean});\n return true;\n }\n logger.error(err.message);\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n }\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n }\n }\n\n async function readContent(correlationId) {\n logger.info(`Reading content from mongo for id: ${correlationId} in ${collection}`);\n const clean = sanitize(correlationId);\n const result = await db.collection(collection).findOne({correlationId: clean}); // njsscan-ignore: node_nosqli_injection\n\n if (result) {\n return {\n contentType: result.contentType,\n readStream: gridFSBucket.openDownloadStreamByName(clean)\n };\n }\n\n throw new ApiError(httpStatus.NOT_FOUND);\n }\n\n async function removeContent(params) {\n logger.info(`Removing content from mongo for id: ${params.correlationId} in ${collection}`);\n const clean = sanitize(params.correlationId);\n\n const result = await db.collection(collection).findOne({correlationId: clean}); // njsscan-ignore: node_nosqli_injection\n logger.silly(`mongo/removeContent: result ${JSON.stringify(result)}`);\n\n if (result) {\n await gridFSBucket.delete(clean);\n return true;\n }\n\n return true;\n }\n\n function getOne({operation, queueItemState}) {\n const clean = {queueItemState: sanitize(queueItemState)};\n try {\n if (operation === undefined) {\n logger.silly(`Checking DB ${collection} for ${JSON.stringify(clean.queueItemState)}`);\n return db.collection(collection).findOne({...clean});\n }\n\n const clean2 = {operation: sanitize(operation)};\n logger.silly(`Checking DB ${collection} for ${clean.queueItemState} + ${clean2.operation}`);\n return db.collection(collection).findOne({...clean, ...clean2});\n } catch (error) {\n logError(error);\n }\n }\n\n function getStream(correlationId) {\n logger.info(`Forming stream from mongo for ${correlationId} in ${collection}`);\n const clean = sanitize(correlationId);\n try {\n // Return content stream\n return gridFSBucket.openDownloadStreamByName(clean);\n } catch (error) {\n logError(error);\n }\n }\n\n async function pushIds({correlationId, handledIds, rejectedIds}) {\n logger.verbose(`Push ids (${handledIds.length}) and rejectedIds (${rejectedIds.length}) ${correlationId} to ${collection}`);\n logger.debug(`ids (${handledIds.length}): ${JSON.stringify(handledIds)}, rejectedIds ${rejectedIds.length}: ${JSON.stringify(rejectedIds)}`);\n const clean = sanitize(correlationId);\n await db.collection(collection).updateOne({\n correlationId: clean\n }, {\n $set: {\n modificationTime: moment().toDate()\n },\n $push: {\n handledIds: {$each: handledIds},\n rejectedIds: {$each: rejectedIds}\n }\n });\n }\n\n async function pushMessages({correlationId, messages, messageField = 'messages'}) {\n logger.verbose(`Push messages (${messages.length}) to ${messageField} ${correlationId} to ${collection}`);\n logger.silly(`Messages (${messages.length}): ${JSON.stringify(messages)}}`);\n const clean = sanitize(correlationId);\n const cleanMessageField = sanitize(messageField);\n await db.collection(collection).updateOne({\n correlationId: clean\n }, {\n $set: {\n modificationTime: moment().toDate()\n },\n $push: {\n [cleanMessageField]: {$each: messages}\n }\n });\n }\n\n function setState({correlationId, state, errorMessage = '', errorStatus = ''}) {\n const errorString = errorMessage || errorStatus ? `, Error message: '${errorMessage}', Error status: '${errorStatus}'` : '';\n logger.info(`Setting queue-item state ${state} for ${correlationId}${errorString} to ${collection}`);\n const clean = sanitize(correlationId);\n return db.collection(collection).findOneAndUpdate({\n correlationId: clean\n }, {\n $set: {\n queueItemState: state,\n modificationTime: moment().toDate(),\n errorMessage,\n errorStatus\n }\n }, {projection: {_id: 0}, returnNewDocument: true});\n }\n}\n"],"file":"mongo.js"}
|
|
1
|
+
{"version":3,"sources":["../src/mongo.js"],"names":["MONGO_URI","collection","logger","client","MongoClient","connect","useNewUrlParser","useUnifiedTopology","db","gridFSBucket","GridFSBucket","bucketName","createPrio","createBulk","checkAndSetState","query","queryById","remove","readContent","removeContent","getOne","getStream","setState","pushIds","pushMessages","correlationId","cataloger","oCatalogerIn","operation","noop","undefined","unique","prio","time","toDate","newQueueItem","operationSettings","queueItemState","QUEUE_ITEM_STATE","VALIDATOR","PENDING_VALIDATION","creationTime","modificationTime","handledIds","rejectedIds","result","insertOne","acknowledged","info","ApiError","httpStatus","INTERNAL_SERVER_ERROR","error","contentType","recordLoadParams","stream","UPLOADING","WAITING_FOR_RECORDS","Promise","resolve","reject","outputStream","openUploadStream","on","chunk","write","end","state","errorMessage","errorStatus","timeOut","checkTimeOut","params","find","projection","_id","toArray","debug","length","checkModTime","findOne","queueitemState","oldState","ABORT","ERROR","includes","silly","timeoutTime","add","isBefore","REQUEST_TIMEOUT","JSON","stringify","clean","noContent","deleteOne","err","MongoDriverError","message","indexOf","readStream","openDownloadStreamByName","NOT_FOUND","delete","clean2","verbose","updateOne","$set","$push","$each","messages","messageField","cleanMessageField","errorString","stateInQueueItemStates","Object","values","stateInQueueItemStatesValidator","stateInQueueItemStatesImporter","IMPORTER","BAD_REQUEST","updateValues","keys","forEach","key","findOneAndUpdate","returnNewDocument"],"mappings":";;;;;;;AA4BA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AAnCA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAWA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAEe,wBAAgBA,SAAhB,EAA2BC,UAA3B,EAAuC;AACpD,QAAMC,MAAM,GAAG,0CAAf,CADoD,CAGpD;;AACA,QAAMC,MAAM,GAAG,MAAMC,qBAAYC,OAAZ,CAAoBL,SAApB,EAA+B;AAACM,IAAAA,eAAe,EAAE,IAAlB;AAAwBC,IAAAA,kBAAkB,EAAE;AAA5C,GAA/B,CAArB;AACA,QAAMC,EAAE,GAAGL,MAAM,CAACK,EAAP,CAAU,UAAV,CAAX;AACA,QAAMC,YAAY,GAAG,IAAIC,qBAAJ,CAAiBF,EAAjB,EAAqB;AAACG,IAAAA,UAAU,EAAEV;AAAb,GAArB,CAArB;AAEA,SAAO;AAACW,IAAAA,UAAD;AAAaC,IAAAA,UAAb;AAAyBC,IAAAA,gBAAzB;AAA2CC,IAAAA,KAA3C;AAAkDC,IAAAA,SAAlD;AAA6DC,IAAAA,MAA7D;AAAqEC,IAAAA,WAArE;AAAkFC,IAAAA,aAAlF;AAAiGC,IAAAA,MAAjG;AAAyGC,IAAAA,SAAzG;AAAoHC,IAAAA,QAApH;AAA8HC,IAAAA,OAA9H;AAAuIC,IAAAA;AAAvI,GAAP;;AAEA,iBAAeZ,UAAf,CAA0B;AAACa,IAAAA,aAAD;AAAgBC,IAAAA,SAAhB;AAA2BC,IAAAA,YAA3B;AAAyCC,IAAAA,SAAzC;AAAoDC,IAAAA,IAAI,GAAGC,SAA3D;AAAsEC,IAAAA,MAAM,GAAGD,SAA/E;AAA0FE,IAAAA,IAAI,GAAG;AAAjG,GAA1B,EAAkI;AAChI,UAAMC,IAAI,GAAG,uBAASC,MAAT,EAAb;AACA,UAAMC,YAAY,GAAG;AACnBV,MAAAA,aADmB;AAEnBC,MAAAA,SAFmB;AAGnBE,MAAAA,SAHmB;AAInBD,MAAAA,YAJmB;AAKnBS,MAAAA,iBAAiB,EAAE;AAACL,QAAAA,MAAD;AAASF,QAAAA,IAAT;AAAeG,QAAAA;AAAf,OALA;AAMnBK,MAAAA,cAAc,EAAEC,4BAAiBC,SAAjB,CAA2BC,kBANxB;AAOnBC,MAAAA,YAAY,EAAER,IAPK;AAQnBS,MAAAA,gBAAgB,EAAET,IARC;AASnBU,MAAAA,UAAU,EAAE,EATO;AAUnBC,MAAAA,WAAW,EAAE;AAVM,KAArB;;AAYA,QAAI;AACF,YAAMC,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B6C,SAA1B,CAAoCX,YAApC,CAArB;;AACA,UAAIU,MAAM,CAACE,YAAX,EAAyB;AACvB7C,QAAAA,MAAM,CAAC8C,IAAP,CAAa,2BAA0BpB,SAAU,IAAGH,aAAc,qBAAoBxB,UAAW,EAAjG;AACA;AACD;;AACD,YAAM,IAAIgD,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD,KAPD,CAOE,OAAOC,KAAP,EAAc;AACd,2BAASA,KAAT;AACA,YAAM,IAAIH,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD;AACF;;AAED,WAAStC,UAAT,CAAoB;AAACY,IAAAA,aAAD;AAAgBC,IAAAA,SAAhB;AAA2BC,IAAAA,YAA3B;AAAyCC,IAAAA,SAAzC;AAAoDyB,IAAAA,WAApD;AAAiEC,IAAAA,gBAAjE;AAAmFC,IAAAA,MAAnF;AAA2FvB,IAAAA,IAAI,GAAG;AAAlG,GAApB,EAA8H;AAC5H,UAAMC,IAAI,GAAG,uBAASC,MAAT,EAAb;AACA,UAAMC,YAAY,GAAG;AACnBV,MAAAA,aADmB;AAEnBC,MAAAA,SAFmB;AAGnBC,MAAAA,YAHmB;AAInBC,MAAAA,SAJmB;AAKnBQ,MAAAA,iBAAiB,EAAE;AAACJ,QAAAA;AAAD,OALA;AAMnBqB,MAAAA,WANmB;AAOnBC,MAAAA,gBAPmB;AAQnBjB,MAAAA,cAAc,EAAEkB,MAAM,GAAGjB,4BAAiBC,SAAjB,CAA2BiB,SAA9B,GAA0ClB,4BAAiBC,SAAjB,CAA2BkB,mBARxE;AASnBhB,MAAAA,YAAY,EAAER,IATK;AAUnBS,MAAAA,gBAAgB,EAAET,IAVC;AAWnBU,MAAAA,UAAU,EAAE,EAXO;AAYnBC,MAAAA,WAAW,EAAE;AAZM,KAArB;;AAcA,QAAI;AACF;AACApC,MAAAA,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B6C,SAA1B,CAAoCX,YAApC;AACAjC,MAAAA,MAAM,CAAC8C,IAAP,CAAa,2BAA0BpB,SAAU,IAAGH,aAAc,qBAAoBxB,UAAW,GAAjG;AACD,KAJD,CAIE,OAAOmD,KAAP,EAAc;AACd,2BAASA,KAAT;AACA,YAAM,IAAIH,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD;;AAED,QAAI,CAACI,MAAL,EAAa;AACX,aAAO9B,aAAP;AACD;;AAED,WAAO,IAAIiC,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtC,YAAMC,YAAY,GAAGpD,YAAY,CAACqD,gBAAb,CAA8BrC,aAA9B,CAArB;AAEA8B,MAAAA,MAAM,CACHQ,EADH,CACM,OADN,EACeH,MADf,EAEGG,EAFH,CAEM,MAFN,EAEcC,KAAK,IAAIH,YAAY,CAACI,KAAb,CAAmBD,KAAnB,CAFvB,EAGGD,EAHH,CAGM,KAHN,EAGa,MAAMF,YAAY,CAACK,GAAb,CAAiBpC,SAAjB,EAA4BA,SAA5B,EAAuC,MAAM;AAC5D6B,QAAAA,OAAO,CAAClC,aAAD,CAAP;AACD,OAFgB,CAHnB;AAMD,KATM,CAAP;AAUD,GA5EmD,CA8EpD;;;AACA,iBAAeX,gBAAf,CAAgC;AAACW,IAAAA,aAAD;AAAgB0C,IAAAA,KAAhB;AAAuBC,IAAAA,YAAY,GAAGtC,SAAtC;AAAiDuC,IAAAA,WAAW,GAAGvC;AAA/D,GAAhC,EAA2G;AACzG;AACA;AACA,UAAMwC,OAAO,GAAG,MAAMC,YAAY,CAAC9C,aAAD,CAAlC;;AACA,QAAI6C,OAAJ,EAAa;AACX,aAAOhD,QAAQ,CAAC;AAACG,QAAAA,aAAD;AAAgB0C,QAAAA,KAAhB;AAAuBC,QAAAA,YAAvB;AAAqCC,QAAAA;AAArC,OAAD,CAAf;AACD;;AACD,WAAO,KAAP;AACD;;AAED,iBAAetD,KAAf,CAAqByD,MAArB,EAA6B;AAC3B,UAAM3B,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BwE,IAA1B,CAA+BD,MAA/B,EAAuC;AAACE,MAAAA,UAAU,EAAE;AAACC,QAAAA,GAAG,EAAE;AAAN;AAAb,KAAvC,EAClBC,OADkB,EAArB;AAEA1E,IAAAA,MAAM,CAAC2E,KAAP,CAAc,iBAAgBhC,MAAM,CAACiC,MAAP,GAAgB,CAAhB,GAAoB,QAApB,GAA+B,YAAa,EAA1E;AACA,WAAOjC,MAAP;AACD;;AAED,iBAAe7B,SAAf,CAAyBS,aAAzB,EAAwCsD,YAAxC,EAAsD;AACpD,UAAMlC,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B+E,OAA1B,CAAkC;AAACvD,MAAAA;AAAD,KAAlC,CAArB;;AACA,QAAIsD,YAAJ,EAAkB;AAChB,YAAMT,OAAO,GAAG,MAAMC,YAAY,CAAC9C,aAAD,CAAlC;;AACA,UAAI6C,OAAJ,EAAa;AACX,eAAOtD,SAAS,CAACS,aAAD,CAAhB;AACD;;AACD,aAAOoB,MAAP;AACD;;AAED,WAAOA,MAAP;AACD,GA3GmD,CA6GpD;AACA;AACA;AACA;;;AAEA,iBAAe0B,YAAf,CAA4B9C,aAA5B,EAA2C;AACzC,UAAM;AAACiB,MAAAA,gBAAD;AAAmBuC,MAAAA,cAAc,EAAEC;AAAnC,QAA+C,MAAM1E,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B+E,OAA1B,CAAkC;AAACvD,MAAAA;AAAD,KAAlC,CAA3D;;AAEA,QAAI,CAACa,4BAAiB6C,KAAlB,EAAyB7C,4BAAiB8C,KAA1C,EAAiDC,QAAjD,CAA0DH,QAA1D,CAAJ,EAAyE;AACvEhF,MAAAA,MAAM,CAACoF,KAAP,CAAc,GAAE7D,aAAc,uBAAsByD,QAAS,EAA7D;AACA,aAAO,KAAP;AACD;;AAED,UAAMK,WAAW,GAAG,qBAAO7C,gBAAP,EAAyB8C,GAAzB,CAA6B,CAA7B,EAAgC,GAAhC,CAApB;AACAtF,IAAAA,MAAM,CAACoF,KAAP,CAAc,aAAYC,WAAY,EAAtC;;AAEA,QAAIA,WAAW,CAACE,QAAZ,EAAJ,EAA4B;AAC1B,YAAMnE,QAAQ,CAAC;AAACG,QAAAA,aAAD;AAAgB0C,QAAAA,KAAK,EAAE7B,4BAAiB6C,KAAxC;AAA+Cd,QAAAA,WAAW,EAAEnB,oBAAWwC,eAAvE;AAAwFtB,QAAAA,YAAY,EAAG,cAAac,QAAS;AAA7H,OAAD,CAAd;AACA,aAAO,KAAP;AACD;;AAED,WAAO,IAAP;AACD;;AAED,iBAAejE,MAAf,CAAsBuD,MAAtB,EAA8B;AAC5BtE,IAAAA,MAAM,CAACoF,KAAP,CAAc,GAAEK,IAAI,CAACC,SAAL,CAAepB,MAAf,CAAuB,EAAvC;AACAtE,IAAAA,MAAM,CAAC8C,IAAP,CAAa,wBAAuB/C,UAAW,SAAQuE,MAAM,CAAC/C,aAAc,EAA5E;AACA,UAAMoE,KAAK,GAAG,4BAASrB,MAAM,CAAC/C,aAAhB,CAAd;AACAvB,IAAAA,MAAM,CAACoF,KAAP,CAAc,wBAAuBK,IAAI,CAACC,SAAL,CAAeC,KAAf,CAAsB,EAA3D;;AAEA,QAAI;AACF;AACA;AACA,YAAMC,SAAS,GAAG,MAAM3E,aAAa,CAACqD,MAAD,CAArC;;AACA,UAAIsB,SAAJ,EAAe;AACb,cAAMtF,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8F,SAA1B,CAAoC;AAACtE,UAAAA,aAAa,EAAEoE;AAAhB,SAApC,CAAN;AACA,eAAO,IAAP;AACD;AACF,KARD,CAQE,OAAOG,GAAP,EAAY;AACZ,UAAIA,GAAG,YAAYC,yBAAnB,EAAqC;AACnC,YAAID,GAAG,CAACE,OAAJ,CAAYC,OAAZ,CAAoB,uBAApB,MAAiD,CAAC,CAAtD,EAAyD;AACvDjG,UAAAA,MAAM,CAACoF,KAAP,CAAc,oDAAmDK,IAAI,CAACC,SAAL,CAAeC,KAAf,CAAsB,SAAQ5F,UAAW,EAA1G;AACA,gBAAMO,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8F,SAA1B,CAAoC;AAACtE,YAAAA,aAAa,EAAEoE;AAAhB,WAApC,CAAN;AACA,iBAAO,IAAP;AACD;;AACD3F,QAAAA,MAAM,CAACkD,KAAP,CAAa4C,GAAG,CAACE,OAAjB;AACA,cAAM,IAAIjD,qBAAJ,CAAaC,oBAAWC,qBAAxB,EAA+C6C,GAAG,CAACE,OAAnD,CAAN;AACD;;AACD,YAAM,IAAIjD,qBAAJ,CAAaC,oBAAWC,qBAAxB,EAA+C6C,GAAG,CAACE,OAAnD,CAAN;AACD;AACF;;AAED,iBAAehF,WAAf,CAA2BO,aAA3B,EAA0C;AACxCvB,IAAAA,MAAM,CAAC8C,IAAP,CAAa,sCAAqCvB,aAAc,OAAMxB,UAAW,EAAjF;AACA,UAAM4F,KAAK,GAAG,4BAASpE,aAAT,CAAd;AACA,UAAMoB,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B+E,OAA1B,CAAkC;AAACvD,MAAAA,aAAa,EAAEoE;AAAhB,KAAlC,CAArB,CAHwC,CAGwC;;AAEhF,QAAIhD,MAAJ,EAAY;AACV,aAAO;AACLQ,QAAAA,WAAW,EAAER,MAAM,CAACQ,WADf;AAEL+C,QAAAA,UAAU,EAAE3F,YAAY,CAAC4F,wBAAb,CAAsCR,KAAtC;AAFP,OAAP;AAID;;AAED,UAAM,IAAI5C,qBAAJ,CAAaC,oBAAWoD,SAAxB,CAAN;AACD;;AAED,iBAAenF,aAAf,CAA6BqD,MAA7B,EAAqC;AACnCtE,IAAAA,MAAM,CAAC8C,IAAP,CAAa,uCAAsCwB,MAAM,CAAC/C,aAAc,OAAMxB,UAAW,EAAzF;AACA,UAAM4F,KAAK,GAAG,4BAASrB,MAAM,CAAC/C,aAAhB,CAAd;AAEA,UAAMoB,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B+E,OAA1B,CAAkC;AAACvD,MAAAA,aAAa,EAAEoE;AAAhB,KAAlC,CAArB,CAJmC,CAI6C;;AAChF3F,IAAAA,MAAM,CAACoF,KAAP,CAAc,+BAA8BK,IAAI,CAACC,SAAL,CAAe/C,MAAf,CAAuB,EAAnE;;AAEA,QAAIA,MAAJ,EAAY;AACV,YAAMpC,YAAY,CAAC8F,MAAb,CAAoBV,KAApB,CAAN;AACA,aAAO,IAAP;AACD;;AAED,WAAO,IAAP;AACD;;AAED,WAASzE,MAAT,CAAgB;AAACQ,IAAAA,SAAD;AAAYS,IAAAA;AAAZ,GAAhB,EAA6C;AAC3C,UAAMwD,KAAK,GAAG;AAACxD,MAAAA,cAAc,EAAE,4BAASA,cAAT;AAAjB,KAAd;;AACA,QAAI;AACF,UAAIT,SAAS,KAAKE,SAAlB,EAA6B;AAC3B5B,QAAAA,MAAM,CAACoF,KAAP,CAAc,eAAcrF,UAAW,QAAO0F,IAAI,CAACC,SAAL,CAAeC,KAAK,CAACxD,cAArB,CAAqC,EAAnF;AACA,eAAO7B,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B+E,OAA1B,CAAkC,EAAC,GAAGa;AAAJ,SAAlC,CAAP;AACD;;AAED,YAAMW,MAAM,GAAG;AAAC5E,QAAAA,SAAS,EAAE,4BAASA,SAAT;AAAZ,OAAf;AACA1B,MAAAA,MAAM,CAACoF,KAAP,CAAc,eAAcrF,UAAW,QAAO4F,KAAK,CAACxD,cAAe,MAAKmE,MAAM,CAAC5E,SAAU,EAAzF;AACA,aAAOpB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B+E,OAA1B,CAAkC,EAAC,GAAGa,KAAJ;AAAW,WAAGW;AAAd,OAAlC,CAAP;AACD,KATD,CASE,OAAOpD,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,WAAS/B,SAAT,CAAmBI,aAAnB,EAAkC;AAChCvB,IAAAA,MAAM,CAAC8C,IAAP,CAAa,iCAAgCvB,aAAc,OAAMxB,UAAW,EAA5E;AACA,UAAM4F,KAAK,GAAG,4BAASpE,aAAT,CAAd;;AACA,QAAI;AACF;AACA,aAAOhB,YAAY,CAAC4F,wBAAb,CAAsCR,KAAtC,CAAP;AACD,KAHD,CAGE,OAAOzC,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAe7B,OAAf,CAAuB;AAACE,IAAAA,aAAD;AAAgBkB,IAAAA,UAAhB;AAA4BC,IAAAA;AAA5B,GAAvB,EAAiE;AAC/D1C,IAAAA,MAAM,CAACuG,OAAP,CAAgB,aAAY9D,UAAU,CAACmC,MAAO,sBAAqBlC,WAAW,CAACkC,MAAO,KAAIrD,aAAc,OAAMxB,UAAW,EAAzH;AACAC,IAAAA,MAAM,CAAC2E,KAAP,CAAc,QAAOlC,UAAU,CAACmC,MAAO,MAAKa,IAAI,CAACC,SAAL,CAAejD,UAAf,CAA2B,iBAAgBC,WAAW,CAACkC,MAAO,KAAIa,IAAI,CAACC,SAAL,CAAehD,WAAf,CAA4B,EAA1I;AACA,UAAMiD,KAAK,GAAG,4BAASpE,aAAT,CAAd;AACA,UAAMjB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0ByG,SAA1B,CAAoC;AACxCjF,MAAAA,aAAa,EAAEoE;AADyB,KAApC,EAEH;AACDc,MAAAA,IAAI,EAAE;AACJjE,QAAAA,gBAAgB,EAAE,uBAASR,MAAT;AADd,OADL;AAID0E,MAAAA,KAAK,EAAE;AACLjE,QAAAA,UAAU,EAAE;AAACkE,UAAAA,KAAK,EAAElE;AAAR,SADP;AAELC,QAAAA,WAAW,EAAE;AAACiE,UAAAA,KAAK,EAAEjE;AAAR;AAFR;AAJN,KAFG,CAAN;AAWD;;AAED,iBAAepB,YAAf,CAA4B;AAACC,IAAAA,aAAD;AAAgBqF,IAAAA,QAAhB;AAA0BC,IAAAA,YAAY,GAAG;AAAzC,GAA5B,EAAkF;AAChF7G,IAAAA,MAAM,CAACuG,OAAP,CAAgB,kBAAiBK,QAAQ,CAAChC,MAAO,QAAOiC,YAAa,IAAGtF,aAAc,OAAMxB,UAAW,EAAvG;AACAC,IAAAA,MAAM,CAACoF,KAAP,CAAc,aAAYwB,QAAQ,CAAChC,MAAO,MAAKa,IAAI,CAACC,SAAL,CAAekB,QAAf,CAAyB,GAAxE;AACA,UAAMjB,KAAK,GAAG,4BAASpE,aAAT,CAAd;AACA,UAAMuF,iBAAiB,GAAG,4BAASD,YAAT,CAA1B;AACA,UAAMvG,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0ByG,SAA1B,CAAoC;AACxCjF,MAAAA,aAAa,EAAEoE;AADyB,KAApC,EAEH;AACDc,MAAAA,IAAI,EAAE;AACJjE,QAAAA,gBAAgB,EAAE,uBAASR,MAAT;AADd,OADL;AAID0E,MAAAA,KAAK,EAAE;AACL,SAACI,iBAAD,GAAqB;AAACH,UAAAA,KAAK,EAAEC;AAAR;AADhB;AAJN,KAFG,CAAN;AAUD;;AAED,WAASxF,QAAT,CAAkB;AAACG,IAAAA,aAAD;AAAgB0C,IAAAA,KAAhB;AAAuBC,IAAAA,YAAY,GAAGtC,SAAtC;AAAiDuC,IAAAA,WAAW,GAAGvC;AAA/D,GAAlB,EAA6F;AAC3F,UAAMmF,WAAW,GAAG7C,YAAY,IAAIC,WAAhB,GAA+B,qBAAoBD,YAAY,IAAI,EAAG,qBAAoBC,WAAW,IAAI,EAAG,GAA5G,GAAiH,EAArI;AACAnE,IAAAA,MAAM,CAAC8C,IAAP,CAAa,4BAA2BmB,KAAM,QAAO1C,aAAc,GAAEwF,WAAY,OAAMhH,UAAW,EAAlG;AAEA,UAAMiH,sBAAsB,GAAGC,MAAM,CAACC,MAAP,CAAc9E,2BAAd,EAAgC6D,OAAhC,CAAwChC,KAAxC,IAAiD,CAAC,CAAjF;AACA,UAAMkD,+BAA+B,GAAGF,MAAM,CAACC,MAAP,CAAc9E,4BAAiBC,SAA/B,EAA0C4D,OAA1C,CAAkDhC,KAAlD,IAA2D,CAAC,CAApG;AACA,UAAMmD,8BAA8B,GAAGH,MAAM,CAACC,MAAP,CAAc9E,4BAAiBiF,QAA/B,EAAyCpB,OAAzC,CAAiDhC,KAAjD,IAA0D,CAAC,CAAlG;;AACA,QAAI,CAAC+C,sBAAD,IAA2B,CAACG,+BAA5B,IAA+D,CAACC,8BAApE,EAAoG;AAClG,YAAM,IAAIrE,qBAAJ,CAAaC,oBAAWsE,WAAxB,EAAqC,6BAArC,CAAN;AACD;;AAED,UAAM3B,KAAK,GAAG,4BAASpE,aAAT,CAAd;AACA,UAAMgG,YAAY,GAAG;AACnBpF,MAAAA,cAAc,EAAE8B,KADG;AAEnBzB,MAAAA,gBAAgB,EAAE,uBAASR,MAAT,EAFC;AAGnBkC,MAAAA,YAHmB;AAInBC,MAAAA;AAJmB,KAArB,CAZ2F,CAmB3F;AACA;;AACA8C,IAAAA,MAAM,CAACO,IAAP,CAAYD,YAAZ,EAA0BE,OAA1B,CAAkCC,GAAG,IAAIH,YAAY,CAACG,GAAD,CAAZ,KAAsB9F,SAAtB,IAAmC,OAAO2F,YAAY,CAACG,GAAD,CAA/F;AAEA,WAAOpH,EAAE,CAACP,UAAH,CAAcA,UAAd,EACJ4H,gBADI,CAEH;AAACpG,MAAAA,aAAa,EAAEoE;AAAhB,KAFG,EAGH;AAACc,MAAAA,IAAI,EAAEc;AAAP,KAHG,EAIH;AAAC/C,MAAAA,UAAU,EAAE;AAACC,QAAAA,GAAG,EAAE;AAAN,OAAb;AAAuBmD,MAAAA,iBAAiB,EAAE;AAA1C,KAJG,CAAP;AAMD;AACF","sourcesContent":["/**\n*\n* @licstart The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020-2021 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program. If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport {MongoClient, GridFSBucket, MongoDriverError} from 'mongodb';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {Error as ApiError} from '@natlibfi/melinda-commons';\nimport {QUEUE_ITEM_STATE} from './constants';\nimport {logError} from './utils.js';\nimport moment from 'moment';\nimport httpStatus from 'http-status';\nimport sanitize from 'mongo-sanitize';\n\n/* QueueItem:\n{\n \"correlationId\":\"FOO\",\n \"cataloger\":\"xxx0000\",\n \"oCatalogerIn\":\"xxx0000\"\n \"operation\":\"UPDATE\",\n \"contentType\":\"application/json\",\n \"recordLoadParams\": {\n \"pActiveLibrary\": \"XXX00\",\n \"pInputFile\": \"filename.seq\",\n \"pRejectFile\": \"filename.rej\",\n \"pLogFile\": \"filename.syslog\",\n \"pOldNew\": \"NEW\"\n },\n \"queueItemState\":\"PENDING_QUEUING\",\n \"creationTime\":\"2020-01-01T00:00:00.000Z\",\n \"modificationTime\":\"2020-01-01T00:00:01.000Z\",\n \"handledIds\": []\n}\n*/\n\nexport default async function (MONGO_URI, collection) {\n const logger = createLogger();\n\n // Connect to mongo (MONGO)\n const client = await MongoClient.connect(MONGO_URI, {useNewUrlParser: true, useUnifiedTopology: true});\n const db = client.db('rest-api');\n const gridFSBucket = new GridFSBucket(db, {bucketName: collection});\n\n return {createPrio, createBulk, checkAndSetState, query, queryById, remove, readContent, removeContent, getOne, getStream, setState, pushIds, pushMessages};\n\n async function createPrio({correlationId, cataloger, oCatalogerIn, operation, noop = undefined, unique = undefined, prio = true}) {\n const time = moment().toDate();\n const newQueueItem = {\n correlationId,\n cataloger,\n operation,\n oCatalogerIn,\n operationSettings: {unique, noop, prio},\n queueItemState: QUEUE_ITEM_STATE.VALIDATOR.PENDING_VALIDATION,\n creationTime: time,\n modificationTime: time,\n handledIds: [],\n rejectedIds: []\n };\n try {\n const result = await db.collection(collection).insertOne(newQueueItem);\n if (result.acknowledged) {\n logger.info(`New PRIO queue item for ${operation} ${correlationId} has been made in ${collection}`);\n return;\n }\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n } catch (error) {\n logError(error);\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n }\n }\n\n function createBulk({correlationId, cataloger, oCatalogerIn, operation, contentType, recordLoadParams, stream, prio = false}) {\n const time = moment().toDate();\n const newQueueItem = {\n correlationId,\n cataloger,\n oCatalogerIn,\n operation,\n operationSettings: {prio},\n contentType,\n recordLoadParams,\n queueItemState: stream ? QUEUE_ITEM_STATE.VALIDATOR.UPLOADING : QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS,\n creationTime: time,\n modificationTime: time,\n handledIds: [],\n rejectedIds: []\n };\n try {\n // No await here, promises later\n db.collection(collection).insertOne(newQueueItem);\n logger.info(`New BULK queue item for ${operation} ${correlationId} has been made in ${collection}!`);\n } catch (error) {\n logError(error);\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n }\n\n if (!stream) {\n return correlationId;\n }\n\n return new Promise((resolve, reject) => {\n const outputStream = gridFSBucket.openUploadStream(correlationId);\n\n stream\n .on('error', reject)\n .on('data', chunk => outputStream.write(chunk))\n .on('end', () => outputStream.end(undefined, undefined, () => {\n resolve(correlationId);\n }));\n });\n }\n\n // Check state that the queueItem has not waited too long and set state\n async function checkAndSetState({correlationId, state, errorMessage = undefined, errorStatus = undefined}) {\n // checkTimeOut returns true, if queueItem is fresher than 1 minute and it's state is not ABORT/ERROR\n // otherwise it sets queueItem to state ABORT (408, 'Timeout')\n const timeOut = await checkTimeOut(correlationId);\n if (timeOut) {\n return setState({correlationId, state, errorMessage, errorStatus});\n }\n return false;\n }\n\n async function query(params) {\n const result = await db.collection(collection).find(params, {projection: {_id: 0}})\n .toArray();\n logger.debug(`Query result: ${result.length > 0 ? 'Found!' : 'Not found!'}`);\n return result;\n }\n\n async function queryById(correlationId, checkModTime) {\n const result = await db.collection(collection).findOne({correlationId});\n if (checkModTime) {\n const timeOut = await checkTimeOut(correlationId);\n if (timeOut) {\n return queryById(correlationId);\n }\n return result;\n }\n\n return result;\n }\n\n // Check that if the item has waited too long\n // If the last modification time for the queueItem is older than 1 minute\n // set state to ABORT and return false, otherwise return true\n // If the state is already ABORT or ERROR return false\n\n async function checkTimeOut(correlationId) {\n const {modificationTime, queueitemState: oldState} = await db.collection(collection).findOne({correlationId});\n\n if ([QUEUE_ITEM_STATE.ABORT, QUEUE_ITEM_STATE.ERROR].includes(oldState)) {\n logger.silly(`${correlationId} has already state: ${oldState}`);\n return false;\n }\n\n const timeoutTime = moment(modificationTime).add(1, 'm');\n logger.silly(`timeOut @ ${timeoutTime}`);\n\n if (timeoutTime.isBefore()) {\n await setState({correlationId, state: QUEUE_ITEM_STATE.ABORT, errorStatus: httpStatus.REQUEST_TIMEOUT, errorMessage: `Timeout in ${oldState}`});\n return false;\n }\n\n return true;\n }\n\n async function remove(params) {\n logger.silly(`${JSON.stringify(params)}`);\n logger.info(`Removing from Mongo (${collection}) id: ${params.correlationId}`);\n const clean = sanitize(params.correlationId);\n logger.silly(`mongo/remove: clean: ${JSON.stringify(clean)}`);\n\n try {\n //const metadataResult = await getFileMetadata({filename: clean});\n //logger.debug(`mongo/remove: metadataResult: ${JSON.stringify(metadataResult)}`);\n const noContent = await removeContent(params);\n if (noContent) {\n await db.collection(collection).deleteOne({correlationId: clean});\n return true;\n }\n } catch (err) {\n if (err instanceof MongoDriverError) {\n if (err.message.indexOf('File not found for id') !== -1) {\n logger.silly(`mongo/remove: File not found, removing queueItem ${JSON.stringify(clean)} from ${collection}`);\n await db.collection(collection).deleteOne({correlationId: clean});\n return true;\n }\n logger.error(err.message);\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n }\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n }\n }\n\n async function readContent(correlationId) {\n logger.info(`Reading content from mongo for id: ${correlationId} in ${collection}`);\n const clean = sanitize(correlationId);\n const result = await db.collection(collection).findOne({correlationId: clean}); // njsscan-ignore: node_nosqli_injection\n\n if (result) {\n return {\n contentType: result.contentType,\n readStream: gridFSBucket.openDownloadStreamByName(clean)\n };\n }\n\n throw new ApiError(httpStatus.NOT_FOUND);\n }\n\n async function removeContent(params) {\n logger.info(`Removing content from mongo for id: ${params.correlationId} in ${collection}`);\n const clean = sanitize(params.correlationId);\n\n const result = await db.collection(collection).findOne({correlationId: clean}); // njsscan-ignore: node_nosqli_injection\n logger.silly(`mongo/removeContent: result ${JSON.stringify(result)}`);\n\n if (result) {\n await gridFSBucket.delete(clean);\n return true;\n }\n\n return true;\n }\n\n function getOne({operation, queueItemState}) {\n const clean = {queueItemState: sanitize(queueItemState)};\n try {\n if (operation === undefined) {\n logger.silly(`Checking DB ${collection} for ${JSON.stringify(clean.queueItemState)}`);\n return db.collection(collection).findOne({...clean});\n }\n\n const clean2 = {operation: sanitize(operation)};\n logger.silly(`Checking DB ${collection} for ${clean.queueItemState} + ${clean2.operation}`);\n return db.collection(collection).findOne({...clean, ...clean2});\n } catch (error) {\n logError(error);\n }\n }\n\n function getStream(correlationId) {\n logger.info(`Forming stream from mongo for ${correlationId} in ${collection}`);\n const clean = sanitize(correlationId);\n try {\n // Return content stream\n return gridFSBucket.openDownloadStreamByName(clean);\n } catch (error) {\n logError(error);\n }\n }\n\n async function pushIds({correlationId, handledIds, rejectedIds}) {\n logger.verbose(`Push ids (${handledIds.length}) and rejectedIds (${rejectedIds.length}) ${correlationId} to ${collection}`);\n logger.debug(`ids (${handledIds.length}): ${JSON.stringify(handledIds)}, rejectedIds ${rejectedIds.length}: ${JSON.stringify(rejectedIds)}`);\n const clean = sanitize(correlationId);\n await db.collection(collection).updateOne({\n correlationId: clean\n }, {\n $set: {\n modificationTime: moment().toDate()\n },\n $push: {\n handledIds: {$each: handledIds},\n rejectedIds: {$each: rejectedIds}\n }\n });\n }\n\n async function pushMessages({correlationId, messages, messageField = 'messages'}) {\n logger.verbose(`Push messages (${messages.length}) to ${messageField} ${correlationId} to ${collection}`);\n logger.silly(`Messages (${messages.length}): ${JSON.stringify(messages)}}`);\n const clean = sanitize(correlationId);\n const cleanMessageField = sanitize(messageField);\n await db.collection(collection).updateOne({\n correlationId: clean\n }, {\n $set: {\n modificationTime: moment().toDate()\n },\n $push: {\n [cleanMessageField]: {$each: messages}\n }\n });\n }\n\n function setState({correlationId, state, errorMessage = undefined, errorStatus = undefined}) {\n const errorString = errorMessage || errorStatus ? `, Error message: '${errorMessage || ''}', Error status: '${errorStatus || ''}'` : '';\n logger.info(`Setting queue-item state ${state} for ${correlationId}${errorString} to ${collection}`);\n\n const stateInQueueItemStates = Object.values(QUEUE_ITEM_STATE).indexOf(state) > -1;\n const stateInQueueItemStatesValidator = Object.values(QUEUE_ITEM_STATE.VALIDATOR).indexOf(state) > -1;\n const stateInQueueItemStatesImporter = Object.values(QUEUE_ITEM_STATE.IMPORTER).indexOf(state) > -1;\n if (!stateInQueueItemStates && !stateInQueueItemStatesValidator && !stateInQueueItemStatesImporter) {\n throw new ApiError(httpStatus.BAD_REQUEST, 'Trying to set invalid state');\n }\n\n const clean = sanitize(correlationId);\n const updateValues = {\n queueItemState: state,\n modificationTime: moment().toDate(),\n errorMessage,\n errorStatus\n };\n\n // Do not update value that are undefined\n // eslint-disable-next-line functional/immutable-data\n Object.keys(updateValues).forEach(key => updateValues[key] === undefined && delete updateValues[key]);\n\n return db.collection(collection)\n .findOneAndUpdate(\n {correlationId: clean},\n {$set: updateValues},\n {projection: {_id: 0}, returnNewDocument: true}\n );\n }\n}\n"],"file":"mongo.js"}
|
package/package.json
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
"url": "git@github.com:natlibfi/melinda-rest-api-commons.git"
|
|
15
15
|
},
|
|
16
16
|
"license": "AGPL-3.0+",
|
|
17
|
-
"version": "2.1
|
|
17
|
+
"version": "2.2.0-alpha.1",
|
|
18
18
|
"main": "./dist/index.js",
|
|
19
19
|
"engines": {
|
|
20
20
|
"node": ">=14"
|
|
@@ -34,12 +34,12 @@
|
|
|
34
34
|
},
|
|
35
35
|
"dependencies": {
|
|
36
36
|
"@natlibfi/fixura": "^2.2.1",
|
|
37
|
-
"@natlibfi/marc-record": "^
|
|
38
|
-
"@natlibfi/marc-record-serializers": "^8.
|
|
39
|
-
"@natlibfi/marc-record-validate": "^
|
|
40
|
-
"@natlibfi/marc-record-validators-melinda": "^
|
|
37
|
+
"@natlibfi/marc-record": "^7.0.0",
|
|
38
|
+
"@natlibfi/marc-record-serializers": "^8.1.0",
|
|
39
|
+
"@natlibfi/marc-record-validate": "^7.0.1",
|
|
40
|
+
"@natlibfi/marc-record-validators-melinda": "^9.0.4",
|
|
41
41
|
"@natlibfi/melinda-backend-commons": "^2.0.5",
|
|
42
|
-
"@natlibfi/melinda-commons": "^12.0.
|
|
42
|
+
"@natlibfi/melinda-commons": "^12.0.3",
|
|
43
43
|
"amqplib": ">=0.8.0 <1.0.0",
|
|
44
44
|
"http-status": "^1.5.0",
|
|
45
45
|
"moment": "^2.29.1",
|
|
@@ -51,16 +51,16 @@
|
|
|
51
51
|
"@babel/core": "^7.15.8",
|
|
52
52
|
"@babel/plugin-syntax-dynamic-import": "^7.8.3",
|
|
53
53
|
"@babel/preset-env": "^7.15.8",
|
|
54
|
-
"@natlibfi/eslint-config-melinda-backend": "^
|
|
54
|
+
"@natlibfi/eslint-config-melinda-backend": "^2.0.0",
|
|
55
55
|
"@babel/eslint-parser": "^7.15.8",
|
|
56
56
|
"babel-plugin-dynamic-import-node": "^2.3.3",
|
|
57
57
|
"babel-plugin-istanbul": "^6.1.1",
|
|
58
58
|
"babel-plugin-rewire": "^1.2.0",
|
|
59
59
|
"chai": "^4.3.4",
|
|
60
60
|
"cross-env": "^7.0.3",
|
|
61
|
-
"eslint": "^
|
|
61
|
+
"eslint": "^8.10.0",
|
|
62
62
|
"eslint-plugin-import": "^2.25.2",
|
|
63
|
-
"mocha": "^9.1
|
|
63
|
+
"mocha": "^9.2.1",
|
|
64
64
|
"nodemon": "^2.0.14",
|
|
65
65
|
"nyc": "^15.1.0"
|
|
66
66
|
},
|