@natlibfi/melinda-rest-api-commons 2.0.0-alpha.1 → 2.0.0-alpha.13

package/dist/amqp.js

@@ -62,7 +62,6 @@ async function _default(AMQP_URL) {
     consumeRawChunk,
     consumeOne,
     consumeRaw,
-    ackNReplyMessages,
     ackMessages,
     nackMessages,
     sendToQueue,
@@ -71,7 +70,7 @@ async function _default(AMQP_URL) {
   };
 
   async function checkQueue(queue, style = 'basic', purge = false, toRecord = true) {
-    logger.debug(`checkQueue: ${queue}, Style: ${style}`);
+    logger.verbose(`checkQueue: ${queue}, Style: ${style}: Purge: ${purge}, toRecord: ${toRecord}`);
 
     try {
       const channelInfo = await channel.assertQueue(queue, {
@@ -80,7 +79,7 @@ async function _default(AMQP_URL) {
 
       if (purge) {
         await purgeQueue(purge);
-        logger.debug(`Queue ${queue} has purged ${channelInfo.messageCount} records`);
+        logger.debug(`Queue ${queue} has purged ${channelInfo.messageCount} messages`);
         return checkQueue(queue, style);
       }
 
@@ -89,28 +88,33 @@ async function _default(AMQP_URL) {
         return false;
       }
 
-      logger.debug(`Queue ${queue} has ${channelInfo.messageCount} records`);
+      logger.silly(`Queue ${queue} has ${channelInfo.messageCount} messages`);
 
       if (style === 'messages') {
         return channelInfo.messageCount;
-      }
+      } // Note: returns a message + a record
+
 
       if (style === 'one') {
         return consumeOne(queue);
-      }
+      } // Note: returns just a message
+
 
       if (style === 'raw') {
         return consumeRaw(queue);
-      }
+      } // Note: returns a chunk of messages + records
+
 
       if (style === 'rawChunk') {
         return consumeRawChunk(queue);
-      }
+      } // Note: returns a chunk of messages or a chunk of messages + records (depending on toRecord)
+
 
       if (/^.{8}-.{4}-.{4}-.{4}-.{12}$/u.test(style)) {
-        logger.debug(`checkQueue: regexp match ${style}`);
+        logger.silly(`checkQueue: regexp match ${style}`);
         return consumeByCorrelationId(queue, style, toRecord);
-      }
+      } // Note: returns a chunk of messages + records
+
 
       if (style === 'basic') {
         return consumeChunk(queue);
@@ -135,20 +139,30 @@ async function _default(AMQP_URL) {
     try {
       await channel.assertQueue(queue, {
         durable: true
-      });
+      }); // getData: next chunk (100) messages
+
       const queMessages = await getData(queue);
       const headers = getHeaderInfo(queMessages[0]);
-      logger.debug(`Filtering messages by ${JSON.stringify(headers)}`); // Check that cataloger match! headers
+      logger.debug(`Filtering messages by cataloger in ${JSON.stringify(headers)}`);
+      let filterIn = 0; // eslint-disable-line functional/no-let
+
+      let filterOut = 0; // eslint-disable-line functional/no-let
+      // Check that cataloger match! headers
 
       const messages = queMessages.filter(message => {
         if (message.properties.headers.cataloger === headers.cataloger) {
+          filterIn++; // eslint-disable-line no-plusplus
+
           return true;
         } // Nack unwanted ones
 
 
         channel.nack(message, false, true);
+        filterOut++; // eslint-disable-line no-plusplus
+
         return false;
       });
+      logger.debug(`Filtering result: valid: ${filterIn} non-valid: ${filterOut}`);
       const records = await messagesToRecords(messages);
       return {
         headers,
@@ -166,19 +180,29 @@ async function _default(AMQP_URL) {
     try {
       await channel.assertQueue(queue, {
         durable: true
-      });
+      }); // get next chunk (100) messages
+
       const queMessages = await getData(queue);
-      logger.debug(`Filtering messages by ${correlationId}`); // Check that cataloger match! headers
+      logger.debug(`Filtering messages by ${correlationId}`);
+      let filterIn = 0; // eslint-disable-line functional/no-let
+
+      let filterOut = 0; // eslint-disable-line functional/no-let
+      // Check that correlationId matches query
 
       const messages = queMessages.filter(message => {
         if (message.properties.correlationId === correlationId) {
+          filterIn++; // eslint-disable-line no-plusplus
+
           return true;
         } // Nack unwanted ones
 
 
         channel.nack(message, false, true);
+        filterOut++; // eslint-disable-line no-plusplus
+
         return false;
       });
+      logger.debug(`Filtering result: valid: ${filterIn} non-valid: ${filterOut}`);
       const headers = getHeaderInfo(messages[0]);
 
       if (toRecord) {
@@ -205,20 +229,31 @@ async function _default(AMQP_URL) {
     try {
       await channel.assertQueue(queue, {
         durable: true
-      });
+      }); // Get next chunk (100) messages
+
       const queMessages = await getData(queue);
       const headers = getHeaderInfo(queMessages[0]);
-      logger.debug(`Filtering messages by ${JSON.stringify(headers)} and timeout`); // Check that cataloger match! headers
+      logger.debug(`Filtering messages by ${JSON.stringify(headers)}`);
+      let filterIn = 0; // eslint-disable-line functional/no-let
+
+      let filterOut = 0; // eslint-disable-line functional/no-let
+      // Check that cataloger match! headers
 
       const messages = queMessages.filter(message => {
         if (message.properties.headers.cataloger === headers.cataloger) {
+          filterIn++; // eslint-disable-line no-plusplus
+
           return true;
         } // Nack unwanted ones
 
 
         channel.nack(message, false, true);
+        filterOut++; // eslint-disable-line no-plusplus
+
         return false;
       });
+      logger.debug(`Filtering result: valid: ${filterIn} non-valid: ${filterOut}`); // return raw (do not convert messages to records)
+
       return {
         headers,
         messages
@@ -229,7 +264,7 @@ async function _default(AMQP_URL) {
   }
 
   async function consumeOne(queue) {
-    logger.verbose(`Prepared to consume one from queue: ${queue}`);
+    logger.verbose(`Prepared to consumeOne from queue: ${queue}`);
 
     try {
       await channel.assertQueue(queue, {
@@ -255,7 +290,7 @@ async function _default(AMQP_URL) {
   }
 
   async function consumeRaw(queue) {
-    logger.verbose(`Prepared to consume raw from queue: ${queue}`);
+    logger.verbose(`Prepared to consumeRaw from queue: ${queue}`);
 
     try {
       await channel.assertQueue(queue, {
@@ -266,56 +301,9 @@ async function _default(AMQP_URL) {
     } catch (error) {
       (0, _utils.logError)(error);
     }
-  } // ACK records
-
-
-  async function ackNReplyMessages({
-    status,
-    messages,
-    payloads
-  }) {
-    logger.verbose('Ack and reply messages!');
-    logger.debug(`Ack and reply messages. status: ${JSON.stringify(status)} messages: ${messages} payloads: ${JSON.stringify(payloads)}`);
-    logger.debug(`Handling ${messages.length} messages`);
-    await messages.forEach((message, index) => {
-      const headers = getHeaderInfo(message);
-      logger.debug(`Message: ${message}, index: ${index}`);
-      logger.debug(`Headers: ${JSON.stringify(headers)}`); // payloads ids are parsed both form old payloads including just array of ids and new payloads including both array of ids and array of rejected ids
-
-      const ids = payloads.ids || payloads;
-      const rejectedIds = payloads.rejectedIds || [];
-      logger.debug(`ids: ${JSON.stringify(ids)}, rejectedIds: ${JSON.stringify(rejectedIds)}`);
-
-      if (ids.length < 1 && rejectedIds.length > 0) {
-        logger.debug(`Got 0 valid ids and rejected ${rejectedIds}`);
-        sendToQueue({
-          queue: message.properties.correlationId,
-          correlationId: message.properties.correlationId,
-          headers,
-          data: {
-            status: _httpStatus.default.UNPROCESSABLE_ENTITY,
-            payload: rejectedIds
-          }
-        });
-        return channel.ack(message);
-      } // Reply consumer gets: {"data":{"status":"UPDATED","payload":"0"}}
-
-
-      sendToQueue({
-        queue: message.properties.correlationId,
-        correlationId: message.properties.correlationId,
-        headers,
-        data: {
-          status,
-          payload: ids[index]
-        }
-      });
-      channel.ack(message);
-    });
   }
 
   function ackMessages(messages) {
-    logger.verbose(`Ack messages! (${messages.length})`);
     messages.forEach(message => {
       logger.silly(`Ack message ${message.properties.correlationId}`);
       channel.ack(message);
@@ -323,7 +311,6 @@ async function _default(AMQP_URL) {
   }
 
   function nackMessages(messages) {
-    logger.verbose(`Nack messages! (${messages.length})`);
     messages.forEach(message => {
       logger.silly(`Nack message ${message.properties.correlationId}`);
       channel.nack(message);
@@ -336,11 +323,13 @@ async function _default(AMQP_URL) {
     headers,
     data
   }) {
+    logger.verbose(`Send message for ${correlationId} to queue: ${queue}`);
+
     try {
-      logger.silly(`Record queue ${queue}`);
-      logger.silly(`Record correlationId ${correlationId}`);
-      logger.silly(`Record data ${JSON.stringify(data)}`);
-      logger.silly(`Record headers ${JSON.stringify(headers)}`);
+      logger.silly(`Queue ${queue}`);
+      logger.silly(`CorrelationId ${correlationId}`);
+      logger.silly(`Data ${JSON.stringify(data)}`);
+      logger.silly(`Headers ${JSON.stringify(headers)}`);
       await channel.assertQueue(queue, {
         durable: true
       });
@@ -358,6 +347,7 @@ async function _default(AMQP_URL) {
   }
 
   async function removeQueue(queue) {
+    logger.verbose(`Removing queue ${queue}.`);
     await channel.deleteQueue(queue);
   } // ----------------
   // Helper functions
@@ -365,7 +355,7 @@ async function _default(AMQP_URL) {
 
 
   function messagesToRecords(messages) {
-    logger.verbose(`Parsing messages (${messages.length})to records`);
+    logger.verbose(`Parsing messages (${messages.length}) to records`);
     return messages.map(message => {
       const content = JSON.parse(message.content.toString());
       return new _marcRecord.MarcRecord(content.data);
@@ -379,9 +369,9 @@ async function _default(AMQP_URL) {
       const {
         messageCount
       } = await channel.checkQueue(queue);
-      logger.debug(`There is ${messageCount} messages in queue ${queue}`);
+      logger.silly(`There is ${messageCount} messages in queue ${queue}`);
       const messagesToGet = messageCount >= _constants.CHUNK_SIZE ? _constants.CHUNK_SIZE : messageCount;
-      logger.debug(`Getting ${messagesToGet} messages from queue ${queue}`);
+      logger.silly(`Getting ${messagesToGet} messages from queue ${queue}`);
       const messages = await pump(messagesToGet);
       logger.debug(`Returning ${messages.length} unique messages`);
       return messages;

package/dist/amqp.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/amqp.js"],"names":["AMQP_URL","connection","amqplib","connect","channel","createChannel","logger","checkQueue","consumeChunk","consumeRawChunk","consumeOne","consumeRaw","ackNReplyMessages","ackMessages","nackMessages","sendToQueue","removeQueue","messagesToRecords","queue","style","purge","toRecord","debug","channelInfo","assertQueue","durable","purgeQueue","messageCount","test","consumeByCorrelationId","ApiError","httpStatus","UNPROCESSABLE_ENTITY","error","verbose","queMessages","getData","headers","getHeaderInfo","JSON","stringify","messages","filter","message","properties","cataloger","nack","records","correlationId","get","status","payloads","length","forEach","index","ids","rejectedIds","data","payload","ack","silly","Buffer","from","persistent","deleteQueue","map","content","parse","toString","MarcRecord","messagesToGet","CHUNK_SIZE","pump","count","results","identifiers","identifier","deliveryTag","fields","includes","concat"],"mappings":";;;;;;;AA8BA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AApCA;;AACA;;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAUe,wBAAgBA,QAAhB,EAA0B;AACvC,QAAMC,UAAU,GAAG,MAAMC,iBAAQC,OAAR,CAAgBH,QAAhB,CAAzB;AACA,QAAMI,OAAO,GAAG,MAAMH,UAAU,CAACI,aAAX,EAAtB;AACA,QAAMC,MAAM,GAAG,0CAAf;AAEA,SAAO;AAACC,IAAAA,UAAD;AAAaC,IAAAA,YAAb;AAA2BC,IAAAA,eAA3B;AAA4CC,IAAAA,UAA5C;AAAwDC,IAAAA,UAAxD;AAAoEC,IAAAA,iBAApE;AAAuFC,IAAAA,WAAvF;AAAoGC,IAAAA,YAApG;AAAkHC,IAAAA,WAAlH;AAA+HC,IAAAA,WAA/H;AAA4IC,IAAAA;AAA5I,GAAP;;AAEA,iBAAeV,UAAf,CAA0BW,KAA1B,EAAiCC,KAAK,GAAG,OAAzC,EAAkDC,KAAK,GAAG,KAA1D,EAAiEC,QAAQ,GAAG,IAA5E,EAAkF;AAChFf,IAAAA,MAAM,CAACgB,KAAP,CAAc,eAAcJ,KAAM,YAAWC,KAAM,EAAnD;;AACA,QAAI;AACF,YAAMI,WAAW,GAAG,MAAMnB,OAAO,CAACoB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAA1B;;AACA,UAAIL,KAAJ,EAAW;AACT,cAAMM,UAAU,CAACN,KAAD,CAAhB;AACAd,QAAAA,MAAM,CAACgB,KAAP,CAAc,SAAQJ,KAAM,eAAcK,WAAW,CAACI,YAAa,UAAnE;AACA,eAAOpB,UAAU,CAACW,KAAD,EAAQC,KAAR,CAAjB;AACD;;AAED,UAAII,WAAW,CAACI,YAAZ,GAA2B,CAA/B,EAAkC;AAChCrB,QAAAA,MAAM,CAACgB,KAAP,CAAc,eAAcC,WAAW,CAACI,YAAa,MAAKT,KAAM,WAAhE;AACA,eAAO,KAAP;AACD;;AACDZ,MAAAA,MAAM,CAACgB,KAAP,CAAc,SAAQJ,KAAM,QAAOK,WAAW,CAACI,YAAa,UAA5D;;AAEA,UAAIR,KAAK,KAAK,UAAd,EAA0B;AACxB,eAAOI,WAAW,CAACI,YAAnB;AACD;;AAED,UAAIR,KAAK,KAAK,KAAd,EAAqB;AACnB,eAAOT,UAAU,CAACQ,KAAD,CAAjB;AACD;;AAED,UAAIC,KAAK,KAAK,KAAd,EAAqB;AACnB,eAAOR,UAAU,CAACO,KAAD,CAAjB;AACD;;AAED,UAAIC,KAAK,KAAK,UAAd,EAA0B;AACxB,eAAOV,eAAe,CAACS,KAAD,CAAtB;AACD;;AAED,UAAK,8BAAD,CAAiCU,IAAjC,CAAsCT,KAAtC,CAAJ,EAAkD;AAChDb,QAAAA,MAAM,CAACgB,KAAP,CAAc,4BAA2BH,KAAM,EAA/C;AACA,eAAOU,sBAAsB,CAACX,KAAD,EAAQC,KAAR,EAAeE,QAAf,CAA7B;AACD;;AAED,UAAIF,KAAK,KAAK,OAAd,EAAuB;AACrB,eAAOX,YAAY,CAACU,KAAD,CAAnB;AACD,OArCC,CAuCF;;;AACA,YAAM,IAAIY,qBAAJ,CAAaC,oBAAWC,oBAAxB,CAAN;AACD,KAzCD,CAyCE,OAAOC,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;;AAED,aAASP,UAAT,CAAoBN,KAApB,EAA2B;AACzB,UAAIA,KAAJ,EAAW;AACT,eAAOhB,OAAO,CAACsB,UAAR,CAAmBR,KAAnB,CAAP;AACD;AACF;AACF;;AAED,iBAAeV,YAAf,CAA4BU,KAA5B,EAAmC;AACjCZ,IAAAA,MAAM,CAAC4B,OAAP,CAAgB,wCAAuChB,KAAM,EAA7D;;AACA,QAAI;AACF,YAAMd,OAAO,CAACoB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN;AACA,YAAMU,WAAW,GAAG,MAAMC,OAAO,CAAClB,KAAD,CAAjC;AAEA,YAAMmB,OAAO,GAAGC,aAAa,CAACH,WAAW,CAAC,CAAD,CAAZ,CAA7B;AACA7B,MAAAA,MAAM,CAACgB,KAAP,CAAc,yBAAwBiB,IAAI,CAACC,SAAL,CAAeH,OAAf,CAAwB,EAA9D,EALE,CAOF;;AACA,YAAMI,QAAQ,GAAGN,WAAW,CAACO,MAAZ,CAAmBC,OAAO,IAAI;AAC7C,YAAIA,OAAO,CAACC,UAAR,CAAmBP,OAAnB,CAA2BQ,SAA3B,KAAyCR,OAAO,CAACQ,SAArD,EAAgE;AAC9D,iBAAO,IAAP;AACD,SAH4C,CAK7C;;;AACAzC,QAAAA,OAAO,CAAC0C,IAAR,CAAaH,OAAb,EAAsB,KAAtB,EAA6B,IAA7B;AACA,eAAO,KAAP;AACD,OARgB,CAAjB;AAUA,YAAMI,OAAO,GAAG,MAAM9B,iBAAiB,CAACwB,QAAD,CAAvC;AAEA,aAAO;AAACJ,QAAAA,OAAD;AAAUU,QAAAA,OAAV;AAAmBN,QAAAA;AAAnB,OAAP;AACD,KArBD,CAqBE,OAAOR,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAeJ,sBAAf,CAAsCX,KAAtC,EAA6C8B,aAA7C,EAA4D3B,QAAQ,GAAG,IAAvE,EAA6E;AAC3Ef,IAAAA,MAAM,CAAC4B,OAAP,CAAgB,kDAAiDhB,KAAM,EAAvE;;AACA,QAAI;AACF,YAAMd,OAAO,CAACoB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN;AACA,YAAMU,WAAW,GAAG,MAAMC,OAAO,CAAClB,KAAD,CAAjC;AAEAZ,MAAAA,MAAM,CAACgB,KAAP,CAAc,yBAAwB0B,aAAc,EAApD,EAJE,CAMF;;AACA,YAAMP,QAAQ,GAAGN,WAAW,CAACO,MAAZ,CAAmBC,OAAO,IAAI;AAC7C,YAAIA,OAAO,CAACC,UAAR,CAAmBI,aAAnB,KAAqCA,aAAzC,EAAwD;AACtD,iBAAO,IAAP;AACD,SAH4C,CAK7C;;;AACA5C,QAAAA,OAAO,CAAC0C,IAAR,CAAaH,OAAb,EAAsB,KAAtB,EAA6B,IAA7B;AACA,eAAO,KAAP;AACD,OARgB,CAAjB;AASA,YAAMN,OAAO,GAAGC,aAAa,CAACG,QAAQ,CAAC,CAAD,CAAT,CAA7B;;AAEA,UAAIpB,QAAJ,EAAc;AACZ,cAAM0B,OAAO,GAAG,MAAM9B,iBAAiB,CAACwB,QAAD,CAAvC;AAEA,eAAO;AAACJ,UAAAA,OAAD;AAAUU,UAAAA,OAAV;AAAmBN,UAAAA;AAAnB,SAAP;AACD;;AACD,aAAO;AAACJ,QAAAA,OAAD;AAAUI,QAAAA;AAAV,OAAP;AACD,KAxBD,CAwBE,OAAOR,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAexB,eAAf,CAA+BS,KAA/B,EAAsC;AACpCZ,IAAAA,MAAM,CAAC4B,OAAP,CAAgB,2CAA0ChB,KAAM,EAAhE;;AACA,QAAI;AACF,YAAMd,OAAO,CAACoB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN;AACA,YAAMU,WAAW,GAAG,MAAMC,OAAO,CAAClB,KAAD,CAAjC;AAEA,YAAMmB,OAAO,GAAGC,aAAa,CAACH,WAAW,CAAC,CAAD,CAAZ,CAA7B;AACA7B,MAAAA,MAAM,CAACgB,KAAP,CAAc,yBAAwBiB,IAAI,CAACC,SAAL,CAAeH,OAAf,CAAwB,cAA9D,EALE,CAOF;;AACA,YAAMI,QAAQ,GAAGN,WAAW,CAACO,MAAZ,CAAmBC,OAAO,IAAI;AAC7C,YAAIA,OAAO,CAACC,UAAR,CAAmBP,OAAnB,CAA2BQ,SAA3B,KAAyCR,OAAO,CAACQ,SAArD,EAAgE;AAC9D,iBAAO,IAAP;AACD,SAH4C,CAK7C;;;AACAzC,QAAAA,OAAO,CAAC0C,IAAR,CAAaH,OAAb,EAAsB,KAAtB,EAA6B,IAA7B;AACA,eAAO,KAAP;AACD,OARgB,CAAjB;AAUA,aAAO;AAACN,QAAAA,OAAD;AAAUI,QAAAA;AAAV,OAAP;AACD,KAnBD,CAmBE,OAAOR,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAevB,UAAf,CAA0BQ,KAA1B,EAAiC;AAC/BZ,IAAAA,MAAM,CAAC4B,OAAP,CAAgB,uCAAsChB,KAAM,EAA5D;;AACA,QAAI;AACF,YAAMd,OAAO,CAACoB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAGF;;AACA,YAAMkB,OAAO,GAAG,MAAMvC,OAAO,CAAC6C,GAAR,CAAY/B,KAAZ,CAAtB;;AACA,UAAIyB,OAAJ,EAAa;AACX,cAAMN,OAAO,GAAGC,aAAa,CAACK,OAAD,CAA7B;AACA,cAAMI,OAAO,GAAG9B,iBAAiB,CAAC,CAAC0B,OAAD,CAAD,CAAjC;AACA,eAAO;AAACN,UAAAA,OAAD;AAAUU,UAAAA,OAAV;AAAmBN,UAAAA,QAAQ,EAAE,CAACE,OAAD;AAA7B,SAAP;AACD;;AAED,aAAO,KAAP;AACD,KAZD,CAYE,OAAOV,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAetB,UAAf,CAA0BO,KAA1B,EAAiC;AAC/BZ,IAAAA,MAAM,CAAC4B,OAAP,CAAgB,uCAAsChB,KAAM,EAA5D;;AACA,QAAI;AACF,YAAMd,OAAO,CAACoB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAEF;;AACA,aAAO,MAAMrB,OAAO,CAAC6C,GAAR,CAAY/B,KAAZ,CAAb;AACD,KAJD,CAIE,OAAOe,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF,GA9KsC,CAgLvC;;;AACA,iBAAerB,iBAAf,CAAiC;AAACsC,IAAAA,MAAD;AAAST,IAAAA,QAAT;AAAmBU,IAAAA;AAAnB,GAAjC,EAA+D;AAC7D7C,IAAAA,MAAM,CAAC4B,OAAP,CAAe,yBAAf;AACA5B,IAAAA,MAAM,CAACgB,KAAP,CAAc,mCAAkCiB,IAAI,CAACC,SAAL,CAAeU,MAAf,CAAuB,cAAaT,QAAS,cAAaF,IAAI,CAACC,SAAL,CAAeW,QAAf,CAAyB,EAAnI;AACA7C,IAAAA,MAAM,CAACgB,KAAP,CAAc,YAAWmB,QAAQ,CAACW,MAAO,WAAzC;AACA,UAAMX,QAAQ,CAACY,OAAT,CAAiB,CAACV,OAAD,EAAUW,KAAV,KAAoB;AACzC,YAAMjB,OAAO,GAAGC,aAAa,CAACK,OAAD,CAA7B;AAEArC,MAAAA,MAAM,CAACgB,KAAP,CAAc,YAAWqB,OAAQ,YAAWW,KAAM,EAAlD;AACAhD,MAAAA,MAAM,CAACgB,KAAP,CAAc,YAAWiB,IAAI,CAACC,SAAL,CAAeH,OAAf,CAAwB,EAAjD,EAJyC,CAMzC;;AACA,YAAMkB,GAAG,GAAGJ,QAAQ,CAACI,GAAT,IAAgBJ,QAA5B;AACA,YAAMK,WAAW,GAAGL,QAAQ,CAACK,WAAT,IAAwB,EAA5C;AACAlD,MAAAA,MAAM,CAACgB,KAAP,CAAc,QAAOiB,IAAI,CAACC,SAAL,CAAee,GAAf,CAAoB,kBAAiBhB,IAAI,CAACC,SAAL,CAAegB,WAAf,CAA4B,EAAtF;;AAEA,UAAID,GAAG,CAACH,MAAJ,GAAa,CAAb,IAAkBI,WAAW,CAACJ,MAAZ,GAAqB,CAA3C,EAA8C;AAC5C9C,QAAAA,MAAM,CAACgB,KAAP,CAAc,gCAA+BkC,WAAY,EAAzD;AAEAzC,QAAAA,WAAW,CAAC;AACVG,UAAAA,KAAK,EAAEyB,OAAO,CAACC,UAAR,CAAmBI,aADhB;AAEVA,UAAAA,aAAa,EAAEL,OAAO,CAACC,UAAR,CAAmBI,aAFxB;AAGVX,UAAAA,OAHU;AAIVoB,UAAAA,IAAI,EAAE;AACJP,YAAAA,MAAM,EAAEnB,oBAAWC,oBADf;AACqC0B,YAAAA,OAAO,EAAEF;AAD9C;AAJI,SAAD,CAAX;AASA,eAAOpD,OAAO,CAACuD,GAAR,CAAYhB,OAAZ,CAAP;AACD,OAxBwC,CA0BzC;;;AACA5B,MAAAA,WAAW,CAAC;AACVG,QAAAA,KAAK,EAAEyB,OAAO,CAACC,UAAR,CAAmBI,aADhB;AAEVA,QAAAA,aAAa,EAAEL,OAAO,CAACC,UAAR,CAAmBI,aAFxB;AAGVX,QAAAA,OAHU;AAIVoB,QAAAA,IAAI,EAAE;AACJP,UAAAA,MADI;AACIQ,UAAAA,OAAO,EAAEH,GAAG,CAACD,KAAD;AADhB;AAJI,OAAD,CAAX;AASAlD,MAAAA,OAAO,CAACuD,GAAR,CAAYhB,OAAZ;AACD,KArCK,CAAN;AAsCD;;AAED,WAAS9B,WAAT,CAAqB4B,QAArB,EAA+B;AAC7BnC,IAAAA,MAAM,CAAC4B,OAAP,CAAgB,kBAAiBO,QAAQ,CAACW,MAAO,GAAjD;AACAX,IAAAA,QAAQ,CAACY,OAAT,CAAiBV,OAAO,IAAI;AAC1BrC,MAAAA,MAAM,CAACsD,KAAP,CAAc,eAAcjB,OAAO,CAACC,UAAR,CAAmBI,aAAc,EAA7D;AACA5C,MAAAA,OAAO,CAACuD,GAAR,CAAYhB,OAAZ;AACD,KAHD;AAID;;AAED,WAAS7B,YAAT,CAAsB2B,QAAtB,EAAgC;AAC9BnC,IAAAA,MAAM,CAAC4B,OAAP,CAAgB,mBAAkBO,QAAQ,CAACW,MAAO,GAAlD;AACAX,IAAAA,QAAQ,CAACY,OAAT,CAAiBV,OAAO,IAAI;AAC1BrC,MAAAA,MAAM,CAACsD,KAAP,CAAc,gBAAejB,OAAO,CAACC,UAAR,CAAmBI,aAAc,EAA9D;AACA5C,MAAAA,OAAO,CAAC0C,IAAR,CAAaH,OAAb;AACD,KAHD;AAID;;AAED,iBAAe5B,WAAf,CAA2B;AAACG,IAAAA,KAAD;AAAQ8B,IAAAA,aAAR;AAAuBX,IAAAA,OAAvB;AAAgCoB,IAAAA;AAAhC,GAA3B,EAAkE;AAChE,QAAI;AACFnD,MAAAA,MAAM,CAACsD,KAAP,CAAc,gBAAe1C,KAAM,EAAnC;AACAZ,MAAAA,MAAM,CAACsD,KAAP,CAAc,wBAAuBZ,aAAc,EAAnD;AACA1C,MAAAA,MAAM,CAACsD,KAAP,CAAc,eAAcrB,IAAI,CAACC,SAAL,CAAeiB,IAAf,CAAqB,EAAjD;AACAnD,MAAAA,MAAM,CAACsD,KAAP,CAAc,kBAAiBrB,IAAI,CAACC,SAAL,CAAeH,OAAf,CAAwB,EAAvD;AAEA,YAAMjC,OAAO,CAACoB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN;AAEArB,MAAAA,OAAO,CAACW,WAAR,CACEG,KADF,EAEE2C,MAAM,CAACC,IAAP,CAAYvB,IAAI,CAACC,SAAL,CAAe;AAACiB,QAAAA;AAAD,OAAf,CAAZ,CAFF,EAGE;AACET,QAAAA,aADF;AAEEe,QAAAA,UAAU,EAAE,IAFd;AAGE1B,QAAAA;AAHF,OAHF;AAUA/B,MAAAA,MAAM,CAACsD,KAAP,CAAc,yBAAwB1C,KAAM,EAA5C;AACD,KAnBD,CAmBE,OAAOe,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAejB,WAAf,CAA2BE,KAA3B,EAAkC;AAChC,UAAMd,OAAO,CAAC4D,WAAR,CAAoB9C,KAApB,CAAN;AACD,GAxQsC,CA0QvC;AACA;AACA;;;AAEA,WAASD,iBAAT,CAA2BwB,QAA3B,EAAqC;AACnCnC,IAAAA,MAAM,CAAC4B,OAAP,CAAgB,qBAAoBO,QAAQ,CAACW,MAAO,aAApD;AAEA,WAAOX,QAAQ,CAACwB,GAAT,CAAatB,OAAO,IAAI;AAC7B,YAAMuB,OAAO,GAAG3B,IAAI,CAAC4B,KAAL,CAAWxB,OAAO,CAACuB,OAAR,CAAgBE,QAAhB,EAAX,CAAhB;AACA,aAAO,IAAIC,sBAAJ,CAAeH,OAAO,CAACT,IAAvB,CAAP;AACD,KAHM,CAAP;AAID;;AAED,iBAAerB,OAAf,CAAuBlB,KAAvB,EAA8B;AAC5BZ,IAAAA,MAAM,CAAC4B,OAAP,CAAgB,2BAA0BhB,KAAM,EAAhD;;AACA,QAAI;AACF,YAAM;AAACS,QAAAA;AAAD,UAAiB,MAAMvB,OAAO,CAACG,UAAR,CAAmBW,KAAnB,CAA7B;AACAZ,MAAAA,MAAM,CAACgB,KAAP,CAAc,YAAWK,YAAa,sBAAqBT,KAAM,EAAjE;AACA,YAAMoD,aAAa,GAAG3C,YAAY,IAAI4C,qBAAhB,GAA6BA,qBAA7B,GAA0C5C,YAAhE;AACArB,MAAAA,MAAM,CAACgB,KAAP,CAAc,WAAUgD,aAAc,wBAAuBpD,KAAM,EAAnE;AAEA,YAAMuB,QAAQ,GAAG,MAAM+B,IAAI,CAACF,aAAD,CAA3B;AAEAhE,MAAAA,MAAM,CAACgB,KAAP,CAAc,aAAYmB,QAAQ,CAACW,MAAO,kBAA1C;AAEA,aAAOX,QAAP;AACD,KAXD,CAWE,OAAOR,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;;AAED,mBAAeuC,IAAf,CAAoBC,KAApB,EAA2BC,OAAO,GAAG,EAArC,EAAyCC,WAAW,GAAG,EAAvD,EAA2D;AACzD,UAAIF,KAAK,KAAK,CAAd,EAAiB;AACf,eAAOC,OAAP;AACD;;AAED,YAAM/B,OAAO,GAAG,MAAMvC,OAAO,CAAC6C,GAAR,CAAY/B,KAAZ,CAAtB;AACA,YAAM0D,UAAU,GAAG;AACjB5B,QAAAA,aAAa,EAAEL,OAAO,CAACC,UAAR,CAAmBI,aADjB;AAEjB6B,QAAAA,WAAW,EAAElC,OAAO,CAACmC,MAAR,CAAeD;AAFX,OAAnB,CANyD,CAUzD;;AACA,UAAIF,WAAW,CAACI,QAAZ,CAAqBH,UAArB,CAAJ,EAAsC;AACpC,eAAOJ,IAAI,CAACC,KAAK,GAAG,CAAT,EAAYC,OAAZ,EAAqBC,WAArB,CAAX;AACD;;AAED,aAAOH,IAAI,CAACC,KAAK,GAAG,CAAT,EAAYC,OAAO,CAACM,MAAR,CAAerC,OAAf,CAAZ,EAAqCgC,WAAW,CAACK,MAAZ,CAAmBJ,UAAnB,CAArC,CAAX;AACD;AACF;;AAED,WAAStC,aAAT,CAAuBmB,IAAvB,EAA6B;AAC3B,WAAOA,IAAI,CAACb,UAAL,CAAgBP,OAAvB;AACD;AACF","sourcesContent":["/* eslint-disable max-statements */\n/* eslint-disable max-lines */\n/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport amqplib from 'amqplib';\nimport {MarcRecord} from '@natlibfi/marc-record';\nimport {Error as ApiError} from '@natlibfi/melinda-commons';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {CHUNK_SIZE} from './constants';\nimport {logError} from './utils';\nimport httpStatus from 'http-status';\n\nexport default async function (AMQP_URL) {\n  const connection = await amqplib.connect(AMQP_URL);\n  const channel = await connection.createChannel();\n  const logger = createLogger();\n\n  return {checkQueue, consumeChunk, consumeRawChunk, consumeOne, consumeRaw, ackNReplyMessages, ackMessages, nackMessages, sendToQueue, removeQueue, messagesToRecords};\n\n  async function checkQueue(queue, style = 'basic', purge = false, toRecord = true) {\n    logger.debug(`checkQueue: ${queue}, Style: ${style}`);\n    try {\n      const channelInfo = await channel.assertQueue(queue, {durable: true});\n      if (purge) {\n        await purgeQueue(purge);\n        logger.debug(`Queue ${queue} has purged ${channelInfo.messageCount} records`);\n        return checkQueue(queue, style);\n      }\n\n      if (channelInfo.messageCount < 1) {\n        logger.debug(`checkQueue: ${channelInfo.messageCount} - ${queue} is empty`);\n        return false;\n      }\n      logger.debug(`Queue ${queue} has ${channelInfo.messageCount} records`);\n\n      if (style === 'messages') {\n        return channelInfo.messageCount;\n      }\n\n      if (style === 'one') {\n        return consumeOne(queue);\n      }\n\n      if (style === 'raw') {\n        return consumeRaw(queue);\n      }\n\n      if (style === 'rawChunk') {\n        return consumeRawChunk(queue);\n      }\n\n      if ((/^.{8}-.{4}-.{4}-.{4}-.{12}$/u).test(style)) {\n        logger.debug(`checkQueue: regexp match ${style}`);\n        return consumeByCorrelationId(queue, style, toRecord);\n      }\n\n      if (style === 'basic') {\n        return consumeChunk(queue);\n      }\n\n      // Defaults:\n      throw new ApiError(httpStatus.UNPROCESSABLE_ENTITY);\n    } catch (error) {\n      logError(error);\n    }\n\n    function purgeQueue(purge) {\n      if (purge) {\n        return channel.purgeQueue(queue);\n      }\n    }\n  }\n\n  async function consumeChunk(queue) {\n    logger.verbose(`Prepared to consumeChunk from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n      const queMessages = await getData(queue);\n\n      const headers = getHeaderInfo(queMessages[0]);\n      logger.debug(`Filtering messages by ${JSON.stringify(headers)}`);\n\n      // Check that cataloger match! headers\n      const messages = queMessages.filter(message => {\n        if (message.properties.headers.cataloger === headers.cataloger) {\n          return true;\n        }\n\n        // Nack unwanted ones\n        channel.nack(message, false, true);\n        return false;\n      });\n\n      const records = await messagesToRecords(messages);\n\n      return {headers, records, messages};\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeByCorrelationId(queue, correlationId, toRecord = true) {\n    logger.verbose(`Prepared to consumeByCorrelationId from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n      const queMessages = await getData(queue);\n\n      logger.debug(`Filtering messages by ${correlationId}`);\n\n      // Check that cataloger match! headers\n      const messages = queMessages.filter(message => {\n        if (message.properties.correlationId === correlationId) {\n          return true;\n        }\n\n        // Nack unwanted ones\n        channel.nack(message, false, true);\n        return false;\n      });\n      const headers = getHeaderInfo(messages[0]);\n\n      if (toRecord) {\n        const records = await messagesToRecords(messages);\n\n        return {headers, records, messages};\n      }\n      return {headers, messages};\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeRawChunk(queue) {\n    logger.verbose(`Prepared to consumeRawChunk from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n      const queMessages = await getData(queue);\n\n      const headers = getHeaderInfo(queMessages[0]);\n      logger.debug(`Filtering messages by ${JSON.stringify(headers)} and timeout`);\n\n      // Check that cataloger match! headers\n      const messages = queMessages.filter(message => {\n        if (message.properties.headers.cataloger === headers.cataloger) {\n          return true;\n        }\n\n        // Nack unwanted ones\n        channel.nack(message, false, true);\n        return false;\n      });\n\n      return {headers, messages};\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeOne(queue) {\n    logger.verbose(`Prepared to consume one from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n\n      // Returns false if 0 items in queue\n      const message = await channel.get(queue);\n      if (message) {\n        const headers = getHeaderInfo(message);\n        const records = messagesToRecords([message]);\n        return {headers, records, messages: [message]};\n      }\n\n      return false;\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeRaw(queue) {\n    logger.verbose(`Prepared to consume raw from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n      // Returns false if 0 items in queue\n      return await channel.get(queue);\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  // ACK records\n  async function ackNReplyMessages({status, messages, payloads}) {\n    logger.verbose('Ack and reply messages!');\n    logger.debug(`Ack and reply messages. status: ${JSON.stringify(status)} messages: ${messages} payloads: ${JSON.stringify(payloads)}`);\n    logger.debug(`Handling ${messages.length} messages`);\n    await messages.forEach((message, index) => {\n      const headers = getHeaderInfo(message);\n\n      logger.debug(`Message: ${message}, index: ${index}`);\n      logger.debug(`Headers: ${JSON.stringify(headers)}`);\n\n      // payloads ids are parsed both form old payloads including just array of ids and new payloads including both array of ids and array of rejected ids\n      const ids = payloads.ids || payloads;\n      const rejectedIds = payloads.rejectedIds || [];\n      logger.debug(`ids: ${JSON.stringify(ids)}, rejectedIds: ${JSON.stringify(rejectedIds)}`);\n\n      if (ids.length < 1 && rejectedIds.length > 0) {\n        logger.debug(`Got 0 valid ids and rejected ${rejectedIds}`);\n\n        sendToQueue({\n          queue: message.properties.correlationId,\n          correlationId: message.properties.correlationId,\n          headers,\n          data: {\n            status: httpStatus.UNPROCESSABLE_ENTITY, payload: rejectedIds\n          }\n        });\n\n        return channel.ack(message);\n      }\n\n      // Reply consumer gets: {\"data\":{\"status\":\"UPDATED\",\"payload\":\"0\"}}\n      sendToQueue({\n        queue: message.properties.correlationId,\n        correlationId: message.properties.correlationId,\n        headers,\n        data: {\n          status, payload: ids[index]\n        }\n      });\n\n      channel.ack(message);\n    });\n  }\n\n  function ackMessages(messages) {\n    logger.verbose(`Ack messages! (${messages.length})`);\n    messages.forEach(message => {\n      logger.silly(`Ack message ${message.properties.correlationId}`);\n      channel.ack(message);\n    });\n  }\n\n  function nackMessages(messages) {\n    logger.verbose(`Nack messages! (${messages.length})`);\n    messages.forEach(message => {\n      logger.silly(`Nack message ${message.properties.correlationId}`);\n      channel.nack(message);\n    });\n  }\n\n  async function sendToQueue({queue, correlationId, headers, data}) {\n    try {\n      logger.silly(`Record queue ${queue}`);\n      logger.silly(`Record correlationId ${correlationId}`);\n      logger.silly(`Record data ${JSON.stringify(data)}`);\n      logger.silly(`Record headers ${JSON.stringify(headers)}`);\n\n      await channel.assertQueue(queue, {durable: true});\n\n      channel.sendToQueue(\n        queue,\n        Buffer.from(JSON.stringify({data})),\n        {\n          correlationId,\n          persistent: true,\n          headers\n        }\n      );\n\n      logger.silly(`Message send to queue ${queue}`);\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function removeQueue(queue) {\n    await channel.deleteQueue(queue);\n  }\n\n  // ----------------\n  // Helper functions\n  // ----------------\n\n  function messagesToRecords(messages) {\n    logger.verbose(`Parsing messages (${messages.length})to records`);\n\n    return messages.map(message => {\n      const content = JSON.parse(message.content.toString());\n      return new MarcRecord(content.data);\n    });\n  }\n\n  async function getData(queue) {\n    logger.verbose(`Getting queue data from ${queue}`);\n    try {\n      const {messageCount} = await channel.checkQueue(queue);\n      logger.debug(`There is ${messageCount} messages in queue ${queue}`);\n      const messagesToGet = messageCount >= CHUNK_SIZE ? CHUNK_SIZE : messageCount;\n      logger.debug(`Getting ${messagesToGet} messages from queue ${queue}`);\n\n      const messages = await pump(messagesToGet);\n\n      logger.debug(`Returning ${messages.length} unique messages`);\n\n      return messages;\n    } catch (error) {\n      logError(error);\n    }\n\n    async function pump(count, results = [], identifiers = []) {\n      if (count === 0) {\n        return results;\n      }\n\n      const message = await channel.get(queue);\n      const identifier = {\n        correlationId: message.properties.correlationId,\n        deliveryTag: message.fields.deliveryTag\n      };\n      // Filter not unique messages\n      if (identifiers.includes(identifier)) {\n        return pump(count - 1, results, identifiers);\n      }\n\n      return pump(count - 1, results.concat(message), identifiers.concat(identifier));\n    }\n  }\n\n  function getHeaderInfo(data) {\n    return data.properties.headers;\n  }\n}\n"],"file":"amqp.js"}
+{"version":3,"sources":["../src/amqp.js"],"names":["AMQP_URL","connection","amqplib","connect","channel","createChannel","logger","checkQueue","consumeChunk","consumeRawChunk","consumeOne","consumeRaw","ackMessages","nackMessages","sendToQueue","removeQueue","messagesToRecords","queue","style","purge","toRecord","verbose","channelInfo","assertQueue","durable","purgeQueue","debug","messageCount","silly","test","consumeByCorrelationId","ApiError","httpStatus","UNPROCESSABLE_ENTITY","error","queMessages","getData","headers","getHeaderInfo","JSON","stringify","filterIn","filterOut","messages","filter","message","properties","cataloger","nack","records","correlationId","get","forEach","ack","data","Buffer","from","persistent","deleteQueue","length","map","content","parse","toString","MarcRecord","messagesToGet","CHUNK_SIZE","pump","count","results","identifiers","identifier","deliveryTag","fields","includes","concat"],"mappings":";;;;;;;AA8BA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AApCA;;AACA;;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAUe,wBAAgBA,QAAhB,EAA0B;AACvC,QAAMC,UAAU,GAAG,MAAMC,iBAAQC,OAAR,CAAgBH,QAAhB,CAAzB;AACA,QAAMI,OAAO,GAAG,MAAMH,UAAU,CAACI,aAAX,EAAtB;AACA,QAAMC,MAAM,GAAG,0CAAf;AAEA,SAAO;AAACC,IAAAA,UAAD;AAAaC,IAAAA,YAAb;AAA2BC,IAAAA,eAA3B;AAA4CC,IAAAA,UAA5C;AAAwDC,IAAAA,UAAxD;AAAoEC,IAAAA,WAApE;AAAiFC,IAAAA,YAAjF;AAA+FC,IAAAA,WAA/F;AAA4GC,IAAAA,WAA5G;AAAyHC,IAAAA;AAAzH,GAAP;;AAEA,iBAAeT,UAAf,CAA0BU,KAA1B,EAAiCC,KAAK,GAAG,OAAzC,EAAkDC,KAAK,GAAG,KAA1D,EAAiEC,QAAQ,GAAG,IAA5E,EAAkF;AAChFd,IAAAA,MAAM,CAACe,OAAP,CAAgB,eAAcJ,KAAM,YAAWC,KAAM,YAAWC,KAAM,eAAcC,QAAS,EAA7F;;AACA,QAAI;AACF,YAAME,WAAW,GAAG,MAAMlB,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAA1B;;AACA,UAAIL,KAAJ,EAAW;AACT,cAAMM,UAAU,CAACN,KAAD,CAAhB;AACAb,QAAAA,MAAM,CAACoB,KAAP,CAAc,SAAQT,KAAM,eAAcK,WAAW,CAACK,YAAa,WAAnE;AACA,eAAOpB,UAAU,CAACU,KAAD,EAAQC,KAAR,CAAjB;AACD;;AAED,UAAII,WAAW,CAACK,YAAZ,GAA2B,CAA/B,EAAkC;AAChCrB,QAAAA,MAAM,CAACoB,KAAP,CAAc,eAAcJ,WAAW,CAACK,YAAa,MAAKV,KAAM,WAAhE;AACA,eAAO,KAAP;AACD;;AACDX,MAAAA,MAAM,CAACsB,KAAP,CAAc,SAAQX,KAAM,QAAOK,WAAW,CAACK,YAAa,WAA5D;;AAEA,UAAIT,KAAK,KAAK,UAAd,EAA0B;AACxB,eAAOI,WAAW,CAACK,YAAnB;AACD,OAhBC,CAkBF;;;AACA,UAAIT,KAAK,KAAK,KAAd,EAAqB;AACnB,eAAOR,UAAU,CAACO,KAAD,CAAjB;AACD,OArBC,CAuBF;;;AACA,UAAIC,KAAK,KAAK,KAAd,EAAqB;AACnB,eAAOP,UAAU,CAACM,KAAD,CAAjB;AACD,OA1BC,CA4BF;;;AACA,UAAIC,KAAK,KAAK,UAAd,EAA0B;AACxB,eAAOT,eAAe,CAACQ,KAAD,CAAtB;AACD,OA/BC,CAiCF;;;AACA,UAAK,8BAAD,CAAiCY,IAAjC,CAAsCX,KAAtC,CAAJ,EAAkD;AAChDZ,QAAAA,MAAM,CAACsB,KAAP,CAAc,4BAA2BV,KAAM,EAA/C;AACA,eAAOY,sBAAsB,CAACb,KAAD,EAAQC,KAAR,EAAeE,QAAf,CAA7B;AACD,OArCC,CAuCF;;;AACA,UAAIF,KAAK,KAAK,OAAd,EAAuB;AACrB,eAAOV,YAAY,CAACS,KAAD,CAAnB;AACD,OA1CC,CA4CF;;;AACA,YAAM,IAAIc,qBAAJ,CAAaC,oBAAWC,oBAAxB,CAAN;AACD,KA9CD,CA8CE,OAAOC,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;;AAED,aAAST,UAAT,CAAoBN,KAApB,EAA2B;AACzB,UAAIA,KAAJ,EAAW;AACT,eAAOf,OAAO,CAACqB,UAAR,CAAmBR,KAAnB,CAAP;AACD;AACF;AACF;;AAED,iBAAeT,YAAf,CAA4BS,KAA5B,EAAmC;AACjCX,IAAAA,MAAM,CAACe,OAAP,CAAgB,wCAAuCJ,KAAM,EAA7D;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAGF;;AACA,YAAMW,WAAW,GAAG,MAAMC,OAAO,CAACnB,KAAD,CAAjC;AAEA,YAAMoB,OAAO,GAAGC,aAAa,CAACH,WAAW,CAAC,CAAD,CAAZ,CAA7B;AACA7B,MAAAA,MAAM,CAACoB,KAAP,CAAc,sCAAqCa,IAAI,CAACC,SAAL,CAAeH,OAAf,CAAwB,EAA3E;AAEA,UAAII,QAAQ,GAAG,CAAf,CATE,CASgB;;AAClB,UAAIC,SAAS,GAAG,CAAhB,CAVE,CAUiB;AAEnB;;AACA,YAAMC,QAAQ,GAAGR,WAAW,CAACS,MAAZ,CAAmBC,OAAO,IAAI;AAC7C,YAAIA,OAAO,CAACC,UAAR,CAAmBT,OAAnB,CAA2BU,SAA3B,KAAyCV,OAAO,CAACU,SAArD,EAAgE;AAC9DN,UAAAA,QAAQ,GADsD,CAClD;;AACZ,iBAAO,IAAP;AACD,SAJ4C,CAM7C;;;AACArC,QAAAA,OAAO,CAAC4C,IAAR,CAAaH,OAAb,EAAsB,KAAtB,EAA6B,IAA7B;AACAH,QAAAA,SAAS,GARoC,CAQhC;;AACb,eAAO,KAAP;AACD,OAVgB,CAAjB;AAYApC,MAAAA,MAAM,CAACoB,KAAP,CAAc,4BAA2Be,QAAS,eAAcC,SAAU,EAA1E;AACA,YAAMO,OAAO,GAAG,MAAMjC,iBAAiB,CAAC2B,QAAD,CAAvC;AAEA,aAAO;AAACN,QAAAA,OAAD;AAAUY,QAAAA,OAAV;AAAmBN,QAAAA;AAAnB,OAAP;AACD,KA7BD,CA6BE,OAAOT,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAeJ,sBAAf,CAAsCb,KAAtC,EAA6CiC,aAA7C,EAA4D9B,QAAQ,GAAG,IAAvE,EAA6E;AAC3Ed,IAAAA,MAAM,CAACe,OAAP,CAAgB,kDAAiDJ,KAAM,EAAvE;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAEF;;AACA,YAAMW,WAAW,GAAG,MAAMC,OAAO,CAACnB,KAAD,CAAjC;AAEAX,MAAAA,MAAM,CAACoB,KAAP,CAAc,yBAAwBwB,aAAc,EAApD;AAEA,UAAIT,QAAQ,GAAG,CAAf,CAPE,CAOgB;;AAClB,UAAIC,SAAS,GAAG,CAAhB,CARE,CAQiB;AAEnB;;AACA,YAAMC,QAAQ,GAAGR,WAAW,CAACS,MAAZ,CAAmBC,OAAO,IAAI;AAC7C,YAAIA,OAAO,CAACC,UAAR,CAAmBI,aAAnB,KAAqCA,aAAzC,EAAwD;AACtDT,UAAAA,QAAQ,GAD8C,CAC1C;;AACZ,iBAAO,IAAP;AACD,SAJ4C,CAM7C;;;AACArC,QAAAA,OAAO,CAAC4C,IAAR,CAAaH,OAAb,EAAsB,KAAtB,EAA6B,IAA7B;AACAH,QAAAA,SAAS,GARoC,CAQhC;;AACb,eAAO,KAAP;AACD,OAVgB,CAAjB;AAWApC,MAAAA,MAAM,CAACoB,KAAP,CAAc,4BAA2Be,QAAS,eAAcC,SAAU,EAA1E;AAEA,YAAML,OAAO,GAAGC,aAAa,CAACK,QAAQ,CAAC,CAAD,CAAT,CAA7B;;AAEA,UAAIvB,QAAJ,EAAc;AACZ,cAAM6B,OAAO,GAAG,MAAMjC,iBAAiB,CAAC2B,QAAD,CAAvC;AAEA,eAAO;AAACN,UAAAA,OAAD;AAAUY,UAAAA,OAAV;AAAmBN,UAAAA;AAAnB,SAAP;AACD;;AACD,aAAO;AAACN,QAAAA,OAAD;AAAUM,QAAAA;AAAV,OAAP;AACD,KAhCD,CAgCE,OAAOT,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAezB,eAAf,CAA+BQ,KAA/B,EAAsC;AACpCX,IAAAA,MAAM,CAACe,OAAP,CAAgB,2CAA0CJ,KAAM,EAAhE;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAEF;;AACA,YAAMW,WAAW,GAAG,MAAMC,OAAO,CAACnB,KAAD,CAAjC;AAEA,YAAMoB,OAAO,GAAGC,aAAa,CAACH,WAAW,CAAC,CAAD,CAAZ,CAA7B;AACA7B,MAAAA,MAAM,CAACoB,KAAP,CAAc,yBAAwBa,IAAI,CAACC,SAAL,CAAeH,OAAf,CAAwB,EAA9D;AAEA,UAAII,QAAQ,GAAG,CAAf,CARE,CAQgB;;AAClB,UAAIC,SAAS,GAAG,CAAhB,CATE,CASiB;AAEnB;;AACA,YAAMC,QAAQ,GAAGR,WAAW,CAACS,MAAZ,CAAmBC,OAAO,IAAI;AAC7C,YAAIA,OAAO,CAACC,UAAR,CAAmBT,OAAnB,CAA2BU,SAA3B,KAAyCV,OAAO,CAACU,SAArD,EAAgE;AAC9DN,UAAAA,QAAQ,GADsD,CAClD;;AAEZ,iBAAO,IAAP;AACD,SAL4C,CAO7C;;;AACArC,QAAAA,OAAO,CAAC4C,IAAR,CAAaH,OAAb,EAAsB,KAAtB,EAA6B,IAA7B;AACAH,QAAAA,SAAS,GAToC,CAShC;;AAEb,eAAO,KAAP;AACD,OAZgB,CAAjB;AAaApC,MAAAA,MAAM,CAACoB,KAAP,CAAc,4BAA2Be,QAAS,eAAcC,SAAU,EAA1E,EAzBE,CA2BF;;AACA,aAAO;AAACL,QAAAA,OAAD;AAAUM,QAAAA;AAAV,OAAP;AACD,KA7BD,CA6BE,OAAOT,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAexB,UAAf,CAA0BO,KAA1B,EAAiC;AAC/BX,IAAAA,MAAM,CAACe,OAAP,CAAgB,sCAAqCJ,KAAM,EAA3D;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAGF;;AACA,YAAMqB,OAAO,GAAG,MAAMzC,OAAO,CAAC+C,GAAR,CAAYlC,KAAZ,CAAtB;;AACA,UAAI4B,OAAJ,EAAa;AACX,cAAMR,OAAO,GAAGC,aAAa,CAACO,OAAD,CAA7B;AACA,cAAMI,OAAO,GAAGjC,iBAAiB,CAAC,CAAC6B,OAAD,CAAD,CAAjC;AACA,eAAO;AAACR,UAAAA,OAAD;AAAUY,UAAAA,OAAV;AAAmBN,UAAAA,QAAQ,EAAE,CAACE,OAAD;AAA7B,SAAP;AACD;;AAED,aAAO,KAAP;AACD,KAZD,CAYE,OAAOX,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAevB,UAAf,CAA0BM,KAA1B,EAAiC;AAC/BX,IAAAA,MAAM,CAACe,OAAP,CAAgB,sCAAqCJ,KAAM,EAA3D;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAEF;;AACA,aAAO,MAAMpB,OAAO,CAAC+C,GAAR,CAAYlC,KAAZ,CAAb;AACD,KAJD,CAIE,OAAOiB,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,WAAStB,WAAT,CAAqB+B,QAArB,EAA+B;AAC7BA,IAAAA,QAAQ,CAACS,OAAT,CAAiBP,OAAO,IAAI;AAC1BvC,MAAAA,MAAM,CAACsB,KAAP,CAAc,eAAciB,OAAO,CAACC,UAAR,CAAmBI,aAAc,EAA7D;AACA9C,MAAAA,OAAO,CAACiD,GAAR,CAAYR,OAAZ;AACD,KAHD;AAID;;AAED,WAAShC,YAAT,CAAsB8B,QAAtB,EAAgC;AAC9BA,IAAAA,QAAQ,CAACS,OAAT,CAAiBP,OAAO,IAAI;AAC1BvC,MAAAA,MAAM,CAACsB,KAAP,CAAc,gBAAeiB,OAAO,CAACC,UAAR,CAAmBI,aAAc,EAA9D;AACA9C,MAAAA,OAAO,CAAC4C,IAAR,CAAaH,OAAb;AACD,KAHD;AAID;;AAED,iBAAe/B,WAAf,CAA2B;AAACG,IAAAA,KAAD;AAAQiC,IAAAA,aAAR;AAAuBb,IAAAA,OAAvB;AAAgCiB,IAAAA;AAAhC,GAA3B,EAAkE;AAChEhD,IAAAA,MAAM,CAACe,OAAP,CAAgB,oBAAmB6B,aAAc,cAAajC,KAAM,EAApE;;AACA,QAAI;AACFX,MAAAA,MAAM,CAACsB,KAAP,CAAc,SAAQX,KAAM,EAA5B;AACAX,MAAAA,MAAM,CAACsB,KAAP,CAAc,iBAAgBsB,aAAc,EAA5C;AACA5C,MAAAA,MAAM,CAACsB,KAAP,CAAc,QAAOW,IAAI,CAACC,SAAL,CAAec,IAAf,CAAqB,EAA1C;AACAhD,MAAAA,MAAM,CAACsB,KAAP,CAAc,WAAUW,IAAI,CAACC,SAAL,CAAeH,OAAf,CAAwB,EAAhD;AAEA,YAAMjC,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN;AAEApB,MAAAA,OAAO,CAACU,WAAR,CACEG,KADF,EAEEsC,MAAM,CAACC,IAAP,CAAYjB,IAAI,CAACC,SAAL,CAAe;AAACc,QAAAA;AAAD,OAAf,CAAZ,CAFF,EAGE;AACEJ,QAAAA,aADF;AAEEO,QAAAA,UAAU,EAAE,IAFd;AAGEpB,QAAAA;AAHF,OAHF;AAUA/B,MAAAA,MAAM,CAACsB,KAAP,CAAc,yBAAwBX,KAAM,EAA5C;AACD,KAnBD,CAmBE,OAAOiB,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAenB,WAAf,CAA2BE,KAA3B,EAAkC;AAChCX,IAAAA,MAAM,CAACe,OAAP,CAAgB,kBAAiBJ,KAAM,GAAvC;AACA,UAAMb,OAAO,CAACsD,WAAR,CAAoBzC,KAApB,CAAN;AACD,GA1PsC,CA4PvC;AACA;AACA;;;AAEA,WAASD,iBAAT,CAA2B2B,QAA3B,EAAqC;AACnCrC,IAAAA,MAAM,CAACe,OAAP,CAAgB,qBAAoBsB,QAAQ,CAACgB,MAAO,cAApD;AAEA,WAAOhB,QAAQ,CAACiB,GAAT,CAAaf,OAAO,IAAI;AAC7B,YAAMgB,OAAO,GAAGtB,IAAI,CAACuB,KAAL,CAAWjB,OAAO,CAACgB,OAAR,CAAgBE,QAAhB,EAAX,CAAhB;AACA,aAAO,IAAIC,sBAAJ,CAAeH,OAAO,CAACP,IAAvB,CAAP;AACD,KAHM,CAAP;AAID;;AAED,iBAAelB,OAAf,CAAuBnB,KAAvB,EAA8B;AAC5BX,IAAAA,MAAM,CAACe,OAAP,CAAgB,2BAA0BJ,KAAM,EAAhD;;AACA,QAAI;AACF,YAAM;AAACU,QAAAA;AAAD,UAAiB,MAAMvB,OAAO,CAACG,UAAR,CAAmBU,KAAnB,CAA7B;AACAX,MAAAA,MAAM,CAACsB,KAAP,CAAc,YAAWD,YAAa,sBAAqBV,KAAM,EAAjE;AACA,YAAMgD,aAAa,GAAGtC,YAAY,IAAIuC,qBAAhB,GAA6BA,qBAA7B,GAA0CvC,YAAhE;AACArB,MAAAA,MAAM,CAACsB,KAAP,CAAc,WAAUqC,aAAc,wBAAuBhD,KAAM,EAAnE;AAEA,YAAM0B,QAAQ,GAAG,MAAMwB,IAAI,CAACF,aAAD,CAA3B;AAEA3D,MAAAA,MAAM,CAACoB,KAAP,CAAc,aAAYiB,QAAQ,CAACgB,MAAO,kBAA1C;AAEA,aAAOhB,QAAP;AACD,KAXD,CAWE,OAAOT,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;;AAED,mBAAeiC,IAAf,CAAoBC,KAApB,EAA2BC,OAAO,GAAG,EAArC,EAAyCC,WAAW,GAAG,EAAvD,EAA2D;AACzD,UAAIF,KAAK,KAAK,CAAd,EAAiB;AACf,eAAOC,OAAP;AACD;;AAED,YAAMxB,OAAO,GAAG,MAAMzC,OAAO,CAAC+C,GAAR,CAAYlC,KAAZ,CAAtB;AACA,YAAMsD,UAAU,GAAG;AACjBrB,QAAAA,aAAa,EAAEL,OAAO,CAACC,UAAR,CAAmBI,aADjB;AAEjBsB,QAAAA,WAAW,EAAE3B,OAAO,CAAC4B,MAAR,CAAeD;AAFX,OAAnB,CANyD,CAUzD;;AACA,UAAIF,WAAW,CAACI,QAAZ,CAAqBH,UAArB,CAAJ,EAAsC;AACpC,eAAOJ,IAAI,CAACC,KAAK,GAAG,CAAT,EAAYC,OAAZ,EAAqBC,WAArB,CAAX;AACD;;AAED,aAAOH,IAAI,CAACC,KAAK,GAAG,CAAT,EAAYC,OAAO,CAACM,MAAR,CAAe9B,OAAf,CAAZ,EAAqCyB,WAAW,CAACK,MAAZ,CAAmBJ,UAAnB,CAArC,CAAX;AACD;AACF;;AAED,WAASjC,aAAT,CAAuBgB,IAAvB,EAA6B;AAC3B,WAAOA,IAAI,CAACR,UAAL,CAAgBT,OAAvB;AACD;AACF","sourcesContent":["/* eslint-disable max-statements */\n/* eslint-disable max-lines */\n/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport amqplib from 'amqplib';\nimport {MarcRecord} from '@natlibfi/marc-record';\nimport {Error as ApiError} from '@natlibfi/melinda-commons';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {CHUNK_SIZE} from './constants';\nimport {logError} from './utils';\nimport httpStatus from 'http-status';\n\nexport default async function (AMQP_URL) {\n  const connection = await amqplib.connect(AMQP_URL);\n  const channel = await connection.createChannel();\n  const logger = createLogger();\n\n  return {checkQueue, consumeChunk, consumeRawChunk, consumeOne, consumeRaw, ackMessages, nackMessages, sendToQueue, removeQueue, messagesToRecords};\n\n  async function checkQueue(queue, style = 'basic', purge = false, toRecord = true) {\n    logger.verbose(`checkQueue: ${queue}, Style: ${style}: Purge: ${purge}, toRecord: ${toRecord}`);\n    try {\n      const channelInfo = await channel.assertQueue(queue, {durable: true});\n      if (purge) {\n        await purgeQueue(purge);\n        logger.debug(`Queue ${queue} has purged ${channelInfo.messageCount} messages`);\n        return checkQueue(queue, style);\n      }\n\n      if (channelInfo.messageCount < 1) {\n        logger.debug(`checkQueue: ${channelInfo.messageCount} - ${queue} is empty`);\n        return false;\n      }\n      logger.silly(`Queue ${queue} has ${channelInfo.messageCount} messages`);\n\n      if (style === 'messages') {\n        return channelInfo.messageCount;\n      }\n\n      // Note: returns a message + a record\n      if (style === 'one') {\n        return consumeOne(queue);\n      }\n\n      // Note: returns just a message\n      if (style === 'raw') {\n        return consumeRaw(queue);\n      }\n\n      // Note: returns a chunk of messages + records\n      if (style === 'rawChunk') {\n        return consumeRawChunk(queue);\n      }\n\n      // Note: returns a chunk of messages or a chunk of messages + records (depending on toRecord)\n      if ((/^.{8}-.{4}-.{4}-.{4}-.{12}$/u).test(style)) {\n        logger.silly(`checkQueue: regexp match ${style}`);\n        return consumeByCorrelationId(queue, style, toRecord);\n      }\n\n      // Note: returns a chunk of messages + records\n      if (style === 'basic') {\n        return consumeChunk(queue);\n      }\n\n      // Defaults:\n      throw new ApiError(httpStatus.UNPROCESSABLE_ENTITY);\n    } catch (error) {\n      logError(error);\n    }\n\n    function purgeQueue(purge) {\n      if (purge) {\n        return channel.purgeQueue(queue);\n      }\n    }\n  }\n\n  async function consumeChunk(queue) {\n    logger.verbose(`Prepared to consumeChunk from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n\n      // getData: next chunk (100) messages\n      const queMessages = await getData(queue);\n\n      const headers = getHeaderInfo(queMessages[0]);\n      logger.debug(`Filtering messages by cataloger in ${JSON.stringify(headers)}`);\n\n      let filterIn = 0; // eslint-disable-line functional/no-let\n      let filterOut = 0; // eslint-disable-line functional/no-let\n\n      // Check that cataloger match! headers\n      const messages = queMessages.filter(message => {\n        if (message.properties.headers.cataloger === headers.cataloger) {\n          filterIn++; // eslint-disable-line no-plusplus\n          return true;\n        }\n\n        // Nack unwanted ones\n        channel.nack(message, false, true);\n        filterOut++; // eslint-disable-line no-plusplus\n        return false;\n      });\n\n      logger.debug(`Filtering result: valid: ${filterIn} non-valid: ${filterOut}`);\n      const records = await messagesToRecords(messages);\n\n      return {headers, records, messages};\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeByCorrelationId(queue, correlationId, toRecord = true) {\n    logger.verbose(`Prepared to consumeByCorrelationId from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n      // get next chunk (100) messages\n      const queMessages = await getData(queue);\n\n      logger.debug(`Filtering messages by ${correlationId}`);\n\n      let filterIn = 0; // eslint-disable-line functional/no-let\n      let filterOut = 0; // eslint-disable-line functional/no-let\n\n      // Check that correlationId matches query\n      const messages = queMessages.filter(message => {\n        if (message.properties.correlationId === correlationId) {\n          filterIn++; // eslint-disable-line no-plusplus\n          return true;\n        }\n\n        // Nack unwanted ones\n        channel.nack(message, false, true);\n        filterOut++; // eslint-disable-line no-plusplus\n        return false;\n      });\n      logger.debug(`Filtering result: valid: ${filterIn} non-valid: ${filterOut}`);\n\n      const headers = getHeaderInfo(messages[0]);\n\n      if (toRecord) {\n        const records = await messagesToRecords(messages);\n\n        return {headers, records, messages};\n      }\n      return {headers, messages};\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeRawChunk(queue) {\n    logger.verbose(`Prepared to consumeRawChunk from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n      // Get next chunk (100) messages\n      const queMessages = await getData(queue);\n\n      const headers = getHeaderInfo(queMessages[0]);\n      logger.debug(`Filtering messages by ${JSON.stringify(headers)}`);\n\n      let filterIn = 0; // eslint-disable-line functional/no-let\n      let filterOut = 0; // eslint-disable-line functional/no-let\n\n      // Check that cataloger match! headers\n      const messages = queMessages.filter(message => {\n        if (message.properties.headers.cataloger === headers.cataloger) {\n          filterIn++; // eslint-disable-line no-plusplus\n\n          return true;\n        }\n\n        // Nack unwanted ones\n        channel.nack(message, false, true);\n        filterOut++; // eslint-disable-line no-plusplus\n\n        return false;\n      });\n      logger.debug(`Filtering result: valid: ${filterIn} non-valid: ${filterOut}`);\n\n      // return raw (do not convert messages to records)\n      return {headers, messages};\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeOne(queue) {\n    logger.verbose(`Prepared to consumeOne from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n\n      // Returns false if 0 items in queue\n      const message = await channel.get(queue);\n      if (message) {\n        const headers = getHeaderInfo(message);\n        const records = messagesToRecords([message]);\n        return {headers, records, messages: [message]};\n      }\n\n      return false;\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeRaw(queue) {\n    logger.verbose(`Prepared to consumeRaw from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n      // Returns false if 0 items in queue\n      return await channel.get(queue);\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  function ackMessages(messages) {\n    messages.forEach(message => {\n      logger.silly(`Ack message ${message.properties.correlationId}`);\n      channel.ack(message);\n    });\n  }\n\n  function nackMessages(messages) {\n    messages.forEach(message => {\n      logger.silly(`Nack message ${message.properties.correlationId}`);\n      channel.nack(message);\n    });\n  }\n\n  async function sendToQueue({queue, correlationId, headers, data}) {\n    logger.verbose(`Send message for ${correlationId} to queue: ${queue}`);\n    try {\n      logger.silly(`Queue ${queue}`);\n      logger.silly(`CorrelationId ${correlationId}`);\n      logger.silly(`Data ${JSON.stringify(data)}`);\n      logger.silly(`Headers ${JSON.stringify(headers)}`);\n\n      await channel.assertQueue(queue, {durable: true});\n\n      channel.sendToQueue(\n        queue,\n        Buffer.from(JSON.stringify({data})),\n        {\n          correlationId,\n          persistent: true,\n          headers\n        }\n      );\n\n      logger.silly(`Message send to queue ${queue}`);\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function removeQueue(queue) {\n    logger.verbose(`Removing queue ${queue}.`);\n    await channel.deleteQueue(queue);\n  }\n\n  // ----------------\n  // Helper functions\n  // ----------------\n\n  function messagesToRecords(messages) {\n    logger.verbose(`Parsing messages (${messages.length}) to records`);\n\n    return messages.map(message => {\n      const content = JSON.parse(message.content.toString());\n      return new MarcRecord(content.data);\n    });\n  }\n\n  async function getData(queue) {\n    logger.verbose(`Getting queue data from ${queue}`);\n    try {\n      const {messageCount} = await channel.checkQueue(queue);\n      logger.silly(`There is ${messageCount} messages in queue ${queue}`);\n      const messagesToGet = messageCount >= CHUNK_SIZE ? CHUNK_SIZE : messageCount;\n      logger.silly(`Getting ${messagesToGet} messages from queue ${queue}`);\n\n      const messages = await pump(messagesToGet);\n\n      logger.debug(`Returning ${messages.length} unique messages`);\n\n      return messages;\n    } catch (error) {\n      logError(error);\n    }\n\n    async function pump(count, results = [], identifiers = []) {\n      if (count === 0) {\n        return results;\n      }\n\n      const message = await channel.get(queue);\n      const identifier = {\n        correlationId: message.properties.correlationId,\n        deliveryTag: message.fields.deliveryTag\n      };\n      // Filter not unique messages\n      if (identifiers.includes(identifier)) {\n        return pump(count - 1, results, identifiers);\n      }\n\n      return pump(count - 1, results.concat(message), identifiers.concat(identifier));\n    }\n  }\n\n  function getHeaderInfo(data) {\n    return data.properties.headers;\n  }\n}\n"],"file":"amqp.js"}

package/dist/constants.js

@@ -3,7 +3,7 @@
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-exports.conversionFormats = exports.CHUNK_SIZE = exports.OPERATIONS = exports.QUEUE_ITEM_STATE = void 0;
+exports.conversionFormats = exports.QUEUE_ITEM_STATE = exports.OPERATIONS = exports.CHUNK_SIZE = void 0;
 
 /**
 *

package/dist/helpers/conversion.js

@@ -19,6 +19,8 @@ var _utils = require("../utils");
 
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
+/* eslint-disable max-statements */
+
 /**
 *
 * @licstart  The following is the entire license notice for the JavaScript code in this file.

package/dist/helpers/conversion.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/helpers/conversion.js"],"names":["logger","serialize","unserialize","record","format","verbose","conversionFormats","MARCXML","to","ISO2709","JSON","Json","ConversionError","httpStatus","UNSUPPORTED_MEDIA_TYPE","err","BAD_REQUEST","data","validationOptions","subfieldValues","silly","stringify","from"],"mappings":";;;;;;;AA4BA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AAjCA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AASe,oBAAY;AACzB,QAAMA,MAAM,GAAG,0CAAf;AAEA,SAAO;AAACC,IAAAA,SAAD;AAAYC,IAAAA;AAAZ,GAAP;;AAEA,WAASD,SAAT,CAAmBE,MAAnB,EAA2BC,MAA3B,EAAmC;AACjCJ,IAAAA,MAAM,CAACK,OAAP,CAAe,oBAAf;;AACA,QAAI;AACF,UAAID,MAAM,KAAKE,6BAAkBC,OAAjC,EAA0C;AACxC,eAAOA,+BAAQC,EAAR,CAAWL,MAAX,CAAP;AACD;;AAED,UAAIC,MAAM,KAAKE,6BAAkBG,OAAjC,EAA0C;AACxC,eAAOA,+BAAQD,EAAR,CAAWL,MAAX,CAAP;AACD;;AAED,UAAIC,MAAM,KAAKE,6BAAkBI,IAAjC,EAAuC;AACrC,eAAOC,4BAAKH,EAAL,CAAQL,MAAR,CAAP;AACD;;AAED,YAAM,IAAIS,qBAAJ,CAAoBC,oBAAWC,sBAA/B,CAAN;AACD,KAdD,CAcE,OAAOC,GAAP,EAAY;AACZ,2BAASA,GAAT;;AACA,UAAIA,GAAG,YAAYH,qBAAnB,EAAoC;AAAE;AACpC,cAAMG,GAAN;AACD;;AACD,YAAM,IAAIH,qBAAJ,CAAoBC,oBAAWG,WAA/B,EAA4C,gCAA5C,CAAN;AACD;AACF;;AAED,WAASd,WAAT,CAAqBe,IAArB,EAA2Bb,MAA3B,EAAmCc,iBAAiB,GAAG;AAACC,IAAAA,cAAc,EAAE;AAAjB,GAAvD,EAAgF;AAC9EnB,IAAAA,MAAM,CAACK,OAAP,CAAe,sBAAf;AACAL,IAAAA,MAAM,CAACoB,KAAP,CAAc,WAAUhB,MAAO,EAA/B;AACAJ,IAAAA,MAAM,CAACoB,KAAP,CAAc,uBAAsBV,IAAI,CAACW,SAAL,CAAeH,iBAAf,CAAkC,EAAtE;AACAlB,IAAAA,MAAM,CAACoB,KAAP,CAAc,SAAQV,IAAI,CAACW,SAAL,CAAeJ,IAAf,CAAqB,EAA3C;;AACA,QAAI;AACF,UAAIb,MAAM,KAAKE,6BAAkBC,OAAjC,EAA0C;AACxCP,QAAAA,MAAM,CAACK,OAAP,CAAe,4BAAf;AACA,eAAOE,+BAAQe,IAAR,CAAaL,IAAb,EAAmBC,iBAAnB,CAAP;AACD;;AAED,UAAId,MAAM,KAAKE,6BAAkBG,OAAjC,EAA0C;AACxCT,QAAAA,MAAM,CAACK,OAAP,CAAe,4BAAf;AACA,eAAOI,+BAAQa,IAAR,CAAaL,IAAb,EAAmBC,iBAAnB,CAAP;AACD;;AAED,UAAId,MAAM,KAAKE,6BAAkBI,IAAjC,EAAuC;AACrCV,QAAAA,MAAM,CAACK,OAAP,CAAe,yBAAf;AACA,eAAOM,4BAAKW,IAAL,CAAUL,IAAV,EAAgBC,iBAAhB,CAAP;AACD;;AAED,YAAM,IAAIN,qBAAJ,CAAoBC,oBAAWC,sBAA/B,CAAN;AACD,KAjBD,CAiBE,OAAOC,GAAP,EAAY;AACZ,2BAASA,GAAT;;AACA,UAAIA,GAAG,YAAYH,qBAAnB,EAAoC;AAAE;AACpC,cAAMG,GAAN;AACD;;AACD,YAAM,IAAIH,qBAAJ,CAAoBC,oBAAWG,WAA/B,EAA4C,kCAA5C,CAAN;AACD;AACF;AACF","sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport httpStatus from 'http-status';\nimport {MARCXML, ISO2709, Json} from '@natlibfi/marc-record-serializers';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {Error as ConversionError} from '@natlibfi/melinda-commons';\nimport {conversionFormats} from '../constants';\nimport {logError} from '../utils';\n\nexport default function () {\n  const logger = createLogger();\n\n  return {serialize, unserialize};\n\n  function serialize(record, format) {\n    logger.verbose('Serializing record');\n    try {\n      if (format === conversionFormats.MARCXML) {\n        return MARCXML.to(record);\n      }\n\n      if (format === conversionFormats.ISO2709) {\n        return ISO2709.to(record);\n      }\n\n      if (format === conversionFormats.JSON) {\n        return Json.to(record);\n      }\n\n      throw new ConversionError(httpStatus.UNSUPPORTED_MEDIA_TYPE);\n    } catch (err) {\n      logError(err);\n      if (err instanceof ConversionError) { // eslint-disable-line functional/no-conditional-statement\n        throw err;\n      }\n      throw new ConversionError(httpStatus.BAD_REQUEST, 'Error while serializing record');\n    }\n  }\n\n  function unserialize(data, format, validationOptions = {subfieldValues: false}) {\n    logger.verbose('Unserializing record');\n    logger.silly(`Format: ${format}`);\n    logger.silly(`Validation options: ${JSON.stringify(validationOptions)}`);\n    logger.silly(`Data: ${JSON.stringify(data)}`);\n    try {\n      if (format === conversionFormats.MARCXML) {\n        logger.verbose('Unserialize format marcxml');\n        return MARCXML.from(data, validationOptions);\n      }\n\n      if (format === conversionFormats.ISO2709) {\n        logger.verbose('Unserialize format iso2709');\n        return ISO2709.from(data, validationOptions);\n      }\n\n      if (format === conversionFormats.JSON) {\n        logger.verbose('Unserialize format json');\n        return Json.from(data, validationOptions);\n      }\n\n      throw new ConversionError(httpStatus.UNSUPPORTED_MEDIA_TYPE);\n    } catch (err) {\n      logError(err);\n      if (err instanceof ConversionError) { // eslint-disable-line functional/no-conditional-statement\n        throw err;\n      }\n      throw new ConversionError(httpStatus.BAD_REQUEST, 'Error while unserializing record');\n    }\n  }\n}\n"],"file":"conversion.js"}
+{"version":3,"sources":["../../src/helpers/conversion.js"],"names":["logger","serialize","unserialize","record","format","verbose","conversionFormats","MARCXML","to","ISO2709","JSON","Json","ConversionError","httpStatus","UNSUPPORTED_MEDIA_TYPE","err","BAD_REQUEST","data","validationOptions","subfieldValues","silly","stringify","from"],"mappings":";;;;;;;AA6BA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AAlCA;;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AASe,oBAAY;AACzB,QAAMA,MAAM,GAAG,0CAAf;AAEA,SAAO;AAACC,IAAAA,SAAD;AAAYC,IAAAA;AAAZ,GAAP;;AAEA,WAASD,SAAT,CAAmBE,MAAnB,EAA2BC,MAA3B,EAAmC;AACjCJ,IAAAA,MAAM,CAACK,OAAP,CAAe,oBAAf;;AACA,QAAI;AACF,UAAID,MAAM,KAAKE,6BAAkBC,OAAjC,EAA0C;AACxC,eAAOA,+BAAQC,EAAR,CAAWL,MAAX,CAAP;AACD;;AAED,UAAIC,MAAM,KAAKE,6BAAkBG,OAAjC,EAA0C;AACxC,eAAOA,+BAAQD,EAAR,CAAWL,MAAX,CAAP;AACD;;AAED,UAAIC,MAAM,KAAKE,6BAAkBI,IAAjC,EAAuC;AACrC,eAAOC,4BAAKH,EAAL,CAAQL,MAAR,CAAP;AACD;;AAED,YAAM,IAAIS,qBAAJ,CAAoBC,oBAAWC,sBAA/B,CAAN;AACD,KAdD,CAcE,OAAOC,GAAP,EAAY;AACZ,2BAASA,GAAT;;AACA,UAAIA,GAAG,YAAYH,qBAAnB,EAAoC;AAAE;AACpC,cAAMG,GAAN;AACD;;AACD,YAAM,IAAIH,qBAAJ,CAAoBC,oBAAWG,WAA/B,EAA4C,gCAA5C,CAAN;AACD;AACF;;AAED,WAASd,WAAT,CAAqBe,IAArB,EAA2Bb,MAA3B,EAAmCc,iBAAiB,GAAG;AAACC,IAAAA,cAAc,EAAE;AAAjB,GAAvD,EAAgF;AAC9EnB,IAAAA,MAAM,CAACK,OAAP,CAAe,sBAAf;AACAL,IAAAA,MAAM,CAACoB,KAAP,CAAc,WAAUhB,MAAO,EAA/B;AACAJ,IAAAA,MAAM,CAACoB,KAAP,CAAc,uBAAsBV,IAAI,CAACW,SAAL,CAAeH,iBAAf,CAAkC,EAAtE;AACAlB,IAAAA,MAAM,CAACoB,KAAP,CAAc,SAAQV,IAAI,CAACW,SAAL,CAAeJ,IAAf,CAAqB,EAA3C;;AACA,QAAI;AACF,UAAIb,MAAM,KAAKE,6BAAkBC,OAAjC,EAA0C;AACxCP,QAAAA,MAAM,CAACK,OAAP,CAAe,4BAAf;AACA,eAAOE,+BAAQe,IAAR,CAAaL,IAAb,EAAmBC,iBAAnB,CAAP;AACD;;AAED,UAAId,MAAM,KAAKE,6BAAkBG,OAAjC,EAA0C;AACxCT,QAAAA,MAAM,CAACK,OAAP,CAAe,4BAAf;AACA,eAAOI,+BAAQa,IAAR,CAAaL,IAAb,EAAmBC,iBAAnB,CAAP;AACD;;AAED,UAAId,MAAM,KAAKE,6BAAkBI,IAAjC,EAAuC;AACrCV,QAAAA,MAAM,CAACK,OAAP,CAAe,yBAAf;AACA,eAAOM,4BAAKW,IAAL,CAAUL,IAAV,EAAgBC,iBAAhB,CAAP;AACD;;AAED,YAAM,IAAIN,qBAAJ,CAAoBC,oBAAWC,sBAA/B,CAAN;AACD,KAjBD,CAiBE,OAAOC,GAAP,EAAY;AACZ,2BAASA,GAAT;;AACA,UAAIA,GAAG,YAAYH,qBAAnB,EAAoC;AAAE;AACpC,cAAMG,GAAN;AACD;;AACD,YAAM,IAAIH,qBAAJ,CAAoBC,oBAAWG,WAA/B,EAA4C,kCAA5C,CAAN;AACD;AACF;AACF","sourcesContent":["/* eslint-disable max-statements */\n/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport httpStatus from 'http-status';\nimport {MARCXML, ISO2709, Json} from '@natlibfi/marc-record-serializers';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {Error as ConversionError} from '@natlibfi/melinda-commons';\nimport {conversionFormats} from '../constants';\nimport {logError} from '../utils';\n\nexport default function () {\n  const logger = createLogger();\n\n  return {serialize, unserialize};\n\n  function serialize(record, format) {\n    logger.verbose('Serializing record');\n    try {\n      if (format === conversionFormats.MARCXML) {\n        return MARCXML.to(record);\n      }\n\n      if (format === conversionFormats.ISO2709) {\n        return ISO2709.to(record);\n      }\n\n      if (format === conversionFormats.JSON) {\n        return Json.to(record);\n      }\n\n      throw new ConversionError(httpStatus.UNSUPPORTED_MEDIA_TYPE);\n    } catch (err) {\n      logError(err);\n      if (err instanceof ConversionError) { // eslint-disable-line functional/no-conditional-statement\n        throw err;\n      }\n      throw new ConversionError(httpStatus.BAD_REQUEST, 'Error while serializing record');\n    }\n  }\n\n  function unserialize(data, format, validationOptions = {subfieldValues: false}) {\n    logger.verbose('Unserializing record');\n    logger.silly(`Format: ${format}`);\n    logger.silly(`Validation options: ${JSON.stringify(validationOptions)}`);\n    logger.silly(`Data: ${JSON.stringify(data)}`);\n    try {\n      if (format === conversionFormats.MARCXML) {\n        logger.verbose('Unserialize format marcxml');\n        return MARCXML.from(data, validationOptions);\n      }\n\n      if (format === conversionFormats.ISO2709) {\n        logger.verbose('Unserialize format iso2709');\n        return ISO2709.from(data, validationOptions);\n      }\n\n      if (format === conversionFormats.JSON) {\n        logger.verbose('Unserialize format json');\n        return Json.from(data, validationOptions);\n      }\n\n      throw new ConversionError(httpStatus.UNSUPPORTED_MEDIA_TYPE);\n    } catch (err) {\n      logError(err);\n      if (err instanceof ConversionError) { // eslint-disable-line functional/no-conditional-statement\n        throw err;\n      }\n      throw new ConversionError(httpStatus.BAD_REQUEST, 'Error while unserializing record');\n    }\n  }\n}\n"],"file":"conversion.js"}

package/dist/helpers/format.js

@@ -3,8 +3,8 @@
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-exports.formatRecord = formatRecord;
 exports.BIB_FORMAT_SETTINGS = void 0;
+exports.formatRecord = formatRecord;
 
 var _marcRecord = require("@natlibfi/marc-record");
 

package/dist/index.js

@@ -11,12 +11,6 @@ var _exportNames = {
   format: true,
   logError: true
 };
-Object.defineProperty(exports, "mongoFactory", {
-  enumerable: true,
-  get: function () {
-    return _mongo.default;
-  }
-});
 Object.defineProperty(exports, "amqpFactory", {
   enumerable: true,
   get: function () {
@@ -29,19 +23,25 @@ Object.defineProperty(exports, "conversions", {
     return _conversion.default;
   }
 });
-Object.defineProperty(exports, "validations", {
+exports.format = void 0;
+Object.defineProperty(exports, "logError", {
   enumerable: true,
   get: function () {
-    return _validation.default;
+    return _utils.logError;
   }
 });
-Object.defineProperty(exports, "logError", {
+Object.defineProperty(exports, "mongoFactory", {
   enumerable: true,
   get: function () {
-    return _utils.logError;
+    return _mongo.default;
+  }
+});
+Object.defineProperty(exports, "validations", {
+  enumerable: true,
+  get: function () {
+    return _validation.default;
   }
 });
-exports.format = void 0;
 
 var _mongo = _interopRequireDefault(require("./mongo"));
 

package/dist/mongo.js

@@ -23,6 +23,8 @@ var _mongoSanitize = _interopRequireDefault(require("mongo-sanitize"));
 
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
+/* eslint-disable max-lines */
+
 /* eslint-disable max-statements */
 
 /**
@@ -97,14 +99,17 @@ async function _default(MONGO_URI, collection) {
     getStream,
     setState,
     pushIds,
-    pushId
+    pushMessages
   };
 
   async function createPrio({
     correlationId,
     cataloger,
     oCatalogerIn,
-    operation
+    operation,
+    noop = undefined,
+    unique = undefined,
+    prio = undefined
   }) {
     const time = (0, _moment.default)().toDate();
     const newQueueItem = {
@@ -112,16 +117,23 @@ async function _default(MONGO_URI, collection) {
       cataloger,
       operation,
       oCatalogerIn,
-      queueItemState: _constants.PRIO_QUEUE_ITEM_STATE.PENDING_VALIDATION,
+      operationSettings: {
+        unique,
+        noop,
+        prio
+      },
+      queueItemState: _constants.QUEUE_ITEM_STATE.VALIDATOR.PENDING_VALIDATION,
       creationTime: time,
       modificationTime: time,
-      handledId: ''
+      handledIds: [],
+      rejectedIds: []
     };
 
     try {
       const result = await db.collection(collection).insertOne(newQueueItem);
 
       if (result.acknowledged) {
+        logger.info(`New prio queue item for ${correlationId} has been made in ${collection}`);
         return;
       }
 
@@ -149,15 +161,17 @@ async function _default(MONGO_URI, collection) {
       operation,
       contentType,
       recordLoadParams,
-      queueItemState: _constants.QUEUE_ITEM_STATE.UPLOADING,
+      queueItemState: _constants.QUEUE_ITEM_STATE.VALIDATOR.UPLOADING,
       creationTime: time,
       modificationTime: time,
-      handledIds: []
+      handledIds: [],
+      rejectedIds: []
     };
 
     try {
+      // No await here, promises later
       db.collection(collection).insertOne(newQueueItem);
-      logger.log('info', 'New queue item has been made!');
+      logger.info(`New bulk queue item for ${correlationId} has been made in ${collection}!`);
     } catch (error) {
       (0, _utils.logError)(error);
       throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR);
@@ -169,18 +183,25 @@ async function _default(MONGO_URI, collection) {
         resolve(correlationId);
       }));
     });
-  }
+  } // Check state that the queueItem has not waited too long and set state
+
 
   async function checkAndSetState({
     correlationId,
-    state
+    state,
+    errorMessage = '',
+    errorStatus = ''
   }) {
+    // checkTimeOut returns true, if queueItem is fresher than 1 minute
+    // otherwise it sets queueItem to state ABORT (408, 'Timeout')
     const timeOut = await checkTimeOut(correlationId);
 
     if (timeOut) {
       return setState({
         correlationId,
-        state
+        state,
+        errorMessage,
+        errorStatus
       });
     }
 
@@ -193,7 +214,7 @@ async function _default(MONGO_URI, collection) {
         _id: 0
       }
     }).toArray();
-    logger.info(`Query result: ${result.length > 0 ? 'Found!' : 'Not found!'}`);
+    logger.debug(`Query result: ${result.length > 0 ? 'Found!' : 'Not found!'}`);
     return result;
   }
 
@@ -213,7 +234,10 @@ async function _default(MONGO_URI, collection) {
     }
 
     return result;
-  }
+  } // Check that if the item has waited too long
+  // If the last modification time for the queueItem is older than 1 minute
+  // set state to ABORT and return false, otherwise return true
+
 
   async function checkTimeOut(correlationId) {
     const result = await db.collection(collection).findOne({
@@ -223,9 +247,12 @@ async function _default(MONGO_URI, collection) {
     logger.silly(`timeOut @ ${timeoutTime}`);
 
     if (timeoutTime.isBefore()) {
+      const oldState = result.queueItemState;
       await setState({
         correlationId,
-        state: _constants.PRIO_QUEUE_ITEM_STATE.ABORT
+        state: _constants.QUEUE_ITEM_STATE.ABORT,
+        errorStatus: _httpStatus.default.REQUEST_TIMEOUT,
+        errorMessage: `Timeout in ${oldState}`
       });
       return false;
     }
@@ -235,14 +262,13 @@ async function _default(MONGO_URI, collection) {
 
   async function remove(params) {
     logger.debug(`${JSON.stringify(params)}`);
-    logger.info(`Removing from Mongo id: ${params.correlationId}`);
+    logger.info(`Removing from Mongo (${collection}) id: ${params.correlationId}`);
     const clean = (0, _mongoSanitize.default)(params.correlationId);
+    logger.silly(`mongo/remove: clean: ${JSON.stringify(clean)}`);
 
     try {
-      await getFileMetadata({
-        gridFSBucket,
-        filename: clean
-      });
+      //const metadataResult = await getFileMetadata({filename: clean});
+      //logger.debug(`mongo/remove: metadataResult: ${JSON.stringify(metadataResult)}`);
       const noContent = await removeContent(params);
 
       if (noContent) {
@@ -252,30 +278,32 @@ async function _default(MONGO_URI, collection) {
         return true;
       }
     } catch (err) {
-      if (err.status === _httpStatus.default.NOT_FOUND) {
-        await db.collection(collection).deleteOne({
-          correlationId: clean
-        });
-        return true;
+      if (err instanceof _mongodb.MongoDriverError) {
+        logger.error(err.message);
+
+        if (err.message.indexOf('File not found for id') !== -1) {
+          logger.debug(`mongo/remove: File not found, removing queueItem ${JSON.stringify(clean)} from ${collection}`);
+          await db.collection(collection).deleteOne({
+            correlationId: clean
+          });
+          return true;
+        }
+
+        throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR, err.message);
       }
 
-      throw err;
+      throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR, err.message);
     }
   }
 
   async function readContent(correlationId) {
-    logger.info(`Reading content for id: ${correlationId}`);
+    logger.info(`Reading content for id: ${correlationId} in ${collection}`);
     const clean = (0, _mongoSanitize.default)(correlationId);
     const result = await db.collection(collection).findOne({
       correlationId: clean
-    }); //ignore: node_nosqli_injection
+    }); // njsscan-ignore: node_nosqli_injection
 
     if (result) {
-      // Check if the file exists
-      await getFileMetadata({
-        gridFSBucket,
-        filename: clean
-      });
       return {
         contentType: result.contentType,
         readStream: gridFSBucket.openDownloadStreamByName(clean)
@@ -286,22 +314,20 @@ async function _default(MONGO_URI, collection) {
   }
 
   async function removeContent(params) {
-    logger.info(`Removing content for id: ${params.correlationId}`);
+    logger.info(`Removing content for id: ${params.correlationId} in ${collection}`);
     const clean = (0, _mongoSanitize.default)(params.correlationId);
     const result = await db.collection(collection).findOne({
       correlationId: clean
-    }); //ignore: node_nosqli_injection
+    }); // njsscan-ignore: node_nosqli_injection
+
+    logger.debug(`mongo/removeContent: result ${JSON.stringify(result)}`);
 
     if (result) {
-      const {
-        _id: fileId
-      } = await getFileMetadata({
-        gridFSBucket,
-        filename: clean
-      });
-      await gridFSBucket.delete(fileId);
+      await gridFSBucket.delete(clean);
       return true;
     }
+
+    return true;
   }
 
   function getOne({
@@ -314,34 +340,29 @@ async function _default(MONGO_URI, collection) {
 
     try {
       if (operation === undefined) {
-        logger.log('silly', `Checking DB for ${JSON.stringify(clean.queueItemState)}`);
+        logger.silly(`Checking DB ${collection} for ${JSON.stringify(clean.queueItemState)}`);
         return db.collection(collection).findOne({ ...clean
-        }); //ignore: node_nosqli_injection
+        });
       }
 
       const clean2 = {
         operation: (0, _mongoSanitize.default)(operation)
       };
-      logger.log('silly', `Checking DB for ${clean.queueItemState} + ${clean2.operation}`);
+      logger.silly(`Checking DB ${collection} for ${clean.queueItemState} + ${clean2.operation}`);
       return db.collection(collection).findOne({ ...clean,
         ...clean2
-      }); //ignore: node_nosqli_injection
+      });
     } catch (error) {
       (0, _utils.logError)(error);
     }
   }
 
-  async function getStream(correlationId) {
-    logger.info(`Forming stream from db: ${correlationId}`);
+  function getStream(correlationId) {
+    logger.info(`Forming stream from db: ${correlationId} in ${collection}`);
     const clean = (0, _mongoSanitize.default)(correlationId);
 
     try {
-      // Check that content is there
-      await getFileMetadata({
-        gridFSBucket,
-        filename: clean
-      }); // Return content stream
-
+      // Return content stream
       return gridFSBucket.openDownloadStreamByName(clean);
     } catch (error) {
       (0, _utils.logError)(error);
@@ -350,9 +371,11 @@ async function _default(MONGO_URI, collection) {
 
   async function pushIds({
     correlationId,
-    ids
+    handledIds,
+    rejectedIds
   }) {
-    logger.debug(`Push queue-item ids to list: ${correlationId}, ${ids}`);
+    logger.info(`Push queue-item ids to list: ${correlationId} to ${collection}`);
+    logger.debug(`ids: ${JSON.stringify(handledIds)}, rejectedIds: ${JSON.stringify(rejectedIds)}`);
     const clean = (0, _mongoSanitize.default)(correlationId);
     await db.collection(collection).updateOne({
       correlationId: clean
@@ -362,24 +385,34 @@ async function _default(MONGO_URI, collection) {
       },
       $push: {
         handledIds: {
-          $each: ids
+          $each: handledIds
+        },
+        rejectedIds: {
+          $each: rejectedIds
         }
       }
     });
   }
 
-  async function pushId({
+  async function pushMessages({
     correlationId,
-    id
+    messages,
+    messageField = 'messages'
   }) {
-    logger.debug(`Push queue-item id: ${correlationId}, ${id}`);
+    logger.info(`Push messages to ${messageField} ${correlationId} to ${collection}`);
+    logger.debug(`Messages: ${JSON.stringify(messages)}}`);
     const clean = (0, _mongoSanitize.default)(correlationId);
+    const cleanMessageField = (0, _mongoSanitize.default)(messageField);
     await db.collection(collection).updateOne({
       correlationId: clean
     }, {
       $set: {
-        modificationTime: (0, _moment.default)().toDate(),
-        handledId: id
+        modificationTime: (0, _moment.default)().toDate()
+      },
+      $push: {
+        [cleanMessageField]: {
+          $each: messages
+        }
       }
     });
   }
@@ -387,9 +420,11 @@ async function _default(MONGO_URI, collection) {
   function setState({
     correlationId,
     state,
-    errorMessage = ''
+    errorMessage = '',
+    errorStatus = ''
   }) {
-    logger.log('info', `Setting queue-item state: ${correlationId}, ${state}, ${errorMessage}`);
+    const errorString = errorMessage || errorStatus ? `, Error message: '${errorMessage}', Error status: '${errorStatus}'` : '';
+    logger.log('info', `Setting queue-item state: ${correlationId}${state}${errorString} to ${collection}`);
     const clean = (0, _mongoSanitize.default)(correlationId);
     return db.collection(collection).findOneAndUpdate({
       correlationId: clean
@@ -397,7 +432,8 @@ async function _default(MONGO_URI, collection) {
       $set: {
         queueItemState: state,
         modificationTime: (0, _moment.default)().toDate(),
-        errorMessage
+        errorMessage,
+        errorStatus
       }
     }, {
       projection: {
@@ -406,16 +442,5 @@ async function _default(MONGO_URI, collection) {
       returnNewDocument: true
     });
   }
-
-  function getFileMetadata({
-    gridFSBucket,
-    filename
-  }) {
-    return new Promise((resolve, reject) => {
-      gridFSBucket.find({
-        filename
-      }).on('error', reject).on('data', resolve).on('end', () => reject(new _melindaCommons.Error(_httpStatus.default.NOT_FOUND, 'No content')));
-    });
-  }
 }
 //# sourceMappingURL=mongo.js.map

package/dist/mongo.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/mongo.js"],"names":["MONGO_URI","collection","logger","client","MongoClient","connect","useNewUrlParser","useUnifiedTopology","db","gridFSBucket","GridFSBucket","bucketName","createPrio","createBulk","checkAndSetState","query","queryById","remove","readContent","removeContent","getOne","getStream","setState","pushIds","pushId","correlationId","cataloger","oCatalogerIn","operation","time","toDate","newQueueItem","queueItemState","PRIO_QUEUE_ITEM_STATE","PENDING_VALIDATION","creationTime","modificationTime","handledId","result","insertOne","acknowledged","ApiError","httpStatus","INTERNAL_SERVER_ERROR","error","contentType","recordLoadParams","stream","QUEUE_ITEM_STATE","UPLOADING","handledIds","log","Promise","resolve","reject","outputStream","openUploadStream","on","chunk","write","end","undefined","state","timeOut","checkTimeOut","params","find","projection","_id","toArray","info","length","checkModTime","findOne","timeoutTime","add","silly","isBefore","ABORT","debug","JSON","stringify","clean","getFileMetadata","filename","noContent","deleteOne","err","status","NOT_FOUND","readStream","openDownloadStreamByName","fileId","delete","clean2","ids","updateOne","$set","$push","$each","id","errorMessage","findOneAndUpdate","returnNewDocument"],"mappings":";;;;;;;AA6BA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AApCA;;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAWA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAEe,wBAAgBA,SAAhB,EAA2BC,UAA3B,EAAuC;AACpD,QAAMC,MAAM,GAAG,0CAAf,CADoD,CAGpD;;AACA,QAAMC,MAAM,GAAG,MAAMC,qBAAYC,OAAZ,CAAoBL,SAApB,EAA+B;AAACM,IAAAA,eAAe,EAAE,IAAlB;AAAwBC,IAAAA,kBAAkB,EAAE;AAA5C,GAA/B,CAArB;AACA,QAAMC,EAAE,GAAGL,MAAM,CAACK,EAAP,CAAU,UAAV,CAAX;AACA,QAAMC,YAAY,GAAG,IAAIC,qBAAJ,CAAiBF,EAAjB,EAAqB;AAACG,IAAAA,UAAU,EAAEV;AAAb,GAArB,CAArB;AAEA,SAAO;AAACW,IAAAA,UAAD;AAAaC,IAAAA,UAAb;AAAyBC,IAAAA,gBAAzB;AAA2CC,IAAAA,KAA3C;AAAkDC,IAAAA,SAAlD;AAA6DC,IAAAA,MAA7D;AAAqEC,IAAAA,WAArE;AAAkFC,IAAAA,aAAlF;AAAiGC,IAAAA,MAAjG;AAAyGC,IAAAA,SAAzG;AAAoHC,IAAAA,QAApH;AAA8HC,IAAAA,OAA9H;AAAuIC,IAAAA;AAAvI,GAAP;;AAEA,iBAAeZ,UAAf,CAA0B;AAACa,IAAAA,aAAD;AAAgBC,IAAAA,SAAhB;AAA2BC,IAAAA,YAA3B;AAAyCC,IAAAA;AAAzC,GAA1B,EAA+E;AAC7E,UAAMC,IAAI,GAAG,uBAASC,MAAT,EAAb;AACA,UAAMC,YAAY,GAAG;AACnBN,MAAAA,aADmB;AAEnBC,MAAAA,SAFmB;AAGnBE,MAAAA,SAHmB;AAInBD,MAAAA,YAJmB;AAKnBK,MAAAA,cAAc,EAAEC,iCAAsBC,kBALnB;AAMnBC,MAAAA,YAAY,EAAEN,IANK;AAOnBO,MAAAA,gBAAgB,EAAEP,IAPC;AAQnBQ,MAAAA,SAAS,EAAE;AARQ,KAArB;;AAUA,QAAI;AACF,YAAMC,MAAM,GAAG,MAAM9B,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BsC,SAA1B,CAAoCR,YAApC,CAArB;;AACA,UAAIO,MAAM,CAACE,YAAX,EAAyB;AACvB;AACD;;AACD,YAAM,IAAIC,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD,KAND,CAME,OAAOC,KAAP,EAAc;AACd,2BAASA,KAAT;AACA,YAAM,IAAIH,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD;AACF;;AAED,WAAS9B,UAAT,CAAoB;AAACY,IAAAA,aAAD;AAAgBC,IAAAA,SAAhB;AAA2BC,IAAAA,YAA3B;AAAyCC,IAAAA,SAAzC;AAAoDiB,IAAAA,WAApD;AAAiEC,IAAAA,gBAAjE;AAAmFC,IAAAA;AAAnF,GAApB,EAAgH;AAC9G,UAAMlB,IAAI,GAAG,uBAASC,MAAT,EAAb;AACA,UAAMC,YAAY,GAAG;AACnBN,MAAAA,aADmB;AAEnBC,MAAAA,SAFmB;AAGnBC,MAAAA,YAHmB;AAInBC,MAAAA,SAJmB;AAKnBiB,MAAAA,WALmB;AAMnBC,MAAAA,gBANmB;AAOnBd,MAAAA,cAAc,EAAEgB,4BAAiBC,SAPd;AAQnBd,MAAAA,YAAY,EAAEN,IARK;AASnBO,MAAAA,gBAAgB,EAAEP,IATC;AAUnBqB,MAAAA,UAAU,EAAE;AAVO,KAArB;;AAYA,QAAI;AACF1C,MAAAA,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BsC,SAA1B,CAAoCR,YAApC;AACA7B,MAAAA,MAAM,CAACiD,GAAP,CAAW,MAAX,EAAmB,+BAAnB;AACD,KAHD,CAGE,OAAOP,KAAP,EAAc;AACd,2BAASA,KAAT;AACA,YAAM,IAAIH,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD;;AAED,WAAO,IAAIS,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtC,YAAMC,YAAY,GAAG9C,YAAY,CAAC+C,gBAAb,CAA8B/B,aAA9B,CAArB;AAEAsB,MAAAA,MAAM,CACHU,EADH,CACM,OADN,EACeH,MADf,EAEGG,EAFH,CAEM,MAFN,EAEcC,KAAK,IAAIH,YAAY,CAACI,KAAb,CAAmBD,KAAnB,CAFvB,EAGGD,EAHH,CAGM,KAHN,EAGa,MAAMF,YAAY,CAACK,GAAb,CAAiBC,SAAjB,EAA4BA,SAA5B,EAAuC,MAAM;AAC5DR,QAAAA,OAAO,CAAC5B,aAAD,CAAP;AACD,OAFgB,CAHnB;AAMD,KATM,CAAP;AAUD;;AAED,iBAAeX,gBAAf,CAAgC;AAACW,IAAAA,aAAD;AAAgBqC,IAAAA;AAAhB,GAAhC,EAAwD;AACtD,UAAMC,OAAO,GAAG,MAAMC,YAAY,CAACvC,aAAD,CAAlC;;AACA,QAAIsC,OAAJ,EAAa;AACX,aAAOzC,QAAQ,CAAC;AAACG,QAAAA,aAAD;AAAgBqC,QAAAA;AAAhB,OAAD,CAAf;AACD;;AACD,WAAO,KAAP;AACD;;AAED,iBAAe/C,KAAf,CAAqBkD,MAArB,EAA6B;AAC3B,UAAM3B,MAAM,GAAG,MAAM9B,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BiE,IAA1B,CAA+BD,MAA/B,EAAuC;AAACE,MAAAA,UAAU,EAAE;AAACC,QAAAA,GAAG,EAAE;AAAN;AAAb,KAAvC,EAClBC,OADkB,EAArB;AAEAnE,IAAAA,MAAM,CAACoE,IAAP,CAAa,iBAAgBhC,MAAM,CAACiC,MAAP,GAAgB,CAAhB,GAAoB,QAApB,GAA+B,YAAa,EAAzE;AACA,WAAOjC,MAAP;AACD;;AAED,iBAAetB,SAAf,CAAyBS,aAAzB,EAAwC+C,YAAxC,EAAsD;AACpD,UAAMlC,MAAM,GAAG,MAAM9B,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BwE,OAA1B,CAAkC;AAAChD,MAAAA;AAAD,KAAlC,CAArB;;AACA,QAAI+C,YAAJ,EAAkB;AAChB,YAAMT,OAAO,GAAG,MAAMC,YAAY,CAACvC,aAAD,CAAlC;;AACA,UAAIsC,OAAJ,EAAa;AACX,eAAO/C,SAAS,CAACS,aAAD,CAAhB;AACD;;AACD,aAAOa,MAAP;AACD;;AAED,WAAOA,MAAP;AACD;;AAED,iBAAe0B,YAAf,CAA4BvC,aAA5B,EAA2C;AACzC,UAAMa,MAAM,GAAG,MAAM9B,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BwE,OAA1B,CAAkC;AAAChD,MAAAA;AAAD,KAAlC,CAArB;AACA,UAAMiD,WAAW,GAAG,qBAAOpC,MAAM,CAACF,gBAAd,EAAgCuC,GAAhC,CAAoC,CAApC,EAAuC,GAAvC,CAApB;AACAzE,IAAAA,MAAM,CAAC0E,KAAP,CAAc,aAAYF,WAAY,EAAtC;;AACA,QAAIA,WAAW,CAACG,QAAZ,EAAJ,EAA4B;AAC1B,YAAMvD,QAAQ,CAAC;AAACG,QAAAA,aAAD;AAAgBqC,QAAAA,KAAK,EAAE7B,iCAAsB6C;AAA7C,OAAD,CAAd;AACA,aAAO,KAAP;AACD;;AAED,WAAO,IAAP;AACD;;AAED,iBAAe7D,MAAf,CAAsBgD,MAAtB,EAA8B;AAC5B/D,IAAAA,MAAM,CAAC6E,KAAP,CAAc,GAAEC,IAAI,CAACC,SAAL,CAAehB,MAAf,CAAuB,EAAvC;AACA/D,IAAAA,MAAM,CAACoE,IAAP,CAAa,2BAA0BL,MAAM,CAACxC,aAAc,EAA5D;AACA,UAAMyD,KAAK,GAAG,4BAASjB,MAAM,CAACxC,aAAhB,CAAd;;AAEA,QAAI;AACF,YAAM0D,eAAe,CAAC;AAAC1E,QAAAA,YAAD;AAAe2E,QAAAA,QAAQ,EAAEF;AAAzB,OAAD,CAArB;AACA,YAAMG,SAAS,GAAG,MAAMlE,aAAa,CAAC8C,MAAD,CAArC;;AACA,UAAIoB,SAAJ,EAAe;AACb,cAAM7E,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BqF,SAA1B,CAAoC;AAAC7D,UAAAA,aAAa,EAAEyD;AAAhB,SAApC,CAAN;AACA,eAAO,IAAP;AACD;AACF,KAPD,CAOE,OAAOK,GAAP,EAAY;AACZ,UAAIA,GAAG,CAACC,MAAJ,KAAe9C,oBAAW+C,SAA9B,EAAyC;AACvC,cAAMjF,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BqF,SAA1B,CAAoC;AAAC7D,UAAAA,aAAa,EAAEyD;AAAhB,SAApC,CAAN;AACA,eAAO,IAAP;AACD;;AACD,YAAMK,GAAN;AACD;AACF;;AAED,iBAAerE,WAAf,CAA2BO,aAA3B,EAA0C;AACxCvB,IAAAA,MAAM,CAACoE,IAAP,CAAa,2BAA0B7C,aAAc,EAArD;AACA,UAAMyD,KAAK,GAAG,4BAASzD,aAAT,CAAd;AACA,UAAMa,MAAM,GAAG,MAAM9B,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BwE,OAA1B,CAAkC;AAAChD,MAAAA,aAAa,EAAEyD;AAAhB,KAAlC,CAArB,CAHwC,CAGwC;;AAEhF,QAAI5C,MAAJ,EAAY;AACV;AACA,YAAM6C,eAAe,CAAC;AAAC1E,QAAAA,YAAD;AAAe2E,QAAAA,QAAQ,EAAEF;AAAzB,OAAD,CAArB;AACA,aAAO;AACLrC,QAAAA,WAAW,EAAEP,MAAM,CAACO,WADf;AAEL6C,QAAAA,UAAU,EAAEjF,YAAY,CAACkF,wBAAb,CAAsCT,KAAtC;AAFP,OAAP;AAID;;AAED,UAAM,IAAIzC,qBAAJ,CAAaC,oBAAW+C,SAAxB,CAAN;AACD;;AAED,iBAAetE,aAAf,CAA6B8C,MAA7B,EAAqC;AACnC/D,IAAAA,MAAM,CAACoE,IAAP,CAAa,4BAA2BL,MAAM,CAACxC,aAAc,EAA7D;AACA,UAAMyD,KAAK,GAAG,4BAASjB,MAAM,CAACxC,aAAhB,CAAd;AAEA,UAAMa,MAAM,GAAG,MAAM9B,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BwE,OAA1B,CAAkC;AAAChD,MAAAA,aAAa,EAAEyD;AAAhB,KAAlC,CAArB,CAJmC,CAI6C;;AAChF,QAAI5C,MAAJ,EAAY;AACV,YAAM;AAAC8B,QAAAA,GAAG,EAAEwB;AAAN,UAAgB,MAAMT,eAAe,CAAC;AAAC1E,QAAAA,YAAD;AAAe2E,QAAAA,QAAQ,EAAEF;AAAzB,OAAD,CAA3C;AACA,YAAMzE,YAAY,CAACoF,MAAb,CAAoBD,MAApB,CAAN;AACA,aAAO,IAAP;AACD;AACF;;AAED,WAASxE,MAAT,CAAgB;AAACQ,IAAAA,SAAD;AAAYI,IAAAA;AAAZ,GAAhB,EAA6C;AAC3C,UAAMkD,KAAK,GAAG;AAAClD,MAAAA,cAAc,EAAE,4BAASA,cAAT;AAAjB,KAAd;;AACA,QAAI;AACF,UAAIJ,SAAS,KAAKiC,SAAlB,EAA6B;AAC3B3D,QAAAA,MAAM,CAACiD,GAAP,CAAW,OAAX,EAAqB,mBAAkB6B,IAAI,CAACC,SAAL,CAAeC,KAAK,CAAClD,cAArB,CAAqC,EAA5E;AACA,eAAOxB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BwE,OAA1B,CAAkC,EAAC,GAAGS;AAAJ,SAAlC,CAAP,CAF2B,CAE2B;AACvD;;AAED,YAAMY,MAAM,GAAG;AAAClE,QAAAA,SAAS,EAAE,4BAASA,SAAT;AAAZ,OAAf;AACA1B,MAAAA,MAAM,CAACiD,GAAP,CAAW,OAAX,EAAqB,mBAAkB+B,KAAK,CAAClD,cAAe,MAAK8D,MAAM,CAAClE,SAAU,EAAlF;AACA,aAAOpB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BwE,OAA1B,CAAkC,EAAC,GAAGS,KAAJ;AAAW,WAAGY;AAAd,OAAlC,CAAP,CARE,CAQ+D;AAClE,KATD,CASE,OAAOlD,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAevB,SAAf,CAAyBI,aAAzB,EAAwC;AACtCvB,IAAAA,MAAM,CAACoE,IAAP,CAAa,2BAA0B7C,aAAc,EAArD;AACA,UAAMyD,KAAK,GAAG,4BAASzD,aAAT,CAAd;;AACA,QAAI;AACF;AACA,YAAM0D,eAAe,CAAC;AAAC1E,QAAAA,YAAD;AAAe2E,QAAAA,QAAQ,EAAEF;AAAzB,OAAD,CAArB,CAFE,CAIF;;AACA,aAAOzE,YAAY,CAACkF,wBAAb,CAAsCT,KAAtC,CAAP;AACD,KAND,CAME,OAAOtC,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAerB,OAAf,CAAuB;AAACE,IAAAA,aAAD;AAAgBsE,IAAAA;AAAhB,GAAvB,EAA6C;AAC3C7F,IAAAA,MAAM,CAAC6E,KAAP,CAAc,gCAA+BtD,aAAc,KAAIsE,GAAI,EAAnE;AACA,UAAMb,KAAK,GAAG,4BAASzD,aAAT,CAAd;AACA,UAAMjB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B+F,SAA1B,CAAoC;AACxCvE,MAAAA,aAAa,EAAEyD;AADyB,KAApC,EAEH;AACDe,MAAAA,IAAI,EAAE;AACJ7D,QAAAA,gBAAgB,EAAE,uBAASN,MAAT;AADd,OADL;AAIDoE,MAAAA,KAAK,EAAE;AACLhD,QAAAA,UAAU,EAAE;AAACiD,UAAAA,KAAK,EAAEJ;AAAR;AADP;AAJN,KAFG,CAAN;AAUD;;AAED,iBAAevE,MAAf,CAAsB;AAACC,IAAAA,aAAD;AAAgB2E,IAAAA;AAAhB,GAAtB,EAA2C;AACzClG,IAAAA,MAAM,CAAC6E,KAAP,CAAc,uBAAsBtD,aAAc,KAAI2E,EAAG,EAAzD;AACA,UAAMlB,KAAK,GAAG,4BAASzD,aAAT,CAAd;AACA,UAAMjB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B+F,SAA1B,CAAoC;AACxCvE,MAAAA,aAAa,EAAEyD;AADyB,KAApC,EAEH;AACDe,MAAAA,IAAI,EAAE;AACJ7D,QAAAA,gBAAgB,EAAE,uBAASN,MAAT,EADd;AAEJO,QAAAA,SAAS,EAAE+D;AAFP;AADL,KAFG,CAAN;AAQD;;AAED,WAAS9E,QAAT,CAAkB;AAACG,IAAAA,aAAD;AAAgBqC,IAAAA,KAAhB;AAAuBuC,IAAAA,YAAY,GAAG;AAAtC,GAAlB,EAA6D;AAC3DnG,IAAAA,MAAM,CAACiD,GAAP,CAAW,MAAX,EAAoB,6BAA4B1B,aAAc,KAAIqC,KAAM,KAAIuC,YAAa,EAAzF;AACA,UAAMnB,KAAK,GAAG,4BAASzD,aAAT,CAAd;AACA,WAAOjB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BqG,gBAA1B,CAA2C;AAChD7E,MAAAA,aAAa,EAAEyD;AADiC,KAA3C,EAEJ;AACDe,MAAAA,IAAI,EAAE;AACJjE,QAAAA,cAAc,EAAE8B,KADZ;AAEJ1B,QAAAA,gBAAgB,EAAE,uBAASN,MAAT,EAFd;AAGJuE,QAAAA;AAHI;AADL,KAFI,EAQJ;AAAClC,MAAAA,UAAU,EAAE;AAACC,QAAAA,GAAG,EAAE;AAAN,OAAb;AAAuBmC,MAAAA,iBAAiB,EAAE;AAA1C,KARI,CAAP;AASD;;AAED,WAASpB,eAAT,CAAyB;AAAC1E,IAAAA,YAAD;AAAe2E,IAAAA;AAAf,GAAzB,EAAmD;AACjD,WAAO,IAAIhC,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtC7C,MAAAA,YAAY,CAACyD,IAAb,CAAkB;AAACkB,QAAAA;AAAD,OAAlB,EACG3B,EADH,CACM,OADN,EACeH,MADf,EAEGG,EAFH,CAEM,MAFN,EAEcJ,OAFd,EAGGI,EAHH,CAGM,KAHN,EAGa,MAAMH,MAAM,CAAC,IAAIb,qBAAJ,CAAaC,oBAAW+C,SAAxB,EAAmC,YAAnC,CAAD,CAHzB;AAID,KALM,CAAP;AAMD;AACF","sourcesContent":["/* eslint-disable max-statements */\n/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport {MongoClient, GridFSBucket} from 'mongodb';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {Error as ApiError} from '@natlibfi/melinda-commons';\nimport {QUEUE_ITEM_STATE, PRIO_QUEUE_ITEM_STATE} from './constants';\nimport {logError} from './utils.js';\nimport moment from 'moment';\nimport httpStatus from 'http-status';\nimport sanitize from 'mongo-sanitize';\n\n/* QueueItem:\n{\n  \"correlationId\":\"FOO\",\n  \"cataloger\":\"xxx0000\",\n  \"oCatalogerIn\":\"xxx0000\"\n  \"operation\":\"UPDATE\",\n  \"contentType\":\"application/json\",\n  \"recordLoadParams\": {\n    \"pActiveLibrary\": \"XXX00\",\n    \"pInputFile\": \"filename.seq\",\n    \"pRejectFile\": \"filename.rej\",\n    \"pLogFile\": \"filename.syslog\",\n    \"pOldNew\": \"NEW\"\n  },\n  \"queueItemState\":\"PENDING_QUEUING\",\n  \"creationTime\":\"2020-01-01T00:00:00.000Z\",\n  \"modificationTime\":\"2020-01-01T00:00:01.000Z\",\n  \"handledIds\": []\n}\n*/\n\nexport default async function (MONGO_URI, collection) {\n  const logger = createLogger();\n\n  // Connect to mongo (MONGO)\n  const client = await MongoClient.connect(MONGO_URI, {useNewUrlParser: true, useUnifiedTopology: true});\n  const db = client.db('rest-api');\n  const gridFSBucket = new GridFSBucket(db, {bucketName: collection});\n\n  return {createPrio, createBulk, checkAndSetState, query, queryById, remove, readContent, removeContent, getOne, getStream, setState, pushIds, pushId};\n\n  async function createPrio({correlationId, cataloger, oCatalogerIn, operation}) {\n    const time = moment().toDate();\n    const newQueueItem = {\n      correlationId,\n      cataloger,\n      operation,\n      oCatalogerIn,\n      queueItemState: PRIO_QUEUE_ITEM_STATE.PENDING_VALIDATION,\n      creationTime: time,\n      modificationTime: time,\n      handledId: ''\n    };\n    try {\n      const result = await db.collection(collection).insertOne(newQueueItem);\n      if (result.acknowledged) {\n        return;\n      }\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    } catch (error) {\n      logError(error);\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    }\n  }\n\n  function createBulk({correlationId, cataloger, oCatalogerIn, operation, contentType, recordLoadParams, stream}) {\n    const time = moment().toDate();\n    const newQueueItem = {\n      correlationId,\n      cataloger,\n      oCatalogerIn,\n      operation,\n      contentType,\n      recordLoadParams,\n      queueItemState: QUEUE_ITEM_STATE.UPLOADING,\n      creationTime: time,\n      modificationTime: time,\n      handledIds: []\n    };\n    try {\n      db.collection(collection).insertOne(newQueueItem);\n      logger.log('info', 'New queue item has been made!');\n    } catch (error) {\n      logError(error);\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    }\n\n    return new Promise((resolve, reject) => {\n      const outputStream = gridFSBucket.openUploadStream(correlationId);\n\n      stream\n        .on('error', reject)\n        .on('data', chunk => outputStream.write(chunk))\n        .on('end', () => outputStream.end(undefined, undefined, () => {\n          resolve(correlationId);\n        }));\n    });\n  }\n\n  async function checkAndSetState({correlationId, state}) {\n    const timeOut = await checkTimeOut(correlationId);\n    if (timeOut) {\n      return setState({correlationId, state});\n    }\n    return false;\n  }\n\n  async function query(params) {\n    const result = await db.collection(collection).find(params, {projection: {_id: 0}})\n      .toArray();\n    logger.info(`Query result: ${result.length > 0 ? 'Found!' : 'Not found!'}`);\n    return result;\n  }\n\n  async function queryById(correlationId, checkModTime) {\n    const result = await db.collection(collection).findOne({correlationId});\n    if (checkModTime) {\n      const timeOut = await checkTimeOut(correlationId);\n      if (timeOut) {\n        return queryById(correlationId);\n      }\n      return result;\n    }\n\n    return result;\n  }\n\n  async function checkTimeOut(correlationId) {\n    const result = await db.collection(collection).findOne({correlationId});\n    const timeoutTime = moment(result.modificationTime).add(1, 'm');\n    logger.silly(`timeOut @ ${timeoutTime}`);\n    if (timeoutTime.isBefore()) {\n      await setState({correlationId, state: PRIO_QUEUE_ITEM_STATE.ABORT});\n      return false;\n    }\n\n    return true;\n  }\n\n  async function remove(params) {\n    logger.debug(`${JSON.stringify(params)}`);\n    logger.info(`Removing from Mongo id: ${params.correlationId}`);\n    const clean = sanitize(params.correlationId);\n\n    try {\n      await getFileMetadata({gridFSBucket, filename: clean});\n      const noContent = await removeContent(params);\n      if (noContent) {\n        await db.collection(collection).deleteOne({correlationId: clean});\n        return true;\n      }\n    } catch (err) {\n      if (err.status === httpStatus.NOT_FOUND) {\n        await db.collection(collection).deleteOne({correlationId: clean});\n        return true;\n      }\n      throw err;\n    }\n  }\n\n  async function readContent(correlationId) {\n    logger.info(`Reading content for id: ${correlationId}`);\n    const clean = sanitize(correlationId);\n    const result = await db.collection(collection).findOne({correlationId: clean}); //ignore: node_nosqli_injection\n\n    if (result) {\n      // Check if the file exists\n      await getFileMetadata({gridFSBucket, filename: clean});\n      return {\n        contentType: result.contentType,\n        readStream: gridFSBucket.openDownloadStreamByName(clean)\n      };\n    }\n\n    throw new ApiError(httpStatus.NOT_FOUND);\n  }\n\n  async function removeContent(params) {\n    logger.info(`Removing content for id: ${params.correlationId}`);\n    const clean = sanitize(params.correlationId);\n\n    const result = await db.collection(collection).findOne({correlationId: clean}); //ignore: node_nosqli_injection\n    if (result) {\n      const {_id: fileId} = await getFileMetadata({gridFSBucket, filename: clean});\n      await gridFSBucket.delete(fileId);\n      return true;\n    }\n  }\n\n  function getOne({operation, queueItemState}) {\n    const clean = {queueItemState: sanitize(queueItemState)};\n    try {\n      if (operation === undefined) {\n        logger.log('silly', `Checking DB for ${JSON.stringify(clean.queueItemState)}`);\n        return db.collection(collection).findOne({...clean}); //ignore: node_nosqli_injection\n      }\n\n      const clean2 = {operation: sanitize(operation)};\n      logger.log('silly', `Checking DB for ${clean.queueItemState} + ${clean2.operation}`);\n      return db.collection(collection).findOne({...clean, ...clean2}); //ignore: node_nosqli_injection\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function getStream(correlationId) {\n    logger.info(`Forming stream from db: ${correlationId}`);\n    const clean = sanitize(correlationId);\n    try {\n      // Check that content is there\n      await getFileMetadata({gridFSBucket, filename: clean});\n\n      // Return content stream\n      return gridFSBucket.openDownloadStreamByName(clean);\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function pushIds({correlationId, ids}) {\n    logger.debug(`Push queue-item ids to list: ${correlationId}, ${ids}`);\n    const clean = sanitize(correlationId);\n    await db.collection(collection).updateOne({\n      correlationId: clean\n    }, {\n      $set: {\n        modificationTime: moment().toDate()\n      },\n      $push: {\n        handledIds: {$each: ids}\n      }\n    });\n  }\n\n  async function pushId({correlationId, id}) {\n    logger.debug(`Push queue-item id: ${correlationId}, ${id}`);\n    const clean = sanitize(correlationId);\n    await db.collection(collection).updateOne({\n      correlationId: clean\n    }, {\n      $set: {\n        modificationTime: moment().toDate(),\n        handledId: id\n      }\n    });\n  }\n\n  function setState({correlationId, state, errorMessage = ''}) {\n    logger.log('info', `Setting queue-item state: ${correlationId}, ${state}, ${errorMessage}`);\n    const clean = sanitize(correlationId);\n    return db.collection(collection).findOneAndUpdate({\n      correlationId: clean\n    }, {\n      $set: {\n        queueItemState: state,\n        modificationTime: moment().toDate(),\n        errorMessage\n      }\n    }, {projection: {_id: 0}, returnNewDocument: true});\n  }\n\n  function getFileMetadata({gridFSBucket, filename}) {\n    return new Promise((resolve, reject) => {\n      gridFSBucket.find({filename})\n        .on('error', reject)\n        .on('data', resolve)\n        .on('end', () => reject(new ApiError(httpStatus.NOT_FOUND, 'No content')));\n    });\n  }\n}\n"],"file":"mongo.js"}
+{"version":3,"sources":["../src/mongo.js"],"names":["MONGO_URI","collection","logger","client","MongoClient","connect","useNewUrlParser","useUnifiedTopology","db","gridFSBucket","GridFSBucket","bucketName","createPrio","createBulk","checkAndSetState","query","queryById","remove","readContent","removeContent","getOne","getStream","setState","pushIds","pushMessages","correlationId","cataloger","oCatalogerIn","operation","noop","undefined","unique","prio","time","toDate","newQueueItem","operationSettings","queueItemState","QUEUE_ITEM_STATE","VALIDATOR","PENDING_VALIDATION","creationTime","modificationTime","handledIds","rejectedIds","result","insertOne","acknowledged","info","ApiError","httpStatus","INTERNAL_SERVER_ERROR","error","contentType","recordLoadParams","stream","UPLOADING","Promise","resolve","reject","outputStream","openUploadStream","on","chunk","write","end","state","errorMessage","errorStatus","timeOut","checkTimeOut","params","find","projection","_id","toArray","debug","length","checkModTime","findOne","timeoutTime","add","silly","isBefore","oldState","ABORT","REQUEST_TIMEOUT","JSON","stringify","clean","noContent","deleteOne","err","MongoDriverError","message","indexOf","readStream","openDownloadStreamByName","NOT_FOUND","delete","clean2","updateOne","$set","$push","$each","messages","messageField","cleanMessageField","errorString","log","findOneAndUpdate","returnNewDocument"],"mappings":";;;;;;;AA8BA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AArCA;;AACA;;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAWA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAEe,wBAAgBA,SAAhB,EAA2BC,UAA3B,EAAuC;AACpD,QAAMC,MAAM,GAAG,0CAAf,CADoD,CAGpD;;AACA,QAAMC,MAAM,GAAG,MAAMC,qBAAYC,OAAZ,CAAoBL,SAApB,EAA+B;AAACM,IAAAA,eAAe,EAAE,IAAlB;AAAwBC,IAAAA,kBAAkB,EAAE;AAA5C,GAA/B,CAArB;AACA,QAAMC,EAAE,GAAGL,MAAM,CAACK,EAAP,CAAU,UAAV,CAAX;AACA,QAAMC,YAAY,GAAG,IAAIC,qBAAJ,CAAiBF,EAAjB,EAAqB;AAACG,IAAAA,UAAU,EAAEV;AAAb,GAArB,CAArB;AAEA,SAAO;AAACW,IAAAA,UAAD;AAAaC,IAAAA,UAAb;AAAyBC,IAAAA,gBAAzB;AAA2CC,IAAAA,KAA3C;AAAkDC,IAAAA,SAAlD;AAA6DC,IAAAA,MAA7D;AAAqEC,IAAAA,WAArE;AAAkFC,IAAAA,aAAlF;AAAiGC,IAAAA,MAAjG;AAAyGC,IAAAA,SAAzG;AAAoHC,IAAAA,QAApH;AAA8HC,IAAAA,OAA9H;AAAuIC,IAAAA;AAAvI,GAAP;;AAEA,iBAAeZ,UAAf,CAA0B;AAACa,IAAAA,aAAD;AAAgBC,IAAAA,SAAhB;AAA2BC,IAAAA,YAA3B;AAAyCC,IAAAA,SAAzC;AAAoDC,IAAAA,IAAI,GAAGC,SAA3D;AAAsEC,IAAAA,MAAM,GAAGD,SAA/E;AAA0FE,IAAAA,IAAI,GAAGF;AAAjG,GAA1B,EAAuI;AACrI,UAAMG,IAAI,GAAG,uBAASC,MAAT,EAAb;AACA,UAAMC,YAAY,GAAG;AACnBV,MAAAA,aADmB;AAEnBC,MAAAA,SAFmB;AAGnBE,MAAAA,SAHmB;AAInBD,MAAAA,YAJmB;AAKnBS,MAAAA,iBAAiB,EAAE;AAACL,QAAAA,MAAD;AAASF,QAAAA,IAAT;AAAeG,QAAAA;AAAf,OALA;AAMnBK,MAAAA,cAAc,EAAEC,4BAAiBC,SAAjB,CAA2BC,kBANxB;AAOnBC,MAAAA,YAAY,EAAER,IAPK;AAQnBS,MAAAA,gBAAgB,EAAET,IARC;AASnBU,MAAAA,UAAU,EAAE,EATO;AAUnBC,MAAAA,WAAW,EAAE;AAVM,KAArB;;AAYA,QAAI;AACF,YAAMC,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B6C,SAA1B,CAAoCX,YAApC,CAArB;;AACA,UAAIU,MAAM,CAACE,YAAX,EAAyB;AACvB7C,QAAAA,MAAM,CAAC8C,IAAP,CAAa,2BAA0BvB,aAAc,qBAAoBxB,UAAW,EAApF;AACA;AACD;;AACD,YAAM,IAAIgD,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD,KAPD,CAOE,OAAOC,KAAP,EAAc;AACd,2BAASA,KAAT;AACA,YAAM,IAAIH,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD;AACF;;AAED,WAAStC,UAAT,CAAoB;AAACY,IAAAA,aAAD;AAAgBC,IAAAA,SAAhB;AAA2BC,IAAAA,YAA3B;AAAyCC,IAAAA,SAAzC;AAAoDyB,IAAAA,WAApD;AAAiEC,IAAAA,gBAAjE;AAAmFC,IAAAA;AAAnF,GAApB,EAAgH;AAC9G,UAAMtB,IAAI,GAAG,uBAASC,MAAT,EAAb;AACA,UAAMC,YAAY,GAAG;AACnBV,MAAAA,aADmB;AAEnBC,MAAAA,SAFmB;AAGnBC,MAAAA,YAHmB;AAInBC,MAAAA,SAJmB;AAKnByB,MAAAA,WALmB;AAMnBC,MAAAA,gBANmB;AAOnBjB,MAAAA,cAAc,EAAEC,4BAAiBC,SAAjB,CAA2BiB,SAPxB;AAQnBf,MAAAA,YAAY,EAAER,IARK;AASnBS,MAAAA,gBAAgB,EAAET,IATC;AAUnBU,MAAAA,UAAU,EAAE,EAVO;AAWnBC,MAAAA,WAAW,EAAE;AAXM,KAArB;;AAaA,QAAI;AACF;AACApC,MAAAA,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B6C,SAA1B,CAAoCX,YAApC;AACAjC,MAAAA,MAAM,CAAC8C,IAAP,CAAa,2BAA0BvB,aAAc,qBAAoBxB,UAAW,GAApF;AACD,KAJD,CAIE,OAAOmD,KAAP,EAAc;AACd,2BAASA,KAAT;AACA,YAAM,IAAIH,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD;;AAED,WAAO,IAAIM,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtC,YAAMC,YAAY,GAAGnD,YAAY,CAACoD,gBAAb,CAA8BpC,aAA9B,CAArB;AAEA8B,MAAAA,MAAM,CACHO,EADH,CACM,OADN,EACeH,MADf,EAEGG,EAFH,CAEM,MAFN,EAEcC,KAAK,IAAIH,YAAY,CAACI,KAAb,CAAmBD,KAAnB,CAFvB,EAGGD,EAHH,CAGM,KAHN,EAGa,MAAMF,YAAY,CAACK,GAAb,CAAiBnC,SAAjB,EAA4BA,SAA5B,EAAuC,MAAM;AAC5D4B,QAAAA,OAAO,CAACjC,aAAD,CAAP;AACD,OAFgB,CAHnB;AAMD,KATM,CAAP;AAUD,GAvEmD,CAyEpD;;;AACA,iBAAeX,gBAAf,CAAgC;AAACW,IAAAA,aAAD;AAAgByC,IAAAA,KAAhB;AAAuBC,IAAAA,YAAY,GAAG,EAAtC;AAA0CC,IAAAA,WAAW,GAAG;AAAxD,GAAhC,EAA6F;AAC3F;AACA;AACA,UAAMC,OAAO,GAAG,MAAMC,YAAY,CAAC7C,aAAD,CAAlC;;AACA,QAAI4C,OAAJ,EAAa;AACX,aAAO/C,QAAQ,CAAC;AAACG,QAAAA,aAAD;AAAgByC,QAAAA,KAAhB;AAAuBC,QAAAA,YAAvB;AAAqCC,QAAAA;AAArC,OAAD,CAAf;AACD;;AACD,WAAO,KAAP;AACD;;AAED,iBAAerD,KAAf,CAAqBwD,MAArB,EAA6B;AAC3B,UAAM1B,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BuE,IAA1B,CAA+BD,MAA/B,EAAuC;AAACE,MAAAA,UAAU,EAAE;AAACC,QAAAA,GAAG,EAAE;AAAN;AAAb,KAAvC,EAClBC,OADkB,EAArB;AAEAzE,IAAAA,MAAM,CAAC0E,KAAP,CAAc,iBAAgB/B,MAAM,CAACgC,MAAP,GAAgB,CAAhB,GAAoB,QAApB,GAA+B,YAAa,EAA1E;AACA,WAAOhC,MAAP;AACD;;AAED,iBAAe7B,SAAf,CAAyBS,aAAzB,EAAwCqD,YAAxC,EAAsD;AACpD,UAAMjC,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC;AAACtD,MAAAA;AAAD,KAAlC,CAArB;;AACA,QAAIqD,YAAJ,EAAkB;AAChB,YAAMT,OAAO,GAAG,MAAMC,YAAY,CAAC7C,aAAD,CAAlC;;AACA,UAAI4C,OAAJ,EAAa;AACX,eAAOrD,SAAS,CAACS,aAAD,CAAhB;AACD;;AACD,aAAOoB,MAAP;AACD;;AAED,WAAOA,MAAP;AACD,GAtGmD,CAwGpD;AACA;AACA;;;AACA,iBAAeyB,YAAf,CAA4B7C,aAA5B,EAA2C;AACzC,UAAMoB,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC;AAACtD,MAAAA;AAAD,KAAlC,CAArB;AACA,UAAMuD,WAAW,GAAG,qBAAOnC,MAAM,CAACH,gBAAd,EAAgCuC,GAAhC,CAAoC,CAApC,EAAuC,GAAvC,CAApB;AACA/E,IAAAA,MAAM,CAACgF,KAAP,CAAc,aAAYF,WAAY,EAAtC;;AACA,QAAIA,WAAW,CAACG,QAAZ,EAAJ,EAA4B;AAC1B,YAAMC,QAAQ,GAAGvC,MAAM,CAACR,cAAxB;AACA,YAAMf,QAAQ,CAAC;AAACG,QAAAA,aAAD;AAAgByC,QAAAA,KAAK,EAAE5B,4BAAiB+C,KAAxC;AAA+CjB,QAAAA,WAAW,EAAElB,oBAAWoC,eAAvE;AAAwFnB,QAAAA,YAAY,EAAG,cAAaiB,QAAS;AAA7H,OAAD,CAAd;AACA,aAAO,KAAP;AACD;;AAED,WAAO,IAAP;AACD;;AAED,iBAAenE,MAAf,CAAsBsD,MAAtB,EAA8B;AAC5BrE,IAAAA,MAAM,CAAC0E,KAAP,CAAc,GAAEW,IAAI,CAACC,SAAL,CAAejB,MAAf,CAAuB,EAAvC;AACArE,IAAAA,MAAM,CAAC8C,IAAP,CAAa,wBAAuB/C,UAAW,SAAQsE,MAAM,CAAC9C,aAAc,EAA5E;AACA,UAAMgE,KAAK,GAAG,4BAASlB,MAAM,CAAC9C,aAAhB,CAAd;AACAvB,IAAAA,MAAM,CAACgF,KAAP,CAAc,wBAAuBK,IAAI,CAACC,SAAL,CAAeC,KAAf,CAAsB,EAA3D;;AAEA,QAAI;AACF;AACA;AACA,YAAMC,SAAS,GAAG,MAAMvE,aAAa,CAACoD,MAAD,CAArC;;AACA,UAAImB,SAAJ,EAAe;AACb,cAAMlF,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B0F,SAA1B,CAAoC;AAAClE,UAAAA,aAAa,EAAEgE;AAAhB,SAApC,CAAN;AACA,eAAO,IAAP;AACD;AACF,KARD,CAQE,OAAOG,GAAP,EAAY;AACZ,UAAIA,GAAG,YAAYC,yBAAnB,EAAqC;AACnC3F,QAAAA,MAAM,CAACkD,KAAP,CAAawC,GAAG,CAACE,OAAjB;;AACA,YAAIF,GAAG,CAACE,OAAJ,CAAYC,OAAZ,CAAoB,uBAApB,MAAiD,CAAC,CAAtD,EAAyD;AACvD7F,UAAAA,MAAM,CAAC0E,KAAP,CAAc,oDAAmDW,IAAI,CAACC,SAAL,CAAeC,KAAf,CAAsB,SAAQxF,UAAW,EAA1G;AACA,gBAAMO,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B0F,SAA1B,CAAoC;AAAClE,YAAAA,aAAa,EAAEgE;AAAhB,WAApC,CAAN;AACA,iBAAO,IAAP;AACD;;AACD,cAAM,IAAIxC,qBAAJ,CAAaC,oBAAWC,qBAAxB,EAA+CyC,GAAG,CAACE,OAAnD,CAAN;AACD;;AACD,YAAM,IAAI7C,qBAAJ,CAAaC,oBAAWC,qBAAxB,EAA+CyC,GAAG,CAACE,OAAnD,CAAN;AACD;AACF;;AAED,iBAAe5E,WAAf,CAA2BO,aAA3B,EAA0C;AACxCvB,IAAAA,MAAM,CAAC8C,IAAP,CAAa,2BAA0BvB,aAAc,OAAMxB,UAAW,EAAtE;AACA,UAAMwF,KAAK,GAAG,4BAAShE,aAAT,CAAd;AACA,UAAMoB,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC;AAACtD,MAAAA,aAAa,EAAEgE;AAAhB,KAAlC,CAArB,CAHwC,CAGwC;;AAEhF,QAAI5C,MAAJ,EAAY;AACV,aAAO;AACLQ,QAAAA,WAAW,EAAER,MAAM,CAACQ,WADf;AAEL2C,QAAAA,UAAU,EAAEvF,YAAY,CAACwF,wBAAb,CAAsCR,KAAtC;AAFP,OAAP;AAID;;AAED,UAAM,IAAIxC,qBAAJ,CAAaC,oBAAWgD,SAAxB,CAAN;AACD;;AAED,iBAAe/E,aAAf,CAA6BoD,MAA7B,EAAqC;AACnCrE,IAAAA,MAAM,CAAC8C,IAAP,CAAa,4BAA2BuB,MAAM,CAAC9C,aAAc,OAAMxB,UAAW,EAA9E;AACA,UAAMwF,KAAK,GAAG,4BAASlB,MAAM,CAAC9C,aAAhB,CAAd;AAEA,UAAMoB,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC;AAACtD,MAAAA,aAAa,EAAEgE;AAAhB,KAAlC,CAArB,CAJmC,CAI6C;;AAChFvF,IAAAA,MAAM,CAAC0E,KAAP,CAAc,+BAA8BW,IAAI,CAACC,SAAL,CAAe3C,MAAf,CAAuB,EAAnE;;AAEA,QAAIA,MAAJ,EAAY;AACV,YAAMpC,YAAY,CAAC0F,MAAb,CAAoBV,KAApB,CAAN;AACA,aAAO,IAAP;AACD;;AAED,WAAO,IAAP;AACD;;AAED,WAASrE,MAAT,CAAgB;AAACQ,IAAAA,SAAD;AAAYS,IAAAA;AAAZ,GAAhB,EAA6C;AAC3C,UAAMoD,KAAK,GAAG;AAACpD,MAAAA,cAAc,EAAE,4BAASA,cAAT;AAAjB,KAAd;;AACA,QAAI;AACF,UAAIT,SAAS,KAAKE,SAAlB,EAA6B;AAC3B5B,QAAAA,MAAM,CAACgF,KAAP,CAAc,eAAcjF,UAAW,QAAOsF,IAAI,CAACC,SAAL,CAAeC,KAAK,CAACpD,cAArB,CAAqC,EAAnF;AACA,eAAO7B,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC,EAAC,GAAGU;AAAJ,SAAlC,CAAP;AACD;;AAED,YAAMW,MAAM,GAAG;AAACxE,QAAAA,SAAS,EAAE,4BAASA,SAAT;AAAZ,OAAf;AACA1B,MAAAA,MAAM,CAACgF,KAAP,CAAc,eAAcjF,UAAW,QAAOwF,KAAK,CAACpD,cAAe,MAAK+D,MAAM,CAACxE,SAAU,EAAzF;AACA,aAAOpB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC,EAAC,GAAGU,KAAJ;AAAW,WAAGW;AAAd,OAAlC,CAAP;AACD,KATD,CASE,OAAOhD,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,WAAS/B,SAAT,CAAmBI,aAAnB,EAAkC;AAChCvB,IAAAA,MAAM,CAAC8C,IAAP,CAAa,2BAA0BvB,aAAc,OAAMxB,UAAW,EAAtE;AACA,UAAMwF,KAAK,GAAG,4BAAShE,aAAT,CAAd;;AACA,QAAI;AACF;AACA,aAAOhB,YAAY,CAACwF,wBAAb,CAAsCR,KAAtC,CAAP;AACD,KAHD,CAGE,OAAOrC,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAe7B,OAAf,CAAuB;AAACE,IAAAA,aAAD;AAAgBkB,IAAAA,UAAhB;AAA4BC,IAAAA;AAA5B,GAAvB,EAAiE;AAC/D1C,IAAAA,MAAM,CAAC8C,IAAP,CAAa,gCAA+BvB,aAAc,OAAMxB,UAAW,EAA3E;AACAC,IAAAA,MAAM,CAAC0E,KAAP,CAAc,QAAOW,IAAI,CAACC,SAAL,CAAe7C,UAAf,CAA2B,kBAAiB4C,IAAI,CAACC,SAAL,CAAe5C,WAAf,CAA4B,EAA7F;AACA,UAAM6C,KAAK,GAAG,4BAAShE,aAAT,CAAd;AACA,UAAMjB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BoG,SAA1B,CAAoC;AACxC5E,MAAAA,aAAa,EAAEgE;AADyB,KAApC,EAEH;AACDa,MAAAA,IAAI,EAAE;AACJ5D,QAAAA,gBAAgB,EAAE,uBAASR,MAAT;AADd,OADL;AAIDqE,MAAAA,KAAK,EAAE;AACL5D,QAAAA,UAAU,EAAE;AAAC6D,UAAAA,KAAK,EAAE7D;AAAR,SADP;AAELC,QAAAA,WAAW,EAAE;AAAC4D,UAAAA,KAAK,EAAE5D;AAAR;AAFR;AAJN,KAFG,CAAN;AAWD;;AAED,iBAAepB,YAAf,CAA4B;AAACC,IAAAA,aAAD;AAAgBgF,IAAAA,QAAhB;AAA0BC,IAAAA,YAAY,GAAG;AAAzC,GAA5B,EAAkF;AAChFxG,IAAAA,MAAM,CAAC8C,IAAP,CAAa,oBAAmB0D,YAAa,IAAGjF,aAAc,OAAMxB,UAAW,EAA/E;AACAC,IAAAA,MAAM,CAAC0E,KAAP,CAAc,aAAYW,IAAI,CAACC,SAAL,CAAeiB,QAAf,CAAyB,GAAnD;AACA,UAAMhB,KAAK,GAAG,4BAAShE,aAAT,CAAd;AACA,UAAMkF,iBAAiB,GAAG,4BAASD,YAAT,CAA1B;AACA,UAAMlG,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BoG,SAA1B,CAAoC;AACxC5E,MAAAA,aAAa,EAAEgE;AADyB,KAApC,EAEH;AACDa,MAAAA,IAAI,EAAE;AACJ5D,QAAAA,gBAAgB,EAAE,uBAASR,MAAT;AADd,OADL;AAIDqE,MAAAA,KAAK,EAAE;AACL,SAACI,iBAAD,GAAqB;AAACH,UAAAA,KAAK,EAAEC;AAAR;AADhB;AAJN,KAFG,CAAN;AAUD;;AAED,WAASnF,QAAT,CAAkB;AAACG,IAAAA,aAAD;AAAgByC,IAAAA,KAAhB;AAAuBC,IAAAA,YAAY,GAAG,EAAtC;AAA0CC,IAAAA,WAAW,GAAG;AAAxD,GAAlB,EAA+E;AAC7E,UAAMwC,WAAW,GAAGzC,YAAY,IAAIC,WAAhB,GAA+B,qBAAoBD,YAAa,qBAAoBC,WAAY,GAAhG,GAAqG,EAAzH;AACAlE,IAAAA,MAAM,CAAC2G,GAAP,CAAW,MAAX,EAAoB,6BAA4BpF,aAAc,GAAEyC,KAAM,GAAE0C,WAAY,OAAM3G,UAAW,EAArG;AACA,UAAMwF,KAAK,GAAG,4BAAShE,aAAT,CAAd;AACA,WAAOjB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B6G,gBAA1B,CAA2C;AAChDrF,MAAAA,aAAa,EAAEgE;AADiC,KAA3C,EAEJ;AACDa,MAAAA,IAAI,EAAE;AACJjE,QAAAA,cAAc,EAAE6B,KADZ;AAEJxB,QAAAA,gBAAgB,EAAE,uBAASR,MAAT,EAFd;AAGJiC,QAAAA,YAHI;AAIJC,QAAAA;AAJI;AADL,KAFI,EASJ;AAACK,MAAAA,UAAU,EAAE;AAACC,QAAAA,GAAG,EAAE;AAAN,OAAb;AAAuBqC,MAAAA,iBAAiB,EAAE;AAA1C,KATI,CAAP;AAUD;AACF","sourcesContent":["/* eslint-disable max-lines */\n/* eslint-disable max-statements */\n/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport {MongoClient, GridFSBucket, MongoDriverError} from 'mongodb';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {Error as ApiError} from '@natlibfi/melinda-commons';\nimport {QUEUE_ITEM_STATE} from './constants';\nimport {logError} from './utils.js';\nimport moment from 'moment';\nimport httpStatus from 'http-status';\nimport sanitize from 'mongo-sanitize';\n\n/* QueueItem:\n{\n  \"correlationId\":\"FOO\",\n  \"cataloger\":\"xxx0000\",\n  \"oCatalogerIn\":\"xxx0000\"\n  \"operation\":\"UPDATE\",\n  \"contentType\":\"application/json\",\n  \"recordLoadParams\": {\n    \"pActiveLibrary\": \"XXX00\",\n    \"pInputFile\": \"filename.seq\",\n    \"pRejectFile\": \"filename.rej\",\n    \"pLogFile\": \"filename.syslog\",\n    \"pOldNew\": \"NEW\"\n  },\n  \"queueItemState\":\"PENDING_QUEUING\",\n  \"creationTime\":\"2020-01-01T00:00:00.000Z\",\n  \"modificationTime\":\"2020-01-01T00:00:01.000Z\",\n  \"handledIds\": []\n}\n*/\n\nexport default async function (MONGO_URI, collection) {\n  const logger = createLogger();\n\n  // Connect to mongo (MONGO)\n  const client = await MongoClient.connect(MONGO_URI, {useNewUrlParser: true, useUnifiedTopology: true});\n  const db = client.db('rest-api');\n  const gridFSBucket = new GridFSBucket(db, {bucketName: collection});\n\n  return {createPrio, createBulk, checkAndSetState, query, queryById, remove, readContent, removeContent, getOne, getStream, setState, pushIds, pushMessages};\n\n  async function createPrio({correlationId, cataloger, oCatalogerIn, operation, noop = undefined, unique = undefined, prio = undefined}) {\n    const time = moment().toDate();\n    const newQueueItem = {\n      correlationId,\n      cataloger,\n      operation,\n      oCatalogerIn,\n      operationSettings: {unique, noop, prio},\n      queueItemState: QUEUE_ITEM_STATE.VALIDATOR.PENDING_VALIDATION,\n      creationTime: time,\n      modificationTime: time,\n      handledIds: [],\n      rejectedIds: []\n    };\n    try {\n      const result = await db.collection(collection).insertOne(newQueueItem);\n      if (result.acknowledged) {\n        logger.info(`New prio queue item for ${correlationId} has been made in ${collection}`);\n        return;\n      }\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    } catch (error) {\n      logError(error);\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    }\n  }\n\n  function createBulk({correlationId, cataloger, oCatalogerIn, operation, contentType, recordLoadParams, stream}) {\n    const time = moment().toDate();\n    const newQueueItem = {\n      correlationId,\n      cataloger,\n      oCatalogerIn,\n      operation,\n      contentType,\n      recordLoadParams,\n      queueItemState: QUEUE_ITEM_STATE.VALIDATOR.UPLOADING,\n      creationTime: time,\n      modificationTime: time,\n      handledIds: [],\n      rejectedIds: []\n    };\n    try {\n      // No await here, promises later\n      db.collection(collection).insertOne(newQueueItem);\n      logger.info(`New bulk queue item for ${correlationId} has been made in ${collection}!`);\n    } catch (error) {\n      logError(error);\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    }\n\n    return new Promise((resolve, reject) => {\n      const outputStream = gridFSBucket.openUploadStream(correlationId);\n\n      stream\n        .on('error', reject)\n        .on('data', chunk => outputStream.write(chunk))\n        .on('end', () => outputStream.end(undefined, undefined, () => {\n          resolve(correlationId);\n        }));\n    });\n  }\n\n  // Check state that the queueItem has not waited too long and set state\n  async function checkAndSetState({correlationId, state, errorMessage = '', errorStatus = ''}) {\n    // checkTimeOut returns true, if queueItem is fresher than 1 minute\n    // otherwise it sets queueItem to state ABORT (408, 'Timeout')\n    const timeOut = await checkTimeOut(correlationId);\n    if (timeOut) {\n      return setState({correlationId, state, errorMessage, errorStatus});\n    }\n    return false;\n  }\n\n  async function query(params) {\n    const result = await db.collection(collection).find(params, {projection: {_id: 0}})\n      .toArray();\n    logger.debug(`Query result: ${result.length > 0 ? 'Found!' : 'Not found!'}`);\n    return result;\n  }\n\n  async function queryById(correlationId, checkModTime) {\n    const result = await db.collection(collection).findOne({correlationId});\n    if (checkModTime) {\n      const timeOut = await checkTimeOut(correlationId);\n      if (timeOut) {\n        return queryById(correlationId);\n      }\n      return result;\n    }\n\n    return result;\n  }\n\n  // Check that if the item has waited too long\n  // If the last modification time for the queueItem is older than 1 minute\n  // set state to ABORT and return false, otherwise return true\n  async function checkTimeOut(correlationId) {\n    const result = await db.collection(collection).findOne({correlationId});\n    const timeoutTime = moment(result.modificationTime).add(1, 'm');\n    logger.silly(`timeOut @ ${timeoutTime}`);\n    if (timeoutTime.isBefore()) {\n      const oldState = result.queueItemState;\n      await setState({correlationId, state: QUEUE_ITEM_STATE.ABORT, errorStatus: httpStatus.REQUEST_TIMEOUT, errorMessage: `Timeout in ${oldState}`});\n      return false;\n    }\n\n    return true;\n  }\n\n  async function remove(params) {\n    logger.debug(`${JSON.stringify(params)}`);\n    logger.info(`Removing from Mongo (${collection}) id: ${params.correlationId}`);\n    const clean = sanitize(params.correlationId);\n    logger.silly(`mongo/remove: clean: ${JSON.stringify(clean)}`);\n\n    try {\n      //const metadataResult = await getFileMetadata({filename: clean});\n      //logger.debug(`mongo/remove: metadataResult: ${JSON.stringify(metadataResult)}`);\n      const noContent = await removeContent(params);\n      if (noContent) {\n        await db.collection(collection).deleteOne({correlationId: clean});\n        return true;\n      }\n    } catch (err) {\n      if (err instanceof MongoDriverError) {\n        logger.error(err.message);\n        if (err.message.indexOf('File not found for id') !== -1) {\n          logger.debug(`mongo/remove: File not found, removing queueItem ${JSON.stringify(clean)} from ${collection}`);\n          await db.collection(collection).deleteOne({correlationId: clean});\n          return true;\n        }\n        throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n      }\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n    }\n  }\n\n  async function readContent(correlationId) {\n    logger.info(`Reading content for id: ${correlationId} in ${collection}`);\n    const clean = sanitize(correlationId);\n    const result = await db.collection(collection).findOne({correlationId: clean}); // njsscan-ignore: node_nosqli_injection\n\n    if (result) {\n      return {\n        contentType: result.contentType,\n        readStream: gridFSBucket.openDownloadStreamByName(clean)\n      };\n    }\n\n    throw new ApiError(httpStatus.NOT_FOUND);\n  }\n\n  async function removeContent(params) {\n    logger.info(`Removing content for id: ${params.correlationId} in ${collection}`);\n    const clean = sanitize(params.correlationId);\n\n    const result = await db.collection(collection).findOne({correlationId: clean}); // njsscan-ignore: node_nosqli_injection\n    logger.debug(`mongo/removeContent: result ${JSON.stringify(result)}`);\n\n    if (result) {\n      await gridFSBucket.delete(clean);\n      return true;\n    }\n\n    return true;\n  }\n\n  function getOne({operation, queueItemState}) {\n    const clean = {queueItemState: sanitize(queueItemState)};\n    try {\n      if (operation === undefined) {\n        logger.silly(`Checking DB ${collection} for ${JSON.stringify(clean.queueItemState)}`);\n        return db.collection(collection).findOne({...clean});\n      }\n\n      const clean2 = {operation: sanitize(operation)};\n      logger.silly(`Checking DB ${collection} for ${clean.queueItemState} + ${clean2.operation}`);\n      return db.collection(collection).findOne({...clean, ...clean2});\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  function getStream(correlationId) {\n    logger.info(`Forming stream from db: ${correlationId} in ${collection}`);\n    const clean = sanitize(correlationId);\n    try {\n      // Return content stream\n      return gridFSBucket.openDownloadStreamByName(clean);\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function pushIds({correlationId, handledIds, rejectedIds}) {\n    logger.info(`Push queue-item ids to list: ${correlationId} to ${collection}`);\n    logger.debug(`ids: ${JSON.stringify(handledIds)}, rejectedIds: ${JSON.stringify(rejectedIds)}`);\n    const clean = sanitize(correlationId);\n    await db.collection(collection).updateOne({\n      correlationId: clean\n    }, {\n      $set: {\n        modificationTime: moment().toDate()\n      },\n      $push: {\n        handledIds: {$each: handledIds},\n        rejectedIds: {$each: rejectedIds}\n      }\n    });\n  }\n\n  async function pushMessages({correlationId, messages, messageField = 'messages'}) {\n    logger.info(`Push messages to ${messageField} ${correlationId} to ${collection}`);\n    logger.debug(`Messages: ${JSON.stringify(messages)}}`);\n    const clean = sanitize(correlationId);\n    const cleanMessageField = sanitize(messageField);\n    await db.collection(collection).updateOne({\n      correlationId: clean\n    }, {\n      $set: {\n        modificationTime: moment().toDate()\n      },\n      $push: {\n        [cleanMessageField]: {$each: messages}\n      }\n    });\n  }\n\n  function setState({correlationId, state, errorMessage = '', errorStatus = ''}) {\n    const errorString = errorMessage || errorStatus ? `, Error message: '${errorMessage}', Error status: '${errorStatus}'` : '';\n    logger.log('info', `Setting queue-item state: ${correlationId}${state}${errorString} to ${collection}`);\n    const clean = sanitize(correlationId);\n    return db.collection(collection).findOneAndUpdate({\n      correlationId: clean\n    }, {\n      $set: {\n        queueItemState: state,\n        modificationTime: moment().toDate(),\n        errorMessage,\n        errorStatus\n      }\n    }, {projection: {_id: 0}, returnNewDocument: true});\n  }\n}\n"],"file":"mongo.js"}

package/package.json

@@ -14,7 +14,7 @@
 		"url": "git@github.com:natlibfi/melinda-rest-api-commons.git"
 	},
 	"license": "AGPL-3.0+",
-	"version": "2.0.0-alpha.1",
+	"version": "2.0.0-alpha.13",
 	"main": "./dist/index.js",
 	"engines": {
 		"node": ">=12"
@@ -33,35 +33,35 @@
 		"build": "babel src --source-maps --copy-files --delete-dir-on-start --out-dir=dist"
 	},
 	"dependencies": {
-		"@natlibfi/fixura": "^2.1.4",
+		"@natlibfi/fixura": "^2.2.1",
 		"@natlibfi/marc-record": "^6.1.2",
-		"@natlibfi/marc-record-serializers": "^7.2.8",
+		"@natlibfi/marc-record-serializers": "^8.0.0-alpha.1",
 		"@natlibfi/marc-record-validate": "^6.0.1",
-		"@natlibfi/marc-record-validators-melinda": "^8.2.10",
-		"@natlibfi/melinda-backend-commons": "^2.0.3",
-		"@natlibfi/melinda-commons": "^11.1.1",
+		"@natlibfi/marc-record-validators-melinda": "^8.4.4",
+		"@natlibfi/melinda-backend-commons": "^2.0.5",
+		"@natlibfi/melinda-commons": "^11.1.10-alpha.1",
 		"amqplib": ">=0.8.0 <1.0.0",
 		"http-status": "^1.5.0",
 		"moment": "^2.29.1",
 		"mongo-sanitize": "^1.1.0",
-		"mongodb": "^4.0.1"
+		"mongodb": "^4.1.3"
 	},
 	"devDependencies": {
-		"@babel/cli": "^7.14.8",
-		"@babel/core": "^7.14.8",
+		"@babel/cli": "^7.15.7",
+		"@babel/core": "^7.15.8",
 		"@babel/plugin-syntax-dynamic-import": "^7.8.3",
-		"@babel/preset-env": "^7.14.8",
-		"@natlibfi/eslint-config-melinda-backend": "^1.0.6",
-		"@babel/eslint-parser": "^7.14.7",
+		"@babel/preset-env": "^7.15.8",
+		"@natlibfi/eslint-config-melinda-backend": "^1.1.1",
+		"@babel/eslint-parser": "^7.15.8",
 		"babel-plugin-dynamic-import-node": "^2.3.3",
-		"babel-plugin-istanbul": "^6.0.0",
+		"babel-plugin-istanbul": "^6.1.1",
 		"babel-plugin-rewire": "^1.2.0",
 		"chai": "^4.3.4",
 		"cross-env": "^7.0.3",
 		"eslint": "^7.32.0",
-		"eslint-plugin-import": "^2.23.4",
-		"mocha": "^9.0.2",
-		"nodemon": "^2.0.12",
+		"eslint-plugin-import": "^2.25.2",
+		"mocha": "^9.1.3",
+		"nodemon": "^2.0.14",
 		"nyc": "^15.1.0"
 	},
 	"eslintConfig": {