@natlibfi/melinda-rest-api-commons 2.0.0-alpha.12 → 2.0.0-alpha.13

package/dist/amqp.js

@@ -70,7 +70,7 @@ async function _default(AMQP_URL) {
   };
 
   async function checkQueue(queue, style = 'basic', purge = false, toRecord = true) {
-    logger.debug(`checkQueue: ${queue}, Style: ${style}`);
+    logger.verbose(`checkQueue: ${queue}, Style: ${style}: Purge: ${purge}, toRecord: ${toRecord}`);
 
     try {
       const channelInfo = await channel.assertQueue(queue, {
@@ -79,7 +79,7 @@ async function _default(AMQP_URL) {
 
       if (purge) {
         await purgeQueue(purge);
-        logger.debug(`Queue ${queue} has purged ${channelInfo.messageCount} records`);
+        logger.debug(`Queue ${queue} has purged ${channelInfo.messageCount} messages`);
         return checkQueue(queue, style);
       }
 
@@ -88,29 +88,33 @@ async function _default(AMQP_URL) {
         return false;
       }
 
-      logger.debug(`Queue ${queue} has ${channelInfo.messageCount} records`);
+      logger.silly(`Queue ${queue} has ${channelInfo.messageCount} messages`);
 
       if (style === 'messages') {
         return channelInfo.messageCount;
-      }
+      } // Note: returns a message + a record
+
 
       if (style === 'one') {
         return consumeOne(queue);
-      } // Note: returns just the message
+      } // Note: returns just a message
 
 
       if (style === 'raw') {
         return consumeRaw(queue);
-      }
+      } // Note: returns a chunk of messages + records
+
 
       if (style === 'rawChunk') {
         return consumeRawChunk(queue);
-      }
+      } // Note: returns a chunk of messages or a chunk of messages + records (depending on toRecord)
+
 
       if (/^.{8}-.{4}-.{4}-.{4}-.{12}$/u.test(style)) {
-        logger.debug(`checkQueue: regexp match ${style}`);
+        logger.silly(`checkQueue: regexp match ${style}`);
         return consumeByCorrelationId(queue, style, toRecord);
-      }
+      } // Note: returns a chunk of messages + records
+
 
       if (style === 'basic') {
         return consumeChunk(queue);
@@ -260,7 +264,7 @@ async function _default(AMQP_URL) {
   }
 
   async function consumeOne(queue) {
-    logger.verbose(`Prepared to consume one from queue: ${queue}`);
+    logger.verbose(`Prepared to consumeOne from queue: ${queue}`);
 
     try {
       await channel.assertQueue(queue, {
@@ -286,7 +290,7 @@ async function _default(AMQP_URL) {
   }
 
   async function consumeRaw(queue) {
-    logger.verbose(`Prepared to consume raw from queue: ${queue}`);
+    logger.verbose(`Prepared to consumeRaw from queue: ${queue}`);
 
     try {
       await channel.assertQueue(queue, {
@@ -300,7 +304,6 @@ async function _default(AMQP_URL) {
   }
 
   function ackMessages(messages) {
-    logger.verbose(`Ack messages! (${messages.length})`);
     messages.forEach(message => {
       logger.silly(`Ack message ${message.properties.correlationId}`);
       channel.ack(message);
@@ -308,7 +311,6 @@ async function _default(AMQP_URL) {
   }
 
   function nackMessages(messages) {
-    logger.verbose(`Nack messages! (${messages.length})`);
     messages.forEach(message => {
       logger.silly(`Nack message ${message.properties.correlationId}`);
       channel.nack(message);
@@ -321,11 +323,13 @@ async function _default(AMQP_URL) {
     headers,
     data
   }) {
+    logger.verbose(`Send message for ${correlationId} to queue: ${queue}`);
+
     try {
-      logger.silly(`Record queue ${queue}`);
-      logger.silly(`Record correlationId ${correlationId}`);
-      logger.silly(`Record data ${JSON.stringify(data)}`);
-      logger.silly(`Record headers ${JSON.stringify(headers)}`);
+      logger.silly(`Queue ${queue}`);
+      logger.silly(`CorrelationId ${correlationId}`);
+      logger.silly(`Data ${JSON.stringify(data)}`);
+      logger.silly(`Headers ${JSON.stringify(headers)}`);
       await channel.assertQueue(queue, {
         durable: true
       });
@@ -343,6 +347,7 @@ async function _default(AMQP_URL) {
   }
 
   async function removeQueue(queue) {
+    logger.verbose(`Removing queue ${queue}.`);
     await channel.deleteQueue(queue);
   } // ----------------
   // Helper functions
@@ -350,7 +355,7 @@ async function _default(AMQP_URL) {
 
 
   function messagesToRecords(messages) {
-    logger.verbose(`Parsing messages (${messages.length})to records`);
+    logger.verbose(`Parsing messages (${messages.length}) to records`);
     return messages.map(message => {
       const content = JSON.parse(message.content.toString());
       return new _marcRecord.MarcRecord(content.data);
@@ -364,9 +369,9 @@ async function _default(AMQP_URL) {
       const {
         messageCount
       } = await channel.checkQueue(queue);
-      logger.debug(`There is ${messageCount} messages in queue ${queue}`);
+      logger.silly(`There is ${messageCount} messages in queue ${queue}`);
       const messagesToGet = messageCount >= _constants.CHUNK_SIZE ? _constants.CHUNK_SIZE : messageCount;
-      logger.debug(`Getting ${messagesToGet} messages from queue ${queue}`);
+      logger.silly(`Getting ${messagesToGet} messages from queue ${queue}`);
       const messages = await pump(messagesToGet);
       logger.debug(`Returning ${messages.length} unique messages`);
       return messages;

package/dist/amqp.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/amqp.js"],"names":["AMQP_URL","connection","amqplib","connect","channel","createChannel","logger","checkQueue","consumeChunk","consumeRawChunk","consumeOne","consumeRaw","ackMessages","nackMessages","sendToQueue","removeQueue","messagesToRecords","queue","style","purge","toRecord","debug","channelInfo","assertQueue","durable","purgeQueue","messageCount","test","consumeByCorrelationId","ApiError","httpStatus","UNPROCESSABLE_ENTITY","error","verbose","queMessages","getData","headers","getHeaderInfo","JSON","stringify","filterIn","filterOut","messages","filter","message","properties","cataloger","nack","records","correlationId","get","length","forEach","silly","ack","data","Buffer","from","persistent","deleteQueue","map","content","parse","toString","MarcRecord","messagesToGet","CHUNK_SIZE","pump","count","results","identifiers","identifier","deliveryTag","fields","includes","concat"],"mappings":";;;;;;;AA8BA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AApCA;;AACA;;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAUe,wBAAgBA,QAAhB,EAA0B;AACvC,QAAMC,UAAU,GAAG,MAAMC,iBAAQC,OAAR,CAAgBH,QAAhB,CAAzB;AACA,QAAMI,OAAO,GAAG,MAAMH,UAAU,CAACI,aAAX,EAAtB;AACA,QAAMC,MAAM,GAAG,0CAAf;AAEA,SAAO;AAACC,IAAAA,UAAD;AAAaC,IAAAA,YAAb;AAA2BC,IAAAA,eAA3B;AAA4CC,IAAAA,UAA5C;AAAwDC,IAAAA,UAAxD;AAAoEC,IAAAA,WAApE;AAAiFC,IAAAA,YAAjF;AAA+FC,IAAAA,WAA/F;AAA4GC,IAAAA,WAA5G;AAAyHC,IAAAA;AAAzH,GAAP;;AAEA,iBAAeT,UAAf,CAA0BU,KAA1B,EAAiCC,KAAK,GAAG,OAAzC,EAAkDC,KAAK,GAAG,KAA1D,EAAiEC,QAAQ,GAAG,IAA5E,EAAkF;AAChFd,IAAAA,MAAM,CAACe,KAAP,CAAc,eAAcJ,KAAM,YAAWC,KAAM,EAAnD;;AACA,QAAI;AACF,YAAMI,WAAW,GAAG,MAAMlB,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAA1B;;AACA,UAAIL,KAAJ,EAAW;AACT,cAAMM,UAAU,CAACN,KAAD,CAAhB;AACAb,QAAAA,MAAM,CAACe,KAAP,CAAc,SAAQJ,KAAM,eAAcK,WAAW,CAACI,YAAa,UAAnE;AACA,eAAOnB,UAAU,CAACU,KAAD,EAAQC,KAAR,CAAjB;AACD;;AAED,UAAII,WAAW,CAACI,YAAZ,GAA2B,CAA/B,EAAkC;AAChCpB,QAAAA,MAAM,CAACe,KAAP,CAAc,eAAcC,WAAW,CAACI,YAAa,MAAKT,KAAM,WAAhE;AACA,eAAO,KAAP;AACD;;AACDX,MAAAA,MAAM,CAACe,KAAP,CAAc,SAAQJ,KAAM,QAAOK,WAAW,CAACI,YAAa,UAA5D;;AAEA,UAAIR,KAAK,KAAK,UAAd,EAA0B;AACxB,eAAOI,WAAW,CAACI,YAAnB;AACD;;AAED,UAAIR,KAAK,KAAK,KAAd,EAAqB;AACnB,eAAOR,UAAU,CAACO,KAAD,CAAjB;AACD,OApBC,CAsBF;;;AACA,UAAIC,KAAK,KAAK,KAAd,EAAqB;AACnB,eAAOP,UAAU,CAACM,KAAD,CAAjB;AACD;;AAED,UAAIC,KAAK,KAAK,UAAd,EAA0B;AACxB,eAAOT,eAAe,CAACQ,KAAD,CAAtB;AACD;;AAED,UAAK,8BAAD,CAAiCU,IAAjC,CAAsCT,KAAtC,CAAJ,EAAkD;AAChDZ,QAAAA,MAAM,CAACe,KAAP,CAAc,4BAA2BH,KAAM,EAA/C;AACA,eAAOU,sBAAsB,CAACX,KAAD,EAAQC,KAAR,EAAeE,QAAf,CAA7B;AACD;;AAED,UAAIF,KAAK,KAAK,OAAd,EAAuB;AACrB,eAAOV,YAAY,CAACS,KAAD,CAAnB;AACD,OAtCC,CAwCF;;;AACA,YAAM,IAAIY,qBAAJ,CAAaC,oBAAWC,oBAAxB,CAAN;AACD,KA1CD,CA0CE,OAAOC,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;;AAED,aAASP,UAAT,CAAoBN,KAApB,EAA2B;AACzB,UAAIA,KAAJ,EAAW;AACT,eAAOf,OAAO,CAACqB,UAAR,CAAmBR,KAAnB,CAAP;AACD;AACF;AACF;;AAED,iBAAeT,YAAf,CAA4BS,KAA5B,EAAmC;AACjCX,IAAAA,MAAM,CAAC2B,OAAP,CAAgB,wCAAuChB,KAAM,EAA7D;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAGF;;AACA,YAAMU,WAAW,GAAG,MAAMC,OAAO,CAAClB,KAAD,CAAjC;AAEA,YAAMmB,OAAO,GAAGC,aAAa,CAACH,WAAW,CAAC,CAAD,CAAZ,CAA7B;AACA5B,MAAAA,MAAM,CAACe,KAAP,CAAc,sCAAqCiB,IAAI,CAACC,SAAL,CAAeH,OAAf,CAAwB,EAA3E;AAEA,UAAII,QAAQ,GAAG,CAAf,CATE,CASgB;;AAClB,UAAIC,SAAS,GAAG,CAAhB,CAVE,CAUiB;AAEnB;;AACA,YAAMC,QAAQ,GAAGR,WAAW,CAACS,MAAZ,CAAmBC,OAAO,IAAI;AAC7C,YAAIA,OAAO,CAACC,UAAR,CAAmBT,OAAnB,CAA2BU,SAA3B,KAAyCV,OAAO,CAACU,SAArD,EAAgE;AAC9DN,UAAAA,QAAQ,GADsD,CAClD;;AACZ,iBAAO,IAAP;AACD,SAJ4C,CAM7C;;;AACApC,QAAAA,OAAO,CAAC2C,IAAR,CAAaH,OAAb,EAAsB,KAAtB,EAA6B,IAA7B;AACAH,QAAAA,SAAS,GARoC,CAQhC;;AACb,eAAO,KAAP;AACD,OAVgB,CAAjB;AAYAnC,MAAAA,MAAM,CAACe,KAAP,CAAc,4BAA2BmB,QAAS,eAAcC,SAAU,EAA1E;AACA,YAAMO,OAAO,GAAG,MAAMhC,iBAAiB,CAAC0B,QAAD,CAAvC;AAEA,aAAO;AAACN,QAAAA,OAAD;AAAUY,QAAAA,OAAV;AAAmBN,QAAAA;AAAnB,OAAP;AACD,KA7BD,CA6BE,OAAOV,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAeJ,sBAAf,CAAsCX,KAAtC,EAA6CgC,aAA7C,EAA4D7B,QAAQ,GAAG,IAAvE,EAA6E;AAC3Ed,IAAAA,MAAM,CAAC2B,OAAP,CAAgB,kDAAiDhB,KAAM,EAAvE;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAEF;;AACA,YAAMU,WAAW,GAAG,MAAMC,OAAO,CAAClB,KAAD,CAAjC;AAEAX,MAAAA,MAAM,CAACe,KAAP,CAAc,yBAAwB4B,aAAc,EAApD;AAEA,UAAIT,QAAQ,GAAG,CAAf,CAPE,CAOgB;;AAClB,UAAIC,SAAS,GAAG,CAAhB,CARE,CAQiB;AAEnB;;AACA,YAAMC,QAAQ,GAAGR,WAAW,CAACS,MAAZ,CAAmBC,OAAO,IAAI;AAC7C,YAAIA,OAAO,CAACC,UAAR,CAAmBI,aAAnB,KAAqCA,aAAzC,EAAwD;AACtDT,UAAAA,QAAQ,GAD8C,CAC1C;;AACZ,iBAAO,IAAP;AACD,SAJ4C,CAM7C;;;AACApC,QAAAA,OAAO,CAAC2C,IAAR,CAAaH,OAAb,EAAsB,KAAtB,EAA6B,IAA7B;AACAH,QAAAA,SAAS,GARoC,CAQhC;;AACb,eAAO,KAAP;AACD,OAVgB,CAAjB;AAWAnC,MAAAA,MAAM,CAACe,KAAP,CAAc,4BAA2BmB,QAAS,eAAcC,SAAU,EAA1E;AAEA,YAAML,OAAO,GAAGC,aAAa,CAACK,QAAQ,CAAC,CAAD,CAAT,CAA7B;;AAEA,UAAItB,QAAJ,EAAc;AACZ,cAAM4B,OAAO,GAAG,MAAMhC,iBAAiB,CAAC0B,QAAD,CAAvC;AAEA,eAAO;AAACN,UAAAA,OAAD;AAAUY,UAAAA,OAAV;AAAmBN,UAAAA;AAAnB,SAAP;AACD;;AACD,aAAO;AAACN,QAAAA,OAAD;AAAUM,QAAAA;AAAV,OAAP;AACD,KAhCD,CAgCE,OAAOV,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAevB,eAAf,CAA+BQ,KAA/B,EAAsC;AACpCX,IAAAA,MAAM,CAAC2B,OAAP,CAAgB,2CAA0ChB,KAAM,EAAhE;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAEF;;AACA,YAAMU,WAAW,GAAG,MAAMC,OAAO,CAAClB,KAAD,CAAjC;AAEA,YAAMmB,OAAO,GAAGC,aAAa,CAACH,WAAW,CAAC,CAAD,CAAZ,CAA7B;AACA5B,MAAAA,MAAM,CAACe,KAAP,CAAc,yBAAwBiB,IAAI,CAACC,SAAL,CAAeH,OAAf,CAAwB,EAA9D;AAEA,UAAII,QAAQ,GAAG,CAAf,CARE,CAQgB;;AAClB,UAAIC,SAAS,GAAG,CAAhB,CATE,CASiB;AAEnB;;AACA,YAAMC,QAAQ,GAAGR,WAAW,CAACS,MAAZ,CAAmBC,OAAO,IAAI;AAC7C,YAAIA,OAAO,CAACC,UAAR,CAAmBT,OAAnB,CAA2BU,SAA3B,KAAyCV,OAAO,CAACU,SAArD,EAAgE;AAC9DN,UAAAA,QAAQ,GADsD,CAClD;;AAEZ,iBAAO,IAAP;AACD,SAL4C,CAO7C;;;AACApC,QAAAA,OAAO,CAAC2C,IAAR,CAAaH,OAAb,EAAsB,KAAtB,EAA6B,IAA7B;AACAH,QAAAA,SAAS,GAToC,CAShC;;AAEb,eAAO,KAAP;AACD,OAZgB,CAAjB;AAaAnC,MAAAA,MAAM,CAACe,KAAP,CAAc,4BAA2BmB,QAAS,eAAcC,SAAU,EAA1E,EAzBE,CA2BF;;AACA,aAAO;AAACL,QAAAA,OAAD;AAAUM,QAAAA;AAAV,OAAP;AACD,KA7BD,CA6BE,OAAOV,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAetB,UAAf,CAA0BO,KAA1B,EAAiC;AAC/BX,IAAAA,MAAM,CAAC2B,OAAP,CAAgB,uCAAsChB,KAAM,EAA5D;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAGF;;AACA,YAAMoB,OAAO,GAAG,MAAMxC,OAAO,CAAC8C,GAAR,CAAYjC,KAAZ,CAAtB;;AACA,UAAI2B,OAAJ,EAAa;AACX,cAAMR,OAAO,GAAGC,aAAa,CAACO,OAAD,CAA7B;AACA,cAAMI,OAAO,GAAGhC,iBAAiB,CAAC,CAAC4B,OAAD,CAAD,CAAjC;AACA,eAAO;AAACR,UAAAA,OAAD;AAAUY,UAAAA,OAAV;AAAmBN,UAAAA,QAAQ,EAAE,CAACE,OAAD;AAA7B,SAAP;AACD;;AAED,aAAO,KAAP;AACD,KAZD,CAYE,OAAOZ,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAerB,UAAf,CAA0BM,KAA1B,EAAiC;AAC/BX,IAAAA,MAAM,CAAC2B,OAAP,CAAgB,uCAAsChB,KAAM,EAA5D;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAEF;;AACA,aAAO,MAAMpB,OAAO,CAAC8C,GAAR,CAAYjC,KAAZ,CAAb;AACD,KAJD,CAIE,OAAOe,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,WAASpB,WAAT,CAAqB8B,QAArB,EAA+B;AAC7BpC,IAAAA,MAAM,CAAC2B,OAAP,CAAgB,kBAAiBS,QAAQ,CAACS,MAAO,GAAjD;AACAT,IAAAA,QAAQ,CAACU,OAAT,CAAiBR,OAAO,IAAI;AAC1BtC,MAAAA,MAAM,CAAC+C,KAAP,CAAc,eAAcT,OAAO,CAACC,UAAR,CAAmBI,aAAc,EAA7D;AACA7C,MAAAA,OAAO,CAACkD,GAAR,CAAYV,OAAZ;AACD,KAHD;AAID;;AAED,WAAS/B,YAAT,CAAsB6B,QAAtB,EAAgC;AAC9BpC,IAAAA,MAAM,CAAC2B,OAAP,CAAgB,mBAAkBS,QAAQ,CAACS,MAAO,GAAlD;AACAT,IAAAA,QAAQ,CAACU,OAAT,CAAiBR,OAAO,IAAI;AAC1BtC,MAAAA,MAAM,CAAC+C,KAAP,CAAc,gBAAeT,OAAO,CAACC,UAAR,CAAmBI,aAAc,EAA9D;AACA7C,MAAAA,OAAO,CAAC2C,IAAR,CAAaH,OAAb;AACD,KAHD;AAID;;AAED,iBAAe9B,WAAf,CAA2B;AAACG,IAAAA,KAAD;AAAQgC,IAAAA,aAAR;AAAuBb,IAAAA,OAAvB;AAAgCmB,IAAAA;AAAhC,GAA3B,EAAkE;AAChE,QAAI;AACFjD,MAAAA,MAAM,CAAC+C,KAAP,CAAc,gBAAepC,KAAM,EAAnC;AACAX,MAAAA,MAAM,CAAC+C,KAAP,CAAc,wBAAuBJ,aAAc,EAAnD;AACA3C,MAAAA,MAAM,CAAC+C,KAAP,CAAc,eAAcf,IAAI,CAACC,SAAL,CAAegB,IAAf,CAAqB,EAAjD;AACAjD,MAAAA,MAAM,CAAC+C,KAAP,CAAc,kBAAiBf,IAAI,CAACC,SAAL,CAAeH,OAAf,CAAwB,EAAvD;AAEA,YAAMhC,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN;AAEApB,MAAAA,OAAO,CAACU,WAAR,CACEG,KADF,EAEEuC,MAAM,CAACC,IAAP,CAAYnB,IAAI,CAACC,SAAL,CAAe;AAACgB,QAAAA;AAAD,OAAf,CAAZ,CAFF,EAGE;AACEN,QAAAA,aADF;AAEES,QAAAA,UAAU,EAAE,IAFd;AAGEtB,QAAAA;AAHF,OAHF;AAUA9B,MAAAA,MAAM,CAAC+C,KAAP,CAAc,yBAAwBpC,KAAM,EAA5C;AACD,KAnBD,CAmBE,OAAOe,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAejB,WAAf,CAA2BE,KAA3B,EAAkC;AAChC,UAAMb,OAAO,CAACuD,WAAR,CAAoB1C,KAApB,CAAN;AACD,GAtPsC,CAwPvC;AACA;AACA;;;AAEA,WAASD,iBAAT,CAA2B0B,QAA3B,EAAqC;AACnCpC,IAAAA,MAAM,CAAC2B,OAAP,CAAgB,qBAAoBS,QAAQ,CAACS,MAAO,aAApD;AAEA,WAAOT,QAAQ,CAACkB,GAAT,CAAahB,OAAO,IAAI;AAC7B,YAAMiB,OAAO,GAAGvB,IAAI,CAACwB,KAAL,CAAWlB,OAAO,CAACiB,OAAR,CAAgBE,QAAhB,EAAX,CAAhB;AACA,aAAO,IAAIC,sBAAJ,CAAeH,OAAO,CAACN,IAAvB,CAAP;AACD,KAHM,CAAP;AAID;;AAED,iBAAepB,OAAf,CAAuBlB,KAAvB,EAA8B;AAC5BX,IAAAA,MAAM,CAAC2B,OAAP,CAAgB,2BAA0BhB,KAAM,EAAhD;;AACA,QAAI;AACF,YAAM;AAACS,QAAAA;AAAD,UAAiB,MAAMtB,OAAO,CAACG,UAAR,CAAmBU,KAAnB,CAA7B;AACAX,MAAAA,MAAM,CAACe,KAAP,CAAc,YAAWK,YAAa,sBAAqBT,KAAM,EAAjE;AACA,YAAMgD,aAAa,GAAGvC,YAAY,IAAIwC,qBAAhB,GAA6BA,qBAA7B,GAA0CxC,YAAhE;AACApB,MAAAA,MAAM,CAACe,KAAP,CAAc,WAAU4C,aAAc,wBAAuBhD,KAAM,EAAnE;AAEA,YAAMyB,QAAQ,GAAG,MAAMyB,IAAI,CAACF,aAAD,CAA3B;AAEA3D,MAAAA,MAAM,CAACe,KAAP,CAAc,aAAYqB,QAAQ,CAACS,MAAO,kBAA1C;AAEA,aAAOT,QAAP;AACD,KAXD,CAWE,OAAOV,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;;AAED,mBAAemC,IAAf,CAAoBC,KAApB,EAA2BC,OAAO,GAAG,EAArC,EAAyCC,WAAW,GAAG,EAAvD,EAA2D;AACzD,UAAIF,KAAK,KAAK,CAAd,EAAiB;AACf,eAAOC,OAAP;AACD;;AAED,YAAMzB,OAAO,GAAG,MAAMxC,OAAO,CAAC8C,GAAR,CAAYjC,KAAZ,CAAtB;AACA,YAAMsD,UAAU,GAAG;AACjBtB,QAAAA,aAAa,EAAEL,OAAO,CAACC,UAAR,CAAmBI,aADjB;AAEjBuB,QAAAA,WAAW,EAAE5B,OAAO,CAAC6B,MAAR,CAAeD;AAFX,OAAnB,CANyD,CAUzD;;AACA,UAAIF,WAAW,CAACI,QAAZ,CAAqBH,UAArB,CAAJ,EAAsC;AACpC,eAAOJ,IAAI,CAACC,KAAK,GAAG,CAAT,EAAYC,OAAZ,EAAqBC,WAArB,CAAX;AACD;;AAED,aAAOH,IAAI,CAACC,KAAK,GAAG,CAAT,EAAYC,OAAO,CAACM,MAAR,CAAe/B,OAAf,CAAZ,EAAqC0B,WAAW,CAACK,MAAZ,CAAmBJ,UAAnB,CAArC,CAAX;AACD;AACF;;AAED,WAASlC,aAAT,CAAuBkB,IAAvB,EAA6B;AAC3B,WAAOA,IAAI,CAACV,UAAL,CAAgBT,OAAvB;AACD;AACF","sourcesContent":["/* eslint-disable max-statements */\n/* eslint-disable max-lines */\n/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport amqplib from 'amqplib';\nimport {MarcRecord} from '@natlibfi/marc-record';\nimport {Error as ApiError} from '@natlibfi/melinda-commons';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {CHUNK_SIZE} from './constants';\nimport {logError} from './utils';\nimport httpStatus from 'http-status';\n\nexport default async function (AMQP_URL) {\n  const connection = await amqplib.connect(AMQP_URL);\n  const channel = await connection.createChannel();\n  const logger = createLogger();\n\n  return {checkQueue, consumeChunk, consumeRawChunk, consumeOne, consumeRaw, ackMessages, nackMessages, sendToQueue, removeQueue, messagesToRecords};\n\n  async function checkQueue(queue, style = 'basic', purge = false, toRecord = true) {\n    logger.debug(`checkQueue: ${queue}, Style: ${style}`);\n    try {\n      const channelInfo = await channel.assertQueue(queue, {durable: true});\n      if (purge) {\n        await purgeQueue(purge);\n        logger.debug(`Queue ${queue} has purged ${channelInfo.messageCount} records`);\n        return checkQueue(queue, style);\n      }\n\n      if (channelInfo.messageCount < 1) {\n        logger.debug(`checkQueue: ${channelInfo.messageCount} - ${queue} is empty`);\n        return false;\n      }\n      logger.debug(`Queue ${queue} has ${channelInfo.messageCount} records`);\n\n      if (style === 'messages') {\n        return channelInfo.messageCount;\n      }\n\n      if (style === 'one') {\n        return consumeOne(queue);\n      }\n\n      // Note: returns just the message\n      if (style === 'raw') {\n        return consumeRaw(queue);\n      }\n\n      if (style === 'rawChunk') {\n        return consumeRawChunk(queue);\n      }\n\n      if ((/^.{8}-.{4}-.{4}-.{4}-.{12}$/u).test(style)) {\n        logger.debug(`checkQueue: regexp match ${style}`);\n        return consumeByCorrelationId(queue, style, toRecord);\n      }\n\n      if (style === 'basic') {\n        return consumeChunk(queue);\n      }\n\n      // Defaults:\n      throw new ApiError(httpStatus.UNPROCESSABLE_ENTITY);\n    } catch (error) {\n      logError(error);\n    }\n\n    function purgeQueue(purge) {\n      if (purge) {\n        return channel.purgeQueue(queue);\n      }\n    }\n  }\n\n  async function consumeChunk(queue) {\n    logger.verbose(`Prepared to consumeChunk from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n\n      // getData: next chunk (100) messages\n      const queMessages = await getData(queue);\n\n      const headers = getHeaderInfo(queMessages[0]);\n      logger.debug(`Filtering messages by cataloger in ${JSON.stringify(headers)}`);\n\n      let filterIn = 0; // eslint-disable-line functional/no-let\n      let filterOut = 0; // eslint-disable-line functional/no-let\n\n      // Check that cataloger match! headers\n      const messages = queMessages.filter(message => {\n        if (message.properties.headers.cataloger === headers.cataloger) {\n          filterIn++; // eslint-disable-line no-plusplus\n          return true;\n        }\n\n        // Nack unwanted ones\n        channel.nack(message, false, true);\n        filterOut++; // eslint-disable-line no-plusplus\n        return false;\n      });\n\n      logger.debug(`Filtering result: valid: ${filterIn} non-valid: ${filterOut}`);\n      const records = await messagesToRecords(messages);\n\n      return {headers, records, messages};\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeByCorrelationId(queue, correlationId, toRecord = true) {\n    logger.verbose(`Prepared to consumeByCorrelationId from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n      // get next chunk (100) messages\n      const queMessages = await getData(queue);\n\n      logger.debug(`Filtering messages by ${correlationId}`);\n\n      let filterIn = 0; // eslint-disable-line functional/no-let\n      let filterOut = 0; // eslint-disable-line functional/no-let\n\n      // Check that correlationId matches query\n      const messages = queMessages.filter(message => {\n        if (message.properties.correlationId === correlationId) {\n          filterIn++; // eslint-disable-line no-plusplus\n          return true;\n        }\n\n        // Nack unwanted ones\n        channel.nack(message, false, true);\n        filterOut++; // eslint-disable-line no-plusplus\n        return false;\n      });\n      logger.debug(`Filtering result: valid: ${filterIn} non-valid: ${filterOut}`);\n\n      const headers = getHeaderInfo(messages[0]);\n\n      if (toRecord) {\n        const records = await messagesToRecords(messages);\n\n        return {headers, records, messages};\n      }\n      return {headers, messages};\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeRawChunk(queue) {\n    logger.verbose(`Prepared to consumeRawChunk from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n      // Get next chunk (100) messages\n      const queMessages = await getData(queue);\n\n      const headers = getHeaderInfo(queMessages[0]);\n      logger.debug(`Filtering messages by ${JSON.stringify(headers)}`);\n\n      let filterIn = 0; // eslint-disable-line functional/no-let\n      let filterOut = 0; // eslint-disable-line functional/no-let\n\n      // Check that cataloger match! headers\n      const messages = queMessages.filter(message => {\n        if (message.properties.headers.cataloger === headers.cataloger) {\n          filterIn++; // eslint-disable-line no-plusplus\n\n          return true;\n        }\n\n        // Nack unwanted ones\n        channel.nack(message, false, true);\n        filterOut++; // eslint-disable-line no-plusplus\n\n        return false;\n      });\n      logger.debug(`Filtering result: valid: ${filterIn} non-valid: ${filterOut}`);\n\n      // return raw (do not convert messages to records)\n      return {headers, messages};\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeOne(queue) {\n    logger.verbose(`Prepared to consume one from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n\n      // Returns false if 0 items in queue\n      const message = await channel.get(queue);\n      if (message) {\n        const headers = getHeaderInfo(message);\n        const records = messagesToRecords([message]);\n        return {headers, records, messages: [message]};\n      }\n\n      return false;\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeRaw(queue) {\n    logger.verbose(`Prepared to consume raw from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n      // Returns false if 0 items in queue\n      return await channel.get(queue);\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  function ackMessages(messages) {\n    logger.verbose(`Ack messages! (${messages.length})`);\n    messages.forEach(message => {\n      logger.silly(`Ack message ${message.properties.correlationId}`);\n      channel.ack(message);\n    });\n  }\n\n  function nackMessages(messages) {\n    logger.verbose(`Nack messages! (${messages.length})`);\n    messages.forEach(message => {\n      logger.silly(`Nack message ${message.properties.correlationId}`);\n      channel.nack(message);\n    });\n  }\n\n  async function sendToQueue({queue, correlationId, headers, data}) {\n    try {\n      logger.silly(`Record queue ${queue}`);\n      logger.silly(`Record correlationId ${correlationId}`);\n      logger.silly(`Record data ${JSON.stringify(data)}`);\n      logger.silly(`Record headers ${JSON.stringify(headers)}`);\n\n      await channel.assertQueue(queue, {durable: true});\n\n      channel.sendToQueue(\n        queue,\n        Buffer.from(JSON.stringify({data})),\n        {\n          correlationId,\n          persistent: true,\n          headers\n        }\n      );\n\n      logger.silly(`Message send to queue ${queue}`);\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function removeQueue(queue) {\n    await channel.deleteQueue(queue);\n  }\n\n  // ----------------\n  // Helper functions\n  // ----------------\n\n  function messagesToRecords(messages) {\n    logger.verbose(`Parsing messages (${messages.length})to records`);\n\n    return messages.map(message => {\n      const content = JSON.parse(message.content.toString());\n      return new MarcRecord(content.data);\n    });\n  }\n\n  async function getData(queue) {\n    logger.verbose(`Getting queue data from ${queue}`);\n    try {\n      const {messageCount} = await channel.checkQueue(queue);\n      logger.debug(`There is ${messageCount} messages in queue ${queue}`);\n      const messagesToGet = messageCount >= CHUNK_SIZE ? CHUNK_SIZE : messageCount;\n      logger.debug(`Getting ${messagesToGet} messages from queue ${queue}`);\n\n      const messages = await pump(messagesToGet);\n\n      logger.debug(`Returning ${messages.length} unique messages`);\n\n      return messages;\n    } catch (error) {\n      logError(error);\n    }\n\n    async function pump(count, results = [], identifiers = []) {\n      if (count === 0) {\n        return results;\n      }\n\n      const message = await channel.get(queue);\n      const identifier = {\n        correlationId: message.properties.correlationId,\n        deliveryTag: message.fields.deliveryTag\n      };\n      // Filter not unique messages\n      if (identifiers.includes(identifier)) {\n        return pump(count - 1, results, identifiers);\n      }\n\n      return pump(count - 1, results.concat(message), identifiers.concat(identifier));\n    }\n  }\n\n  function getHeaderInfo(data) {\n    return data.properties.headers;\n  }\n}\n"],"file":"amqp.js"}
+{"version":3,"sources":["../src/amqp.js"],"names":["AMQP_URL","connection","amqplib","connect","channel","createChannel","logger","checkQueue","consumeChunk","consumeRawChunk","consumeOne","consumeRaw","ackMessages","nackMessages","sendToQueue","removeQueue","messagesToRecords","queue","style","purge","toRecord","verbose","channelInfo","assertQueue","durable","purgeQueue","debug","messageCount","silly","test","consumeByCorrelationId","ApiError","httpStatus","UNPROCESSABLE_ENTITY","error","queMessages","getData","headers","getHeaderInfo","JSON","stringify","filterIn","filterOut","messages","filter","message","properties","cataloger","nack","records","correlationId","get","forEach","ack","data","Buffer","from","persistent","deleteQueue","length","map","content","parse","toString","MarcRecord","messagesToGet","CHUNK_SIZE","pump","count","results","identifiers","identifier","deliveryTag","fields","includes","concat"],"mappings":";;;;;;;AA8BA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AApCA;;AACA;;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAUe,wBAAgBA,QAAhB,EAA0B;AACvC,QAAMC,UAAU,GAAG,MAAMC,iBAAQC,OAAR,CAAgBH,QAAhB,CAAzB;AACA,QAAMI,OAAO,GAAG,MAAMH,UAAU,CAACI,aAAX,EAAtB;AACA,QAAMC,MAAM,GAAG,0CAAf;AAEA,SAAO;AAACC,IAAAA,UAAD;AAAaC,IAAAA,YAAb;AAA2BC,IAAAA,eAA3B;AAA4CC,IAAAA,UAA5C;AAAwDC,IAAAA,UAAxD;AAAoEC,IAAAA,WAApE;AAAiFC,IAAAA,YAAjF;AAA+FC,IAAAA,WAA/F;AAA4GC,IAAAA,WAA5G;AAAyHC,IAAAA;AAAzH,GAAP;;AAEA,iBAAeT,UAAf,CAA0BU,KAA1B,EAAiCC,KAAK,GAAG,OAAzC,EAAkDC,KAAK,GAAG,KAA1D,EAAiEC,QAAQ,GAAG,IAA5E,EAAkF;AAChFd,IAAAA,MAAM,CAACe,OAAP,CAAgB,eAAcJ,KAAM,YAAWC,KAAM,YAAWC,KAAM,eAAcC,QAAS,EAA7F;;AACA,QAAI;AACF,YAAME,WAAW,GAAG,MAAMlB,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAA1B;;AACA,UAAIL,KAAJ,EAAW;AACT,cAAMM,UAAU,CAACN,KAAD,CAAhB;AACAb,QAAAA,MAAM,CAACoB,KAAP,CAAc,SAAQT,KAAM,eAAcK,WAAW,CAACK,YAAa,WAAnE;AACA,eAAOpB,UAAU,CAACU,KAAD,EAAQC,KAAR,CAAjB;AACD;;AAED,UAAII,WAAW,CAACK,YAAZ,GAA2B,CAA/B,EAAkC;AAChCrB,QAAAA,MAAM,CAACoB,KAAP,CAAc,eAAcJ,WAAW,CAACK,YAAa,MAAKV,KAAM,WAAhE;AACA,eAAO,KAAP;AACD;;AACDX,MAAAA,MAAM,CAACsB,KAAP,CAAc,SAAQX,KAAM,QAAOK,WAAW,CAACK,YAAa,WAA5D;;AAEA,UAAIT,KAAK,KAAK,UAAd,EAA0B;AACxB,eAAOI,WAAW,CAACK,YAAnB;AACD,OAhBC,CAkBF;;;AACA,UAAIT,KAAK,KAAK,KAAd,EAAqB;AACnB,eAAOR,UAAU,CAACO,KAAD,CAAjB;AACD,OArBC,CAuBF;;;AACA,UAAIC,KAAK,KAAK,KAAd,EAAqB;AACnB,eAAOP,UAAU,CAACM,KAAD,CAAjB;AACD,OA1BC,CA4BF;;;AACA,UAAIC,KAAK,KAAK,UAAd,EAA0B;AACxB,eAAOT,eAAe,CAACQ,KAAD,CAAtB;AACD,OA/BC,CAiCF;;;AACA,UAAK,8BAAD,CAAiCY,IAAjC,CAAsCX,KAAtC,CAAJ,EAAkD;AAChDZ,QAAAA,MAAM,CAACsB,KAAP,CAAc,4BAA2BV,KAAM,EAA/C;AACA,eAAOY,sBAAsB,CAACb,KAAD,EAAQC,KAAR,EAAeE,QAAf,CAA7B;AACD,OArCC,CAuCF;;;AACA,UAAIF,KAAK,KAAK,OAAd,EAAuB;AACrB,eAAOV,YAAY,CAACS,KAAD,CAAnB;AACD,OA1CC,CA4CF;;;AACA,YAAM,IAAIc,qBAAJ,CAAaC,oBAAWC,oBAAxB,CAAN;AACD,KA9CD,CA8CE,OAAOC,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;;AAED,aAAST,UAAT,CAAoBN,KAApB,EAA2B;AACzB,UAAIA,KAAJ,EAAW;AACT,eAAOf,OAAO,CAACqB,UAAR,CAAmBR,KAAnB,CAAP;AACD;AACF;AACF;;AAED,iBAAeT,YAAf,CAA4BS,KAA5B,EAAmC;AACjCX,IAAAA,MAAM,CAACe,OAAP,CAAgB,wCAAuCJ,KAAM,EAA7D;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAGF;;AACA,YAAMW,WAAW,GAAG,MAAMC,OAAO,CAACnB,KAAD,CAAjC;AAEA,YAAMoB,OAAO,GAAGC,aAAa,CAACH,WAAW,CAAC,CAAD,CAAZ,CAA7B;AACA7B,MAAAA,MAAM,CAACoB,KAAP,CAAc,sCAAqCa,IAAI,CAACC,SAAL,CAAeH,OAAf,CAAwB,EAA3E;AAEA,UAAII,QAAQ,GAAG,CAAf,CATE,CASgB;;AAClB,UAAIC,SAAS,GAAG,CAAhB,CAVE,CAUiB;AAEnB;;AACA,YAAMC,QAAQ,GAAGR,WAAW,CAACS,MAAZ,CAAmBC,OAAO,IAAI;AAC7C,YAAIA,OAAO,CAACC,UAAR,CAAmBT,OAAnB,CAA2BU,SAA3B,KAAyCV,OAAO,CAACU,SAArD,EAAgE;AAC9DN,UAAAA,QAAQ,GADsD,CAClD;;AACZ,iBAAO,IAAP;AACD,SAJ4C,CAM7C;;;AACArC,QAAAA,OAAO,CAAC4C,IAAR,CAAaH,OAAb,EAAsB,KAAtB,EAA6B,IAA7B;AACAH,QAAAA,SAAS,GARoC,CAQhC;;AACb,eAAO,KAAP;AACD,OAVgB,CAAjB;AAYApC,MAAAA,MAAM,CAACoB,KAAP,CAAc,4BAA2Be,QAAS,eAAcC,SAAU,EAA1E;AACA,YAAMO,OAAO,GAAG,MAAMjC,iBAAiB,CAAC2B,QAAD,CAAvC;AAEA,aAAO;AAACN,QAAAA,OAAD;AAAUY,QAAAA,OAAV;AAAmBN,QAAAA;AAAnB,OAAP;AACD,KA7BD,CA6BE,OAAOT,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAeJ,sBAAf,CAAsCb,KAAtC,EAA6CiC,aAA7C,EAA4D9B,QAAQ,GAAG,IAAvE,EAA6E;AAC3Ed,IAAAA,MAAM,CAACe,OAAP,CAAgB,kDAAiDJ,KAAM,EAAvE;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAEF;;AACA,YAAMW,WAAW,GAAG,MAAMC,OAAO,CAACnB,KAAD,CAAjC;AAEAX,MAAAA,MAAM,CAACoB,KAAP,CAAc,yBAAwBwB,aAAc,EAApD;AAEA,UAAIT,QAAQ,GAAG,CAAf,CAPE,CAOgB;;AAClB,UAAIC,SAAS,GAAG,CAAhB,CARE,CAQiB;AAEnB;;AACA,YAAMC,QAAQ,GAAGR,WAAW,CAACS,MAAZ,CAAmBC,OAAO,IAAI;AAC7C,YAAIA,OAAO,CAACC,UAAR,CAAmBI,aAAnB,KAAqCA,aAAzC,EAAwD;AACtDT,UAAAA,QAAQ,GAD8C,CAC1C;;AACZ,iBAAO,IAAP;AACD,SAJ4C,CAM7C;;;AACArC,QAAAA,OAAO,CAAC4C,IAAR,CAAaH,OAAb,EAAsB,KAAtB,EAA6B,IAA7B;AACAH,QAAAA,SAAS,GARoC,CAQhC;;AACb,eAAO,KAAP;AACD,OAVgB,CAAjB;AAWApC,MAAAA,MAAM,CAACoB,KAAP,CAAc,4BAA2Be,QAAS,eAAcC,SAAU,EAA1E;AAEA,YAAML,OAAO,GAAGC,aAAa,CAACK,QAAQ,CAAC,CAAD,CAAT,CAA7B;;AAEA,UAAIvB,QAAJ,EAAc;AACZ,cAAM6B,OAAO,GAAG,MAAMjC,iBAAiB,CAAC2B,QAAD,CAAvC;AAEA,eAAO;AAACN,UAAAA,OAAD;AAAUY,UAAAA,OAAV;AAAmBN,UAAAA;AAAnB,SAAP;AACD;;AACD,aAAO;AAACN,QAAAA,OAAD;AAAUM,QAAAA;AAAV,OAAP;AACD,KAhCD,CAgCE,OAAOT,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAezB,eAAf,CAA+BQ,KAA/B,EAAsC;AACpCX,IAAAA,MAAM,CAACe,OAAP,CAAgB,2CAA0CJ,KAAM,EAAhE;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAEF;;AACA,YAAMW,WAAW,GAAG,MAAMC,OAAO,CAACnB,KAAD,CAAjC;AAEA,YAAMoB,OAAO,GAAGC,aAAa,CAACH,WAAW,CAAC,CAAD,CAAZ,CAA7B;AACA7B,MAAAA,MAAM,CAACoB,KAAP,CAAc,yBAAwBa,IAAI,CAACC,SAAL,CAAeH,OAAf,CAAwB,EAA9D;AAEA,UAAII,QAAQ,GAAG,CAAf,CARE,CAQgB;;AAClB,UAAIC,SAAS,GAAG,CAAhB,CATE,CASiB;AAEnB;;AACA,YAAMC,QAAQ,GAAGR,WAAW,CAACS,MAAZ,CAAmBC,OAAO,IAAI;AAC7C,YAAIA,OAAO,CAACC,UAAR,CAAmBT,OAAnB,CAA2BU,SAA3B,KAAyCV,OAAO,CAACU,SAArD,EAAgE;AAC9DN,UAAAA,QAAQ,GADsD,CAClD;;AAEZ,iBAAO,IAAP;AACD,SAL4C,CAO7C;;;AACArC,QAAAA,OAAO,CAAC4C,IAAR,CAAaH,OAAb,EAAsB,KAAtB,EAA6B,IAA7B;AACAH,QAAAA,SAAS,GAToC,CAShC;;AAEb,eAAO,KAAP;AACD,OAZgB,CAAjB;AAaApC,MAAAA,MAAM,CAACoB,KAAP,CAAc,4BAA2Be,QAAS,eAAcC,SAAU,EAA1E,EAzBE,CA2BF;;AACA,aAAO;AAACL,QAAAA,OAAD;AAAUM,QAAAA;AAAV,OAAP;AACD,KA7BD,CA6BE,OAAOT,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAexB,UAAf,CAA0BO,KAA1B,EAAiC;AAC/BX,IAAAA,MAAM,CAACe,OAAP,CAAgB,sCAAqCJ,KAAM,EAA3D;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAGF;;AACA,YAAMqB,OAAO,GAAG,MAAMzC,OAAO,CAAC+C,GAAR,CAAYlC,KAAZ,CAAtB;;AACA,UAAI4B,OAAJ,EAAa;AACX,cAAMR,OAAO,GAAGC,aAAa,CAACO,OAAD,CAA7B;AACA,cAAMI,OAAO,GAAGjC,iBAAiB,CAAC,CAAC6B,OAAD,CAAD,CAAjC;AACA,eAAO;AAACR,UAAAA,OAAD;AAAUY,UAAAA,OAAV;AAAmBN,UAAAA,QAAQ,EAAE,CAACE,OAAD;AAA7B,SAAP;AACD;;AAED,aAAO,KAAP;AACD,KAZD,CAYE,OAAOX,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAevB,UAAf,CAA0BM,KAA1B,EAAiC;AAC/BX,IAAAA,MAAM,CAACe,OAAP,CAAgB,sCAAqCJ,KAAM,EAA3D;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAEF;;AACA,aAAO,MAAMpB,OAAO,CAAC+C,GAAR,CAAYlC,KAAZ,CAAb;AACD,KAJD,CAIE,OAAOiB,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,WAAStB,WAAT,CAAqB+B,QAArB,EAA+B;AAC7BA,IAAAA,QAAQ,CAACS,OAAT,CAAiBP,OAAO,IAAI;AAC1BvC,MAAAA,MAAM,CAACsB,KAAP,CAAc,eAAciB,OAAO,CAACC,UAAR,CAAmBI,aAAc,EAA7D;AACA9C,MAAAA,OAAO,CAACiD,GAAR,CAAYR,OAAZ;AACD,KAHD;AAID;;AAED,WAAShC,YAAT,CAAsB8B,QAAtB,EAAgC;AAC9BA,IAAAA,QAAQ,CAACS,OAAT,CAAiBP,OAAO,IAAI;AAC1BvC,MAAAA,MAAM,CAACsB,KAAP,CAAc,gBAAeiB,OAAO,CAACC,UAAR,CAAmBI,aAAc,EAA9D;AACA9C,MAAAA,OAAO,CAAC4C,IAAR,CAAaH,OAAb;AACD,KAHD;AAID;;AAED,iBAAe/B,WAAf,CAA2B;AAACG,IAAAA,KAAD;AAAQiC,IAAAA,aAAR;AAAuBb,IAAAA,OAAvB;AAAgCiB,IAAAA;AAAhC,GAA3B,EAAkE;AAChEhD,IAAAA,MAAM,CAACe,OAAP,CAAgB,oBAAmB6B,aAAc,cAAajC,KAAM,EAApE;;AACA,QAAI;AACFX,MAAAA,MAAM,CAACsB,KAAP,CAAc,SAAQX,KAAM,EAA5B;AACAX,MAAAA,MAAM,CAACsB,KAAP,CAAc,iBAAgBsB,aAAc,EAA5C;AACA5C,MAAAA,MAAM,CAACsB,KAAP,CAAc,QAAOW,IAAI,CAACC,SAAL,CAAec,IAAf,CAAqB,EAA1C;AACAhD,MAAAA,MAAM,CAACsB,KAAP,CAAc,WAAUW,IAAI,CAACC,SAAL,CAAeH,OAAf,CAAwB,EAAhD;AAEA,YAAMjC,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN;AAEApB,MAAAA,OAAO,CAACU,WAAR,CACEG,KADF,EAEEsC,MAAM,CAACC,IAAP,CAAYjB,IAAI,CAACC,SAAL,CAAe;AAACc,QAAAA;AAAD,OAAf,CAAZ,CAFF,EAGE;AACEJ,QAAAA,aADF;AAEEO,QAAAA,UAAU,EAAE,IAFd;AAGEpB,QAAAA;AAHF,OAHF;AAUA/B,MAAAA,MAAM,CAACsB,KAAP,CAAc,yBAAwBX,KAAM,EAA5C;AACD,KAnBD,CAmBE,OAAOiB,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAenB,WAAf,CAA2BE,KAA3B,EAAkC;AAChCX,IAAAA,MAAM,CAACe,OAAP,CAAgB,kBAAiBJ,KAAM,GAAvC;AACA,UAAMb,OAAO,CAACsD,WAAR,CAAoBzC,KAApB,CAAN;AACD,GA1PsC,CA4PvC;AACA;AACA;;;AAEA,WAASD,iBAAT,CAA2B2B,QAA3B,EAAqC;AACnCrC,IAAAA,MAAM,CAACe,OAAP,CAAgB,qBAAoBsB,QAAQ,CAACgB,MAAO,cAApD;AAEA,WAAOhB,QAAQ,CAACiB,GAAT,CAAaf,OAAO,IAAI;AAC7B,YAAMgB,OAAO,GAAGtB,IAAI,CAACuB,KAAL,CAAWjB,OAAO,CAACgB,OAAR,CAAgBE,QAAhB,EAAX,CAAhB;AACA,aAAO,IAAIC,sBAAJ,CAAeH,OAAO,CAACP,IAAvB,CAAP;AACD,KAHM,CAAP;AAID;;AAED,iBAAelB,OAAf,CAAuBnB,KAAvB,EAA8B;AAC5BX,IAAAA,MAAM,CAACe,OAAP,CAAgB,2BAA0BJ,KAAM,EAAhD;;AACA,QAAI;AACF,YAAM;AAACU,QAAAA;AAAD,UAAiB,MAAMvB,OAAO,CAACG,UAAR,CAAmBU,KAAnB,CAA7B;AACAX,MAAAA,MAAM,CAACsB,KAAP,CAAc,YAAWD,YAAa,sBAAqBV,KAAM,EAAjE;AACA,YAAMgD,aAAa,GAAGtC,YAAY,IAAIuC,qBAAhB,GAA6BA,qBAA7B,GAA0CvC,YAAhE;AACArB,MAAAA,MAAM,CAACsB,KAAP,CAAc,WAAUqC,aAAc,wBAAuBhD,KAAM,EAAnE;AAEA,YAAM0B,QAAQ,GAAG,MAAMwB,IAAI,CAACF,aAAD,CAA3B;AAEA3D,MAAAA,MAAM,CAACoB,KAAP,CAAc,aAAYiB,QAAQ,CAACgB,MAAO,kBAA1C;AAEA,aAAOhB,QAAP;AACD,KAXD,CAWE,OAAOT,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;;AAED,mBAAeiC,IAAf,CAAoBC,KAApB,EAA2BC,OAAO,GAAG,EAArC,EAAyCC,WAAW,GAAG,EAAvD,EAA2D;AACzD,UAAIF,KAAK,KAAK,CAAd,EAAiB;AACf,eAAOC,OAAP;AACD;;AAED,YAAMxB,OAAO,GAAG,MAAMzC,OAAO,CAAC+C,GAAR,CAAYlC,KAAZ,CAAtB;AACA,YAAMsD,UAAU,GAAG;AACjBrB,QAAAA,aAAa,EAAEL,OAAO,CAACC,UAAR,CAAmBI,aADjB;AAEjBsB,QAAAA,WAAW,EAAE3B,OAAO,CAAC4B,MAAR,CAAeD;AAFX,OAAnB,CANyD,CAUzD;;AACA,UAAIF,WAAW,CAACI,QAAZ,CAAqBH,UAArB,CAAJ,EAAsC;AACpC,eAAOJ,IAAI,CAACC,KAAK,GAAG,CAAT,EAAYC,OAAZ,EAAqBC,WAArB,CAAX;AACD;;AAED,aAAOH,IAAI,CAACC,KAAK,GAAG,CAAT,EAAYC,OAAO,CAACM,MAAR,CAAe9B,OAAf,CAAZ,EAAqCyB,WAAW,CAACK,MAAZ,CAAmBJ,UAAnB,CAArC,CAAX;AACD;AACF;;AAED,WAASjC,aAAT,CAAuBgB,IAAvB,EAA6B;AAC3B,WAAOA,IAAI,CAACR,UAAL,CAAgBT,OAAvB;AACD;AACF","sourcesContent":["/* eslint-disable max-statements */\n/* eslint-disable max-lines */\n/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport amqplib from 'amqplib';\nimport {MarcRecord} from '@natlibfi/marc-record';\nimport {Error as ApiError} from '@natlibfi/melinda-commons';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {CHUNK_SIZE} from './constants';\nimport {logError} from './utils';\nimport httpStatus from 'http-status';\n\nexport default async function (AMQP_URL) {\n  const connection = await amqplib.connect(AMQP_URL);\n  const channel = await connection.createChannel();\n  const logger = createLogger();\n\n  return {checkQueue, consumeChunk, consumeRawChunk, consumeOne, consumeRaw, ackMessages, nackMessages, sendToQueue, removeQueue, messagesToRecords};\n\n  async function checkQueue(queue, style = 'basic', purge = false, toRecord = true) {\n    logger.verbose(`checkQueue: ${queue}, Style: ${style}: Purge: ${purge}, toRecord: ${toRecord}`);\n    try {\n      const channelInfo = await channel.assertQueue(queue, {durable: true});\n      if (purge) {\n        await purgeQueue(purge);\n        logger.debug(`Queue ${queue} has purged ${channelInfo.messageCount} messages`);\n        return checkQueue(queue, style);\n      }\n\n      if (channelInfo.messageCount < 1) {\n        logger.debug(`checkQueue: ${channelInfo.messageCount} - ${queue} is empty`);\n        return false;\n      }\n      logger.silly(`Queue ${queue} has ${channelInfo.messageCount} messages`);\n\n      if (style === 'messages') {\n        return channelInfo.messageCount;\n      }\n\n      // Note: returns a message + a record\n      if (style === 'one') {\n        return consumeOne(queue);\n      }\n\n      // Note: returns just a message\n      if (style === 'raw') {\n        return consumeRaw(queue);\n      }\n\n      // Note: returns a chunk of messages + records\n      if (style === 'rawChunk') {\n        return consumeRawChunk(queue);\n      }\n\n      // Note: returns a chunk of messages or a chunk of messages + records (depending on toRecord)\n      if ((/^.{8}-.{4}-.{4}-.{4}-.{12}$/u).test(style)) {\n        logger.silly(`checkQueue: regexp match ${style}`);\n        return consumeByCorrelationId(queue, style, toRecord);\n      }\n\n      // Note: returns a chunk of messages + records\n      if (style === 'basic') {\n        return consumeChunk(queue);\n      }\n\n      // Defaults:\n      throw new ApiError(httpStatus.UNPROCESSABLE_ENTITY);\n    } catch (error) {\n      logError(error);\n    }\n\n    function purgeQueue(purge) {\n      if (purge) {\n        return channel.purgeQueue(queue);\n      }\n    }\n  }\n\n  async function consumeChunk(queue) {\n    logger.verbose(`Prepared to consumeChunk from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n\n      // getData: next chunk (100) messages\n      const queMessages = await getData(queue);\n\n      const headers = getHeaderInfo(queMessages[0]);\n      logger.debug(`Filtering messages by cataloger in ${JSON.stringify(headers)}`);\n\n      let filterIn = 0; // eslint-disable-line functional/no-let\n      let filterOut = 0; // eslint-disable-line functional/no-let\n\n      // Check that cataloger match! headers\n      const messages = queMessages.filter(message => {\n        if (message.properties.headers.cataloger === headers.cataloger) {\n          filterIn++; // eslint-disable-line no-plusplus\n          return true;\n        }\n\n        // Nack unwanted ones\n        channel.nack(message, false, true);\n        filterOut++; // eslint-disable-line no-plusplus\n        return false;\n      });\n\n      logger.debug(`Filtering result: valid: ${filterIn} non-valid: ${filterOut}`);\n      const records = await messagesToRecords(messages);\n\n      return {headers, records, messages};\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeByCorrelationId(queue, correlationId, toRecord = true) {\n    logger.verbose(`Prepared to consumeByCorrelationId from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n      // get next chunk (100) messages\n      const queMessages = await getData(queue);\n\n      logger.debug(`Filtering messages by ${correlationId}`);\n\n      let filterIn = 0; // eslint-disable-line functional/no-let\n      let filterOut = 0; // eslint-disable-line functional/no-let\n\n      // Check that correlationId matches query\n      const messages = queMessages.filter(message => {\n        if (message.properties.correlationId === correlationId) {\n          filterIn++; // eslint-disable-line no-plusplus\n          return true;\n        }\n\n        // Nack unwanted ones\n        channel.nack(message, false, true);\n        filterOut++; // eslint-disable-line no-plusplus\n        return false;\n      });\n      logger.debug(`Filtering result: valid: ${filterIn} non-valid: ${filterOut}`);\n\n      const headers = getHeaderInfo(messages[0]);\n\n      if (toRecord) {\n        const records = await messagesToRecords(messages);\n\n        return {headers, records, messages};\n      }\n      return {headers, messages};\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeRawChunk(queue) {\n    logger.verbose(`Prepared to consumeRawChunk from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n      // Get next chunk (100) messages\n      const queMessages = await getData(queue);\n\n      const headers = getHeaderInfo(queMessages[0]);\n      logger.debug(`Filtering messages by ${JSON.stringify(headers)}`);\n\n      let filterIn = 0; // eslint-disable-line functional/no-let\n      let filterOut = 0; // eslint-disable-line functional/no-let\n\n      // Check that cataloger match! headers\n      const messages = queMessages.filter(message => {\n        if (message.properties.headers.cataloger === headers.cataloger) {\n          filterIn++; // eslint-disable-line no-plusplus\n\n          return true;\n        }\n\n        // Nack unwanted ones\n        channel.nack(message, false, true);\n        filterOut++; // eslint-disable-line no-plusplus\n\n        return false;\n      });\n      logger.debug(`Filtering result: valid: ${filterIn} non-valid: ${filterOut}`);\n\n      // return raw (do not convert messages to records)\n      return {headers, messages};\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeOne(queue) {\n    logger.verbose(`Prepared to consumeOne from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n\n      // Returns false if 0 items in queue\n      const message = await channel.get(queue);\n      if (message) {\n        const headers = getHeaderInfo(message);\n        const records = messagesToRecords([message]);\n        return {headers, records, messages: [message]};\n      }\n\n      return false;\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeRaw(queue) {\n    logger.verbose(`Prepared to consumeRaw from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n      // Returns false if 0 items in queue\n      return await channel.get(queue);\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  function ackMessages(messages) {\n    messages.forEach(message => {\n      logger.silly(`Ack message ${message.properties.correlationId}`);\n      channel.ack(message);\n    });\n  }\n\n  function nackMessages(messages) {\n    messages.forEach(message => {\n      logger.silly(`Nack message ${message.properties.correlationId}`);\n      channel.nack(message);\n    });\n  }\n\n  async function sendToQueue({queue, correlationId, headers, data}) {\n    logger.verbose(`Send message for ${correlationId} to queue: ${queue}`);\n    try {\n      logger.silly(`Queue ${queue}`);\n      logger.silly(`CorrelationId ${correlationId}`);\n      logger.silly(`Data ${JSON.stringify(data)}`);\n      logger.silly(`Headers ${JSON.stringify(headers)}`);\n\n      await channel.assertQueue(queue, {durable: true});\n\n      channel.sendToQueue(\n        queue,\n        Buffer.from(JSON.stringify({data})),\n        {\n          correlationId,\n          persistent: true,\n          headers\n        }\n      );\n\n      logger.silly(`Message send to queue ${queue}`);\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function removeQueue(queue) {\n    logger.verbose(`Removing queue ${queue}.`);\n    await channel.deleteQueue(queue);\n  }\n\n  // ----------------\n  // Helper functions\n  // ----------------\n\n  function messagesToRecords(messages) {\n    logger.verbose(`Parsing messages (${messages.length}) to records`);\n\n    return messages.map(message => {\n      const content = JSON.parse(message.content.toString());\n      return new MarcRecord(content.data);\n    });\n  }\n\n  async function getData(queue) {\n    logger.verbose(`Getting queue data from ${queue}`);\n    try {\n      const {messageCount} = await channel.checkQueue(queue);\n      logger.silly(`There is ${messageCount} messages in queue ${queue}`);\n      const messagesToGet = messageCount >= CHUNK_SIZE ? CHUNK_SIZE : messageCount;\n      logger.silly(`Getting ${messagesToGet} messages from queue ${queue}`);\n\n      const messages = await pump(messagesToGet);\n\n      logger.debug(`Returning ${messages.length} unique messages`);\n\n      return messages;\n    } catch (error) {\n      logError(error);\n    }\n\n    async function pump(count, results = [], identifiers = []) {\n      if (count === 0) {\n        return results;\n      }\n\n      const message = await channel.get(queue);\n      const identifier = {\n        correlationId: message.properties.correlationId,\n        deliveryTag: message.fields.deliveryTag\n      };\n      // Filter not unique messages\n      if (identifiers.includes(identifier)) {\n        return pump(count - 1, results, identifiers);\n      }\n\n      return pump(count - 1, results.concat(message), identifiers.concat(identifier));\n    }\n  }\n\n  function getHeaderInfo(data) {\n    return data.properties.headers;\n  }\n}\n"],"file":"amqp.js"}

package/dist/constants.js

@@ -3,7 +3,7 @@
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-exports.conversionFormats = exports.CHUNK_SIZE = exports.OPERATIONS = exports.QUEUE_ITEM_STATE = void 0;
+exports.conversionFormats = exports.QUEUE_ITEM_STATE = exports.OPERATIONS = exports.CHUNK_SIZE = void 0;
 
 /**
 *

package/dist/helpers/format.js

@@ -3,8 +3,8 @@
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-exports.formatRecord = formatRecord;
 exports.BIB_FORMAT_SETTINGS = void 0;
+exports.formatRecord = formatRecord;
 
 var _marcRecord = require("@natlibfi/marc-record");
 

package/dist/index.js

@@ -11,12 +11,6 @@ var _exportNames = {
   format: true,
   logError: true
 };
-Object.defineProperty(exports, "mongoFactory", {
-  enumerable: true,
-  get: function () {
-    return _mongo.default;
-  }
-});
 Object.defineProperty(exports, "amqpFactory", {
   enumerable: true,
   get: function () {
@@ -29,19 +23,25 @@ Object.defineProperty(exports, "conversions", {
     return _conversion.default;
   }
 });
-Object.defineProperty(exports, "validations", {
+exports.format = void 0;
+Object.defineProperty(exports, "logError", {
   enumerable: true,
   get: function () {
-    return _validation.default;
+    return _utils.logError;
   }
 });
-Object.defineProperty(exports, "logError", {
+Object.defineProperty(exports, "mongoFactory", {
   enumerable: true,
   get: function () {
-    return _utils.logError;
+    return _mongo.default;
+  }
+});
+Object.defineProperty(exports, "validations", {
+  enumerable: true,
+  get: function () {
+    return _validation.default;
   }
 });
-exports.format = void 0;
 
 var _mongo = _interopRequireDefault(require("./mongo"));
 

package/dist/mongo.js

@@ -106,7 +106,10 @@ async function _default(MONGO_URI, collection) {
     correlationId,
     cataloger,
     oCatalogerIn,
-    operation
+    operation,
+    noop = undefined,
+    unique = undefined,
+    prio = undefined
   }) {
     const time = (0, _moment.default)().toDate();
     const newQueueItem = {
@@ -114,17 +117,23 @@ async function _default(MONGO_URI, collection) {
       cataloger,
       operation,
       oCatalogerIn,
+      operationSettings: {
+        unique,
+        noop,
+        prio
+      },
       queueItemState: _constants.QUEUE_ITEM_STATE.VALIDATOR.PENDING_VALIDATION,
       creationTime: time,
       modificationTime: time,
-      handledIds: []
+      handledIds: [],
+      rejectedIds: []
     };
 
     try {
       const result = await db.collection(collection).insertOne(newQueueItem);
 
       if (result.acknowledged) {
-        logger.debug(`Created new prio queue item ${correlationId} in ${collection}`);
+        logger.info(`New prio queue item for ${correlationId} has been made in ${collection}`);
         return;
       }
 
@@ -155,13 +164,14 @@ async function _default(MONGO_URI, collection) {
       queueItemState: _constants.QUEUE_ITEM_STATE.VALIDATOR.UPLOADING,
       creationTime: time,
       modificationTime: time,
-      handledIds: []
+      handledIds: [],
+      rejectedIds: []
     };
 
     try {
       // No await here, promises later
       db.collection(collection).insertOne(newQueueItem);
-      logger.log('info', `New bulk queue item for ${correlationId} has been made in ${collection}!`);
+      logger.info(`New bulk queue item for ${correlationId} has been made in ${collection}!`);
     } catch (error) {
       (0, _utils.logError)(error);
       throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR);
@@ -204,7 +214,7 @@ async function _default(MONGO_URI, collection) {
         _id: 0
       }
     }).toArray();
-    logger.info(`Query result: ${result.length > 0 ? 'Found!' : 'Not found!'}`);
+    logger.debug(`Query result: ${result.length > 0 ? 'Found!' : 'Not found!'}`);
     return result;
   }
 
@@ -227,7 +237,6 @@ async function _default(MONGO_URI, collection) {
   } // Check that if the item has waited too long
   // If the last modification time for the queueItem is older than 1 minute
   // set state to ABORT and return false, otherwise return true
-  // This could also add information about where the timeout occured
 
 
   async function checkTimeOut(correlationId) {
@@ -238,11 +247,12 @@ async function _default(MONGO_URI, collection) {
     logger.silly(`timeOut @ ${timeoutTime}`);
 
     if (timeoutTime.isBefore()) {
+      const oldState = result.queueItemState;
       await setState({
         correlationId,
         state: _constants.QUEUE_ITEM_STATE.ABORT,
         errorStatus: _httpStatus.default.REQUEST_TIMEOUT,
-        errorMessage: 'Timeout'
+        errorMessage: `Timeout in ${oldState}`
       });
       return false;
     }
@@ -254,7 +264,7 @@ async function _default(MONGO_URI, collection) {
     logger.debug(`${JSON.stringify(params)}`);
     logger.info(`Removing from Mongo (${collection}) id: ${params.correlationId}`);
     const clean = (0, _mongoSanitize.default)(params.correlationId);
-    logger.debug(`mongo/remove: clean: ${JSON.stringify(clean)}`);
+    logger.silly(`mongo/remove: clean: ${JSON.stringify(clean)}`);
 
     try {
       //const metadataResult = await getFileMetadata({filename: clean});
@@ -338,7 +348,7 @@ async function _default(MONGO_URI, collection) {
       const clean2 = {
         operation: (0, _mongoSanitize.default)(operation)
       };
-      logger.log('silly', `Checking DB ${collection} for ${clean.queueItemState} + ${clean2.operation}`);
+      logger.silly(`Checking DB ${collection} for ${clean.queueItemState} + ${clean2.operation}`);
       return db.collection(collection).findOne({ ...clean,
         ...clean2
       });
@@ -364,7 +374,7 @@ async function _default(MONGO_URI, collection) {
     handledIds,
     rejectedIds
   }) {
-    logger.debug(`Push queue-item ids to list: ${correlationId} to ${collection}`);
+    logger.info(`Push queue-item ids to list: ${correlationId} to ${collection}`);
     logger.debug(`ids: ${JSON.stringify(handledIds)}, rejectedIds: ${JSON.stringify(rejectedIds)}`);
     const clean = (0, _mongoSanitize.default)(correlationId);
     await db.collection(collection).updateOne({
@@ -389,7 +399,7 @@ async function _default(MONGO_URI, collection) {
     messages,
     messageField = 'messages'
   }) {
-    logger.debug(`Push messages to ${messageField} ${correlationId} to ${collection}`);
+    logger.info(`Push messages to ${messageField} ${correlationId} to ${collection}`);
     logger.debug(`Messages: ${JSON.stringify(messages)}}`);
     const clean = (0, _mongoSanitize.default)(correlationId);
     const cleanMessageField = (0, _mongoSanitize.default)(messageField);
@@ -413,7 +423,8 @@ async function _default(MONGO_URI, collection) {
     errorMessage = '',
     errorStatus = ''
   }) {
-    logger.log('info', `Setting queue-item state: ${correlationId}, ${state}, Error message: '${errorMessage}', Error status: '${errorStatus}' to ${collection}`);
+    const errorString = errorMessage || errorStatus ? `, Error message: '${errorMessage}', Error status: '${errorStatus}'` : '';
+    logger.log('info', `Setting queue-item state: ${correlationId}${state}${errorString} to ${collection}`);
     const clean = (0, _mongoSanitize.default)(correlationId);
     return db.collection(collection).findOneAndUpdate({
       correlationId: clean

package/dist/mongo.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/mongo.js"],"names":["MONGO_URI","collection","logger","client","MongoClient","connect","useNewUrlParser","useUnifiedTopology","db","gridFSBucket","GridFSBucket","bucketName","createPrio","createBulk","checkAndSetState","query","queryById","remove","readContent","removeContent","getOne","getStream","setState","pushIds","pushMessages","correlationId","cataloger","oCatalogerIn","operation","time","toDate","newQueueItem","queueItemState","QUEUE_ITEM_STATE","VALIDATOR","PENDING_VALIDATION","creationTime","modificationTime","handledIds","result","insertOne","acknowledged","debug","ApiError","httpStatus","INTERNAL_SERVER_ERROR","error","contentType","recordLoadParams","stream","UPLOADING","log","Promise","resolve","reject","outputStream","openUploadStream","on","chunk","write","end","undefined","state","errorMessage","errorStatus","timeOut","checkTimeOut","params","find","projection","_id","toArray","info","length","checkModTime","findOne","timeoutTime","add","silly","isBefore","ABORT","REQUEST_TIMEOUT","JSON","stringify","clean","noContent","deleteOne","err","MongoDriverError","message","indexOf","readStream","openDownloadStreamByName","NOT_FOUND","delete","clean2","rejectedIds","updateOne","$set","$push","$each","messages","messageField","cleanMessageField","findOneAndUpdate","returnNewDocument"],"mappings":";;;;;;;AA8BA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AArCA;;AACA;;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAWA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAEe,wBAAgBA,SAAhB,EAA2BC,UAA3B,EAAuC;AACpD,QAAMC,MAAM,GAAG,0CAAf,CADoD,CAGpD;;AACA,QAAMC,MAAM,GAAG,MAAMC,qBAAYC,OAAZ,CAAoBL,SAApB,EAA+B;AAACM,IAAAA,eAAe,EAAE,IAAlB;AAAwBC,IAAAA,kBAAkB,EAAE;AAA5C,GAA/B,CAArB;AACA,QAAMC,EAAE,GAAGL,MAAM,CAACK,EAAP,CAAU,UAAV,CAAX;AACA,QAAMC,YAAY,GAAG,IAAIC,qBAAJ,CAAiBF,EAAjB,EAAqB;AAACG,IAAAA,UAAU,EAAEV;AAAb,GAArB,CAArB;AAEA,SAAO;AAACW,IAAAA,UAAD;AAAaC,IAAAA,UAAb;AAAyBC,IAAAA,gBAAzB;AAA2CC,IAAAA,KAA3C;AAAkDC,IAAAA,SAAlD;AAA6DC,IAAAA,MAA7D;AAAqEC,IAAAA,WAArE;AAAkFC,IAAAA,aAAlF;AAAiGC,IAAAA,MAAjG;AAAyGC,IAAAA,SAAzG;AAAoHC,IAAAA,QAApH;AAA8HC,IAAAA,OAA9H;AAAuIC,IAAAA;AAAvI,GAAP;;AAEA,iBAAeZ,UAAf,CAA0B;AAACa,IAAAA,aAAD;AAAgBC,IAAAA,SAAhB;AAA2BC,IAAAA,YAA3B;AAAyCC,IAAAA;AAAzC,GAA1B,EAA+E;AAC7E,UAAMC,IAAI,GAAG,uBAASC,MAAT,EAAb;AACA,UAAMC,YAAY,GAAG;AACnBN,MAAAA,aADmB;AAEnBC,MAAAA,SAFmB;AAGnBE,MAAAA,SAHmB;AAInBD,MAAAA,YAJmB;AAKnBK,MAAAA,cAAc,EAAEC,4BAAiBC,SAAjB,CAA2BC,kBALxB;AAMnBC,MAAAA,YAAY,EAAEP,IANK;AAOnBQ,MAAAA,gBAAgB,EAAER,IAPC;AAQnBS,MAAAA,UAAU,EAAE;AARO,KAArB;;AAUA,QAAI;AACF,YAAMC,MAAM,GAAG,MAAM/B,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BuC,SAA1B,CAAoCT,YAApC,CAArB;;AACA,UAAIQ,MAAM,CAACE,YAAX,EAAyB;AACvBvC,QAAAA,MAAM,CAACwC,KAAP,CAAc,+BAA8BjB,aAAc,OAAMxB,UAAW,EAA3E;AACA;AACD;;AACD,YAAM,IAAI0C,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD,KAPD,CAOE,OAAOC,KAAP,EAAc;AACd,2BAASA,KAAT;AACA,YAAM,IAAIH,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD;AACF;;AAED,WAAShC,UAAT,CAAoB;AAACY,IAAAA,aAAD;AAAgBC,IAAAA,SAAhB;AAA2BC,IAAAA,YAA3B;AAAyCC,IAAAA,SAAzC;AAAoDmB,IAAAA,WAApD;AAAiEC,IAAAA,gBAAjE;AAAmFC,IAAAA;AAAnF,GAApB,EAAgH;AAC9G,UAAMpB,IAAI,GAAG,uBAASC,MAAT,EAAb;AACA,UAAMC,YAAY,GAAG;AACnBN,MAAAA,aADmB;AAEnBC,MAAAA,SAFmB;AAGnBC,MAAAA,YAHmB;AAInBC,MAAAA,SAJmB;AAKnBmB,MAAAA,WALmB;AAMnBC,MAAAA,gBANmB;AAOnBhB,MAAAA,cAAc,EAAEC,4BAAiBC,SAAjB,CAA2BgB,SAPxB;AAQnBd,MAAAA,YAAY,EAAEP,IARK;AASnBQ,MAAAA,gBAAgB,EAAER,IATC;AAUnBS,MAAAA,UAAU,EAAE;AAVO,KAArB;;AAYA,QAAI;AACF;AACA9B,MAAAA,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BuC,SAA1B,CAAoCT,YAApC;AACA7B,MAAAA,MAAM,CAACiD,GAAP,CAAW,MAAX,EAAoB,2BAA0B1B,aAAc,qBAAoBxB,UAAW,GAA3F;AACD,KAJD,CAIE,OAAO6C,KAAP,EAAc;AACd,2BAASA,KAAT;AACA,YAAM,IAAIH,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD;;AAED,WAAO,IAAIO,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtC,YAAMC,YAAY,GAAG9C,YAAY,CAAC+C,gBAAb,CAA8B/B,aAA9B,CAArB;AAEAwB,MAAAA,MAAM,CACHQ,EADH,CACM,OADN,EACeH,MADf,EAEGG,EAFH,CAEM,MAFN,EAEcC,KAAK,IAAIH,YAAY,CAACI,KAAb,CAAmBD,KAAnB,CAFvB,EAGGD,EAHH,CAGM,KAHN,EAGa,MAAMF,YAAY,CAACK,GAAb,CAAiBC,SAAjB,EAA4BA,SAA5B,EAAuC,MAAM;AAC5DR,QAAAA,OAAO,CAAC5B,aAAD,CAAP;AACD,OAFgB,CAHnB;AAMD,KATM,CAAP;AAUD,GApEmD,CAsEpD;;;AACA,iBAAeX,gBAAf,CAAgC;AAACW,IAAAA,aAAD;AAAgBqC,IAAAA,KAAhB;AAAuBC,IAAAA,YAAY,GAAG,EAAtC;AAA0CC,IAAAA,WAAW,GAAG;AAAxD,GAAhC,EAA6F;AAC3F;AACA;AACA,UAAMC,OAAO,GAAG,MAAMC,YAAY,CAACzC,aAAD,CAAlC;;AACA,QAAIwC,OAAJ,EAAa;AACX,aAAO3C,QAAQ,CAAC;AAACG,QAAAA,aAAD;AAAgBqC,QAAAA,KAAhB;AAAuBC,QAAAA,YAAvB;AAAqCC,QAAAA;AAArC,OAAD,CAAf;AACD;;AACD,WAAO,KAAP;AACD;;AAED,iBAAejD,KAAf,CAAqBoD,MAArB,EAA6B;AAC3B,UAAM5B,MAAM,GAAG,MAAM/B,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BmE,IAA1B,CAA+BD,MAA/B,EAAuC;AAACE,MAAAA,UAAU,EAAE;AAACC,QAAAA,GAAG,EAAE;AAAN;AAAb,KAAvC,EAClBC,OADkB,EAArB;AAEArE,IAAAA,MAAM,CAACsE,IAAP,CAAa,iBAAgBjC,MAAM,CAACkC,MAAP,GAAgB,CAAhB,GAAoB,QAApB,GAA+B,YAAa,EAAzE;AACA,WAAOlC,MAAP;AACD;;AAED,iBAAevB,SAAf,CAAyBS,aAAzB,EAAwCiD,YAAxC,EAAsD;AACpD,UAAMnC,MAAM,GAAG,MAAM/B,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B0E,OAA1B,CAAkC;AAAClD,MAAAA;AAAD,KAAlC,CAArB;;AACA,QAAIiD,YAAJ,EAAkB;AAChB,YAAMT,OAAO,GAAG,MAAMC,YAAY,CAACzC,aAAD,CAAlC;;AACA,UAAIwC,OAAJ,EAAa;AACX,eAAOjD,SAAS,CAACS,aAAD,CAAhB;AACD;;AACD,aAAOc,MAAP;AACD;;AAED,WAAOA,MAAP;AACD,GAnGmD,CAqGpD;AACA;AACA;AACA;;;AACA,iBAAe2B,YAAf,CAA4BzC,aAA5B,EAA2C;AACzC,UAAMc,MAAM,GAAG,MAAM/B,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B0E,OAA1B,CAAkC;AAAClD,MAAAA;AAAD,KAAlC,CAArB;AACA,UAAMmD,WAAW,GAAG,qBAAOrC,MAAM,CAACF,gBAAd,EAAgCwC,GAAhC,CAAoC,CAApC,EAAuC,GAAvC,CAApB;AACA3E,IAAAA,MAAM,CAAC4E,KAAP,CAAc,aAAYF,WAAY,EAAtC;;AACA,QAAIA,WAAW,CAACG,QAAZ,EAAJ,EAA4B;AAC1B,YAAMzD,QAAQ,CAAC;AAACG,QAAAA,aAAD;AAAgBqC,QAAAA,KAAK,EAAE7B,4BAAiB+C,KAAxC;AAA+ChB,QAAAA,WAAW,EAAEpB,oBAAWqC,eAAvE;AAAwFlB,QAAAA,YAAY,EAAE;AAAtG,OAAD,CAAd;AACA,aAAO,KAAP;AACD;;AAED,WAAO,IAAP;AACD;;AAED,iBAAe9C,MAAf,CAAsBkD,MAAtB,EAA8B;AAC5BjE,IAAAA,MAAM,CAACwC,KAAP,CAAc,GAAEwC,IAAI,CAACC,SAAL,CAAehB,MAAf,CAAuB,EAAvC;AACAjE,IAAAA,MAAM,CAACsE,IAAP,CAAa,wBAAuBvE,UAAW,SAAQkE,MAAM,CAAC1C,aAAc,EAA5E;AACA,UAAM2D,KAAK,GAAG,4BAASjB,MAAM,CAAC1C,aAAhB,CAAd;AACAvB,IAAAA,MAAM,CAACwC,KAAP,CAAc,wBAAuBwC,IAAI,CAACC,SAAL,CAAeC,KAAf,CAAsB,EAA3D;;AAEA,QAAI;AACF;AACA;AACA,YAAMC,SAAS,GAAG,MAAMlE,aAAa,CAACgD,MAAD,CAArC;;AACA,UAAIkB,SAAJ,EAAe;AACb,cAAM7E,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BqF,SAA1B,CAAoC;AAAC7D,UAAAA,aAAa,EAAE2D;AAAhB,SAApC,CAAN;AACA,eAAO,IAAP;AACD;AACF,KARD,CAQE,OAAOG,GAAP,EAAY;AACZ,UAAIA,GAAG,YAAYC,yBAAnB,EAAqC;AACnCtF,QAAAA,MAAM,CAAC4C,KAAP,CAAayC,GAAG,CAACE,OAAjB;;AACA,YAAIF,GAAG,CAACE,OAAJ,CAAYC,OAAZ,CAAoB,uBAApB,MAAiD,CAAC,CAAtD,EAAyD;AACvDxF,UAAAA,MAAM,CAACwC,KAAP,CAAc,oDAAmDwC,IAAI,CAACC,SAAL,CAAeC,KAAf,CAAsB,SAAQnF,UAAW,EAA1G;AACA,gBAAMO,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BqF,SAA1B,CAAoC;AAAC7D,YAAAA,aAAa,EAAE2D;AAAhB,WAApC,CAAN;AACA,iBAAO,IAAP;AACD;;AACD,cAAM,IAAIzC,qBAAJ,CAAaC,oBAAWC,qBAAxB,EAA+C0C,GAAG,CAACE,OAAnD,CAAN;AACD;;AACD,YAAM,IAAI9C,qBAAJ,CAAaC,oBAAWC,qBAAxB,EAA+C0C,GAAG,CAACE,OAAnD,CAAN;AACD;AACF;;AAED,iBAAevE,WAAf,CAA2BO,aAA3B,EAA0C;AACxCvB,IAAAA,MAAM,CAACsE,IAAP,CAAa,2BAA0B/C,aAAc,OAAMxB,UAAW,EAAtE;AACA,UAAMmF,KAAK,GAAG,4BAAS3D,aAAT,CAAd;AACA,UAAMc,MAAM,GAAG,MAAM/B,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B0E,OAA1B,CAAkC;AAAClD,MAAAA,aAAa,EAAE2D;AAAhB,KAAlC,CAArB,CAHwC,CAGwC;;AAEhF,QAAI7C,MAAJ,EAAY;AACV,aAAO;AACLQ,QAAAA,WAAW,EAAER,MAAM,CAACQ,WADf;AAEL4C,QAAAA,UAAU,EAAElF,YAAY,CAACmF,wBAAb,CAAsCR,KAAtC;AAFP,OAAP;AAID;;AAED,UAAM,IAAIzC,qBAAJ,CAAaC,oBAAWiD,SAAxB,CAAN;AACD;;AAED,iBAAe1E,aAAf,CAA6BgD,MAA7B,EAAqC;AACnCjE,IAAAA,MAAM,CAACsE,IAAP,CAAa,4BAA2BL,MAAM,CAAC1C,aAAc,OAAMxB,UAAW,EAA9E;AACA,UAAMmF,KAAK,GAAG,4BAASjB,MAAM,CAAC1C,aAAhB,CAAd;AAEA,UAAMc,MAAM,GAAG,MAAM/B,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B0E,OAA1B,CAAkC;AAAClD,MAAAA,aAAa,EAAE2D;AAAhB,KAAlC,CAArB,CAJmC,CAI6C;;AAChFlF,IAAAA,MAAM,CAACwC,KAAP,CAAc,+BAA8BwC,IAAI,CAACC,SAAL,CAAe5C,MAAf,CAAuB,EAAnE;;AAEA,QAAIA,MAAJ,EAAY;AACV,YAAM9B,YAAY,CAACqF,MAAb,CAAoBV,KAApB,CAAN;AACA,aAAO,IAAP;AACD;;AAED,WAAO,IAAP;AACD;;AAED,WAAShE,MAAT,CAAgB;AAACQ,IAAAA,SAAD;AAAYI,IAAAA;AAAZ,GAAhB,EAA6C;AAC3C,UAAMoD,KAAK,GAAG;AAACpD,MAAAA,cAAc,EAAE,4BAASA,cAAT;AAAjB,KAAd;;AACA,QAAI;AACF,UAAIJ,SAAS,KAAKiC,SAAlB,EAA6B;AAC3B3D,QAAAA,MAAM,CAAC4E,KAAP,CAAc,eAAc7E,UAAW,QAAOiF,IAAI,CAACC,SAAL,CAAeC,KAAK,CAACpD,cAArB,CAAqC,EAAnF;AACA,eAAOxB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B0E,OAA1B,CAAkC,EAAC,GAAGS;AAAJ,SAAlC,CAAP;AACD;;AAED,YAAMW,MAAM,GAAG;AAACnE,QAAAA,SAAS,EAAE,4BAASA,SAAT;AAAZ,OAAf;AACA1B,MAAAA,MAAM,CAACiD,GAAP,CAAW,OAAX,EAAqB,eAAclD,UAAW,QAAOmF,KAAK,CAACpD,cAAe,MAAK+D,MAAM,CAACnE,SAAU,EAAhG;AACA,aAAOpB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B0E,OAA1B,CAAkC,EAAC,GAAGS,KAAJ;AAAW,WAAGW;AAAd,OAAlC,CAAP;AACD,KATD,CASE,OAAOjD,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,WAASzB,SAAT,CAAmBI,aAAnB,EAAkC;AAChCvB,IAAAA,MAAM,CAACsE,IAAP,CAAa,2BAA0B/C,aAAc,OAAMxB,UAAW,EAAtE;AACA,UAAMmF,KAAK,GAAG,4BAAS3D,aAAT,CAAd;;AACA,QAAI;AACF;AACA,aAAOhB,YAAY,CAACmF,wBAAb,CAAsCR,KAAtC,CAAP;AACD,KAHD,CAGE,OAAOtC,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAevB,OAAf,CAAuB;AAACE,IAAAA,aAAD;AAAgBa,IAAAA,UAAhB;AAA4B0D,IAAAA;AAA5B,GAAvB,EAAiE;AAC/D9F,IAAAA,MAAM,CAACwC,KAAP,CAAc,gCAA+BjB,aAAc,OAAMxB,UAAW,EAA5E;AACAC,IAAAA,MAAM,CAACwC,KAAP,CAAc,QAAOwC,IAAI,CAACC,SAAL,CAAe7C,UAAf,CAA2B,kBAAiB4C,IAAI,CAACC,SAAL,CAAea,WAAf,CAA4B,EAA7F;AACA,UAAMZ,KAAK,GAAG,4BAAS3D,aAAT,CAAd;AACA,UAAMjB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BgG,SAA1B,CAAoC;AACxCxE,MAAAA,aAAa,EAAE2D;AADyB,KAApC,EAEH;AACDc,MAAAA,IAAI,EAAE;AACJ7D,QAAAA,gBAAgB,EAAE,uBAASP,MAAT;AADd,OADL;AAIDqE,MAAAA,KAAK,EAAE;AACL7D,QAAAA,UAAU,EAAE;AAAC8D,UAAAA,KAAK,EAAE9D;AAAR,SADP;AAEL0D,QAAAA,WAAW,EAAE;AAACI,UAAAA,KAAK,EAAEJ;AAAR;AAFR;AAJN,KAFG,CAAN;AAWD;;AAED,iBAAexE,YAAf,CAA4B;AAACC,IAAAA,aAAD;AAAgB4E,IAAAA,QAAhB;AAA0BC,IAAAA,YAAY,GAAG;AAAzC,GAA5B,EAAkF;AAChFpG,IAAAA,MAAM,CAACwC,KAAP,CAAc,oBAAmB4D,YAAa,IAAG7E,aAAc,OAAMxB,UAAW,EAAhF;AACAC,IAAAA,MAAM,CAACwC,KAAP,CAAc,aAAYwC,IAAI,CAACC,SAAL,CAAekB,QAAf,CAAyB,GAAnD;AACA,UAAMjB,KAAK,GAAG,4BAAS3D,aAAT,CAAd;AACA,UAAM8E,iBAAiB,GAAG,4BAASD,YAAT,CAA1B;AACA,UAAM9F,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BgG,SAA1B,CAAoC;AACxCxE,MAAAA,aAAa,EAAE2D;AADyB,KAApC,EAEH;AACDc,MAAAA,IAAI,EAAE;AACJ7D,QAAAA,gBAAgB,EAAE,uBAASP,MAAT;AADd,OADL;AAIDqE,MAAAA,KAAK,EAAE;AACL,SAACI,iBAAD,GAAqB;AAACH,UAAAA,KAAK,EAAEC;AAAR;AADhB;AAJN,KAFG,CAAN;AAUD;;AAED,WAAS/E,QAAT,CAAkB;AAACG,IAAAA,aAAD;AAAgBqC,IAAAA,KAAhB;AAAuBC,IAAAA,YAAY,GAAG,EAAtC;AAA0CC,IAAAA,WAAW,GAAG;AAAxD,GAAlB,EAA+E;AAC7E9D,IAAAA,MAAM,CAACiD,GAAP,CAAW,MAAX,EAAoB,6BAA4B1B,aAAc,KAAIqC,KAAM,qBAAoBC,YAAa,qBAAoBC,WAAY,QAAO/D,UAAW,EAA3J;AACA,UAAMmF,KAAK,GAAG,4BAAS3D,aAAT,CAAd;AACA,WAAOjB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BuG,gBAA1B,CAA2C;AAChD/E,MAAAA,aAAa,EAAE2D;AADiC,KAA3C,EAEJ;AACDc,MAAAA,IAAI,EAAE;AACJlE,QAAAA,cAAc,EAAE8B,KADZ;AAEJzB,QAAAA,gBAAgB,EAAE,uBAASP,MAAT,EAFd;AAGJiC,QAAAA,YAHI;AAIJC,QAAAA;AAJI;AADL,KAFI,EASJ;AAACK,MAAAA,UAAU,EAAE;AAACC,QAAAA,GAAG,EAAE;AAAN,OAAb;AAAuBmC,MAAAA,iBAAiB,EAAE;AAA1C,KATI,CAAP;AAUD;AACF","sourcesContent":["/* eslint-disable max-lines */\n/* eslint-disable max-statements */\n/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport {MongoClient, GridFSBucket, MongoDriverError} from 'mongodb';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {Error as ApiError} from '@natlibfi/melinda-commons';\nimport {QUEUE_ITEM_STATE} from './constants';\nimport {logError} from './utils.js';\nimport moment from 'moment';\nimport httpStatus from 'http-status';\nimport sanitize from 'mongo-sanitize';\n\n/* QueueItem:\n{\n  \"correlationId\":\"FOO\",\n  \"cataloger\":\"xxx0000\",\n  \"oCatalogerIn\":\"xxx0000\"\n  \"operation\":\"UPDATE\",\n  \"contentType\":\"application/json\",\n  \"recordLoadParams\": {\n    \"pActiveLibrary\": \"XXX00\",\n    \"pInputFile\": \"filename.seq\",\n    \"pRejectFile\": \"filename.rej\",\n    \"pLogFile\": \"filename.syslog\",\n    \"pOldNew\": \"NEW\"\n  },\n  \"queueItemState\":\"PENDING_QUEUING\",\n  \"creationTime\":\"2020-01-01T00:00:00.000Z\",\n  \"modificationTime\":\"2020-01-01T00:00:01.000Z\",\n  \"handledIds\": []\n}\n*/\n\nexport default async function (MONGO_URI, collection) {\n  const logger = createLogger();\n\n  // Connect to mongo (MONGO)\n  const client = await MongoClient.connect(MONGO_URI, {useNewUrlParser: true, useUnifiedTopology: true});\n  const db = client.db('rest-api');\n  const gridFSBucket = new GridFSBucket(db, {bucketName: collection});\n\n  return {createPrio, createBulk, checkAndSetState, query, queryById, remove, readContent, removeContent, getOne, getStream, setState, pushIds, pushMessages};\n\n  async function createPrio({correlationId, cataloger, oCatalogerIn, operation}) {\n    const time = moment().toDate();\n    const newQueueItem = {\n      correlationId,\n      cataloger,\n      operation,\n      oCatalogerIn,\n      queueItemState: QUEUE_ITEM_STATE.VALIDATOR.PENDING_VALIDATION,\n      creationTime: time,\n      modificationTime: time,\n      handledIds: []\n    };\n    try {\n      const result = await db.collection(collection).insertOne(newQueueItem);\n      if (result.acknowledged) {\n        logger.debug(`Created new prio queue item ${correlationId} in ${collection}`);\n        return;\n      }\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    } catch (error) {\n      logError(error);\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    }\n  }\n\n  function createBulk({correlationId, cataloger, oCatalogerIn, operation, contentType, recordLoadParams, stream}) {\n    const time = moment().toDate();\n    const newQueueItem = {\n      correlationId,\n      cataloger,\n      oCatalogerIn,\n      operation,\n      contentType,\n      recordLoadParams,\n      queueItemState: QUEUE_ITEM_STATE.VALIDATOR.UPLOADING,\n      creationTime: time,\n      modificationTime: time,\n      handledIds: []\n    };\n    try {\n      // No await here, promises later\n      db.collection(collection).insertOne(newQueueItem);\n      logger.log('info', `New bulk queue item for ${correlationId} has been made in ${collection}!`);\n    } catch (error) {\n      logError(error);\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    }\n\n    return new Promise((resolve, reject) => {\n      const outputStream = gridFSBucket.openUploadStream(correlationId);\n\n      stream\n        .on('error', reject)\n        .on('data', chunk => outputStream.write(chunk))\n        .on('end', () => outputStream.end(undefined, undefined, () => {\n          resolve(correlationId);\n        }));\n    });\n  }\n\n  // Check state that the queueItem has not waited too long and set state\n  async function checkAndSetState({correlationId, state, errorMessage = '', errorStatus = ''}) {\n    // checkTimeOut returns true, if queueItem is fresher than 1 minute\n    // otherwise it sets queueItem to state ABORT (408, 'Timeout')\n    const timeOut = await checkTimeOut(correlationId);\n    if (timeOut) {\n      return setState({correlationId, state, errorMessage, errorStatus});\n    }\n    return false;\n  }\n\n  async function query(params) {\n    const result = await db.collection(collection).find(params, {projection: {_id: 0}})\n      .toArray();\n    logger.info(`Query result: ${result.length > 0 ? 'Found!' : 'Not found!'}`);\n    return result;\n  }\n\n  async function queryById(correlationId, checkModTime) {\n    const result = await db.collection(collection).findOne({correlationId});\n    if (checkModTime) {\n      const timeOut = await checkTimeOut(correlationId);\n      if (timeOut) {\n        return queryById(correlationId);\n      }\n      return result;\n    }\n\n    return result;\n  }\n\n  // Check that if the item has waited too long\n  // If the last modification time for the queueItem is older than 1 minute\n  // set state to ABORT and return false, otherwise return true\n  // This could also add information about where the timeout occured\n  async function checkTimeOut(correlationId) {\n    const result = await db.collection(collection).findOne({correlationId});\n    const timeoutTime = moment(result.modificationTime).add(1, 'm');\n    logger.silly(`timeOut @ ${timeoutTime}`);\n    if (timeoutTime.isBefore()) {\n      await setState({correlationId, state: QUEUE_ITEM_STATE.ABORT, errorStatus: httpStatus.REQUEST_TIMEOUT, errorMessage: 'Timeout'});\n      return false;\n    }\n\n    return true;\n  }\n\n  async function remove(params) {\n    logger.debug(`${JSON.stringify(params)}`);\n    logger.info(`Removing from Mongo (${collection}) id: ${params.correlationId}`);\n    const clean = sanitize(params.correlationId);\n    logger.debug(`mongo/remove: clean: ${JSON.stringify(clean)}`);\n\n    try {\n      //const metadataResult = await getFileMetadata({filename: clean});\n      //logger.debug(`mongo/remove: metadataResult: ${JSON.stringify(metadataResult)}`);\n      const noContent = await removeContent(params);\n      if (noContent) {\n        await db.collection(collection).deleteOne({correlationId: clean});\n        return true;\n      }\n    } catch (err) {\n      if (err instanceof MongoDriverError) {\n        logger.error(err.message);\n        if (err.message.indexOf('File not found for id') !== -1) {\n          logger.debug(`mongo/remove: File not found, removing queueItem ${JSON.stringify(clean)} from ${collection}`);\n          await db.collection(collection).deleteOne({correlationId: clean});\n          return true;\n        }\n        throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n      }\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n    }\n  }\n\n  async function readContent(correlationId) {\n    logger.info(`Reading content for id: ${correlationId} in ${collection}`);\n    const clean = sanitize(correlationId);\n    const result = await db.collection(collection).findOne({correlationId: clean}); // njsscan-ignore: node_nosqli_injection\n\n    if (result) {\n      return {\n        contentType: result.contentType,\n        readStream: gridFSBucket.openDownloadStreamByName(clean)\n      };\n    }\n\n    throw new ApiError(httpStatus.NOT_FOUND);\n  }\n\n  async function removeContent(params) {\n    logger.info(`Removing content for id: ${params.correlationId} in ${collection}`);\n    const clean = sanitize(params.correlationId);\n\n    const result = await db.collection(collection).findOne({correlationId: clean}); // njsscan-ignore: node_nosqli_injection\n    logger.debug(`mongo/removeContent: result ${JSON.stringify(result)}`);\n\n    if (result) {\n      await gridFSBucket.delete(clean);\n      return true;\n    }\n\n    return true;\n  }\n\n  function getOne({operation, queueItemState}) {\n    const clean = {queueItemState: sanitize(queueItemState)};\n    try {\n      if (operation === undefined) {\n        logger.silly(`Checking DB ${collection} for ${JSON.stringify(clean.queueItemState)}`);\n        return db.collection(collection).findOne({...clean});\n      }\n\n      const clean2 = {operation: sanitize(operation)};\n      logger.log('silly', `Checking DB ${collection} for ${clean.queueItemState} + ${clean2.operation}`);\n      return db.collection(collection).findOne({...clean, ...clean2});\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  function getStream(correlationId) {\n    logger.info(`Forming stream from db: ${correlationId} in ${collection}`);\n    const clean = sanitize(correlationId);\n    try {\n      // Return content stream\n      return gridFSBucket.openDownloadStreamByName(clean);\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function pushIds({correlationId, handledIds, rejectedIds}) {\n    logger.debug(`Push queue-item ids to list: ${correlationId} to ${collection}`);\n    logger.debug(`ids: ${JSON.stringify(handledIds)}, rejectedIds: ${JSON.stringify(rejectedIds)}`);\n    const clean = sanitize(correlationId);\n    await db.collection(collection).updateOne({\n      correlationId: clean\n    }, {\n      $set: {\n        modificationTime: moment().toDate()\n      },\n      $push: {\n        handledIds: {$each: handledIds},\n        rejectedIds: {$each: rejectedIds}\n      }\n    });\n  }\n\n  async function pushMessages({correlationId, messages, messageField = 'messages'}) {\n    logger.debug(`Push messages to ${messageField} ${correlationId} to ${collection}`);\n    logger.debug(`Messages: ${JSON.stringify(messages)}}`);\n    const clean = sanitize(correlationId);\n    const cleanMessageField = sanitize(messageField);\n    await db.collection(collection).updateOne({\n      correlationId: clean\n    }, {\n      $set: {\n        modificationTime: moment().toDate()\n      },\n      $push: {\n        [cleanMessageField]: {$each: messages}\n      }\n    });\n  }\n\n  function setState({correlationId, state, errorMessage = '', errorStatus = ''}) {\n    logger.log('info', `Setting queue-item state: ${correlationId}, ${state}, Error message: '${errorMessage}', Error status: '${errorStatus}' to ${collection}`);\n    const clean = sanitize(correlationId);\n    return db.collection(collection).findOneAndUpdate({\n      correlationId: clean\n    }, {\n      $set: {\n        queueItemState: state,\n        modificationTime: moment().toDate(),\n        errorMessage,\n        errorStatus\n      }\n    }, {projection: {_id: 0}, returnNewDocument: true});\n  }\n}\n"],"file":"mongo.js"}
+{"version":3,"sources":["../src/mongo.js"],"names":["MONGO_URI","collection","logger","client","MongoClient","connect","useNewUrlParser","useUnifiedTopology","db","gridFSBucket","GridFSBucket","bucketName","createPrio","createBulk","checkAndSetState","query","queryById","remove","readContent","removeContent","getOne","getStream","setState","pushIds","pushMessages","correlationId","cataloger","oCatalogerIn","operation","noop","undefined","unique","prio","time","toDate","newQueueItem","operationSettings","queueItemState","QUEUE_ITEM_STATE","VALIDATOR","PENDING_VALIDATION","creationTime","modificationTime","handledIds","rejectedIds","result","insertOne","acknowledged","info","ApiError","httpStatus","INTERNAL_SERVER_ERROR","error","contentType","recordLoadParams","stream","UPLOADING","Promise","resolve","reject","outputStream","openUploadStream","on","chunk","write","end","state","errorMessage","errorStatus","timeOut","checkTimeOut","params","find","projection","_id","toArray","debug","length","checkModTime","findOne","timeoutTime","add","silly","isBefore","oldState","ABORT","REQUEST_TIMEOUT","JSON","stringify","clean","noContent","deleteOne","err","MongoDriverError","message","indexOf","readStream","openDownloadStreamByName","NOT_FOUND","delete","clean2","updateOne","$set","$push","$each","messages","messageField","cleanMessageField","errorString","log","findOneAndUpdate","returnNewDocument"],"mappings":";;;;;;;AA8BA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AArCA;;AACA;;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAWA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAEe,wBAAgBA,SAAhB,EAA2BC,UAA3B,EAAuC;AACpD,QAAMC,MAAM,GAAG,0CAAf,CADoD,CAGpD;;AACA,QAAMC,MAAM,GAAG,MAAMC,qBAAYC,OAAZ,CAAoBL,SAApB,EAA+B;AAACM,IAAAA,eAAe,EAAE,IAAlB;AAAwBC,IAAAA,kBAAkB,EAAE;AAA5C,GAA/B,CAArB;AACA,QAAMC,EAAE,GAAGL,MAAM,CAACK,EAAP,CAAU,UAAV,CAAX;AACA,QAAMC,YAAY,GAAG,IAAIC,qBAAJ,CAAiBF,EAAjB,EAAqB;AAACG,IAAAA,UAAU,EAAEV;AAAb,GAArB,CAArB;AAEA,SAAO;AAACW,IAAAA,UAAD;AAAaC,IAAAA,UAAb;AAAyBC,IAAAA,gBAAzB;AAA2CC,IAAAA,KAA3C;AAAkDC,IAAAA,SAAlD;AAA6DC,IAAAA,MAA7D;AAAqEC,IAAAA,WAArE;AAAkFC,IAAAA,aAAlF;AAAiGC,IAAAA,MAAjG;AAAyGC,IAAAA,SAAzG;AAAoHC,IAAAA,QAApH;AAA8HC,IAAAA,OAA9H;AAAuIC,IAAAA;AAAvI,GAAP;;AAEA,iBAAeZ,UAAf,CAA0B;AAACa,IAAAA,aAAD;AAAgBC,IAAAA,SAAhB;AAA2BC,IAAAA,YAA3B;AAAyCC,IAAAA,SAAzC;AAAoDC,IAAAA,IAAI,GAAGC,SAA3D;AAAsEC,IAAAA,MAAM,GAAGD,SAA/E;AAA0FE,IAAAA,IAAI,GAAGF;AAAjG,GAA1B,EAAuI;AACrI,UAAMG,IAAI,GAAG,uBAASC,MAAT,EAAb;AACA,UAAMC,YAAY,GAAG;AACnBV,MAAAA,aADmB;AAEnBC,MAAAA,SAFmB;AAGnBE,MAAAA,SAHmB;AAInBD,MAAAA,YAJmB;AAKnBS,MAAAA,iBAAiB,EAAE;AAACL,QAAAA,MAAD;AAASF,QAAAA,IAAT;AAAeG,QAAAA;AAAf,OALA;AAMnBK,MAAAA,cAAc,EAAEC,4BAAiBC,SAAjB,CAA2BC,kBANxB;AAOnBC,MAAAA,YAAY,EAAER,IAPK;AAQnBS,MAAAA,gBAAgB,EAAET,IARC;AASnBU,MAAAA,UAAU,EAAE,EATO;AAUnBC,MAAAA,WAAW,EAAE;AAVM,KAArB;;AAYA,QAAI;AACF,YAAMC,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B6C,SAA1B,CAAoCX,YAApC,CAArB;;AACA,UAAIU,MAAM,CAACE,YAAX,EAAyB;AACvB7C,QAAAA,MAAM,CAAC8C,IAAP,CAAa,2BAA0BvB,aAAc,qBAAoBxB,UAAW,EAApF;AACA;AACD;;AACD,YAAM,IAAIgD,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD,KAPD,CAOE,OAAOC,KAAP,EAAc;AACd,2BAASA,KAAT;AACA,YAAM,IAAIH,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD;AACF;;AAED,WAAStC,UAAT,CAAoB;AAACY,IAAAA,aAAD;AAAgBC,IAAAA,SAAhB;AAA2BC,IAAAA,YAA3B;AAAyCC,IAAAA,SAAzC;AAAoDyB,IAAAA,WAApD;AAAiEC,IAAAA,gBAAjE;AAAmFC,IAAAA;AAAnF,GAApB,EAAgH;AAC9G,UAAMtB,IAAI,GAAG,uBAASC,MAAT,EAAb;AACA,UAAMC,YAAY,GAAG;AACnBV,MAAAA,aADmB;AAEnBC,MAAAA,SAFmB;AAGnBC,MAAAA,YAHmB;AAInBC,MAAAA,SAJmB;AAKnByB,MAAAA,WALmB;AAMnBC,MAAAA,gBANmB;AAOnBjB,MAAAA,cAAc,EAAEC,4BAAiBC,SAAjB,CAA2BiB,SAPxB;AAQnBf,MAAAA,YAAY,EAAER,IARK;AASnBS,MAAAA,gBAAgB,EAAET,IATC;AAUnBU,MAAAA,UAAU,EAAE,EAVO;AAWnBC,MAAAA,WAAW,EAAE;AAXM,KAArB;;AAaA,QAAI;AACF;AACApC,MAAAA,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B6C,SAA1B,CAAoCX,YAApC;AACAjC,MAAAA,MAAM,CAAC8C,IAAP,CAAa,2BAA0BvB,aAAc,qBAAoBxB,UAAW,GAApF;AACD,KAJD,CAIE,OAAOmD,KAAP,EAAc;AACd,2BAASA,KAAT;AACA,YAAM,IAAIH,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD;;AAED,WAAO,IAAIM,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtC,YAAMC,YAAY,GAAGnD,YAAY,CAACoD,gBAAb,CAA8BpC,aAA9B,CAArB;AAEA8B,MAAAA,MAAM,CACHO,EADH,CACM,OADN,EACeH,MADf,EAEGG,EAFH,CAEM,MAFN,EAEcC,KAAK,IAAIH,YAAY,CAACI,KAAb,CAAmBD,KAAnB,CAFvB,EAGGD,EAHH,CAGM,KAHN,EAGa,MAAMF,YAAY,CAACK,GAAb,CAAiBnC,SAAjB,EAA4BA,SAA5B,EAAuC,MAAM;AAC5D4B,QAAAA,OAAO,CAACjC,aAAD,CAAP;AACD,OAFgB,CAHnB;AAMD,KATM,CAAP;AAUD,GAvEmD,CAyEpD;;;AACA,iBAAeX,gBAAf,CAAgC;AAACW,IAAAA,aAAD;AAAgByC,IAAAA,KAAhB;AAAuBC,IAAAA,YAAY,GAAG,EAAtC;AAA0CC,IAAAA,WAAW,GAAG;AAAxD,GAAhC,EAA6F;AAC3F;AACA;AACA,UAAMC,OAAO,GAAG,MAAMC,YAAY,CAAC7C,aAAD,CAAlC;;AACA,QAAI4C,OAAJ,EAAa;AACX,aAAO/C,QAAQ,CAAC;AAACG,QAAAA,aAAD;AAAgByC,QAAAA,KAAhB;AAAuBC,QAAAA,YAAvB;AAAqCC,QAAAA;AAArC,OAAD,CAAf;AACD;;AACD,WAAO,KAAP;AACD;;AAED,iBAAerD,KAAf,CAAqBwD,MAArB,EAA6B;AAC3B,UAAM1B,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BuE,IAA1B,CAA+BD,MAA/B,EAAuC;AAACE,MAAAA,UAAU,EAAE;AAACC,QAAAA,GAAG,EAAE;AAAN;AAAb,KAAvC,EAClBC,OADkB,EAArB;AAEAzE,IAAAA,MAAM,CAAC0E,KAAP,CAAc,iBAAgB/B,MAAM,CAACgC,MAAP,GAAgB,CAAhB,GAAoB,QAApB,GAA+B,YAAa,EAA1E;AACA,WAAOhC,MAAP;AACD;;AAED,iBAAe7B,SAAf,CAAyBS,aAAzB,EAAwCqD,YAAxC,EAAsD;AACpD,UAAMjC,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC;AAACtD,MAAAA;AAAD,KAAlC,CAArB;;AACA,QAAIqD,YAAJ,EAAkB;AAChB,YAAMT,OAAO,GAAG,MAAMC,YAAY,CAAC7C,aAAD,CAAlC;;AACA,UAAI4C,OAAJ,EAAa;AACX,eAAOrD,SAAS,CAACS,aAAD,CAAhB;AACD;;AACD,aAAOoB,MAAP;AACD;;AAED,WAAOA,MAAP;AACD,GAtGmD,CAwGpD;AACA;AACA;;;AACA,iBAAeyB,YAAf,CAA4B7C,aAA5B,EAA2C;AACzC,UAAMoB,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC;AAACtD,MAAAA;AAAD,KAAlC,CAArB;AACA,UAAMuD,WAAW,GAAG,qBAAOnC,MAAM,CAACH,gBAAd,EAAgCuC,GAAhC,CAAoC,CAApC,EAAuC,GAAvC,CAApB;AACA/E,IAAAA,MAAM,CAACgF,KAAP,CAAc,aAAYF,WAAY,EAAtC;;AACA,QAAIA,WAAW,CAACG,QAAZ,EAAJ,EAA4B;AAC1B,YAAMC,QAAQ,GAAGvC,MAAM,CAACR,cAAxB;AACA,YAAMf,QAAQ,CAAC;AAACG,QAAAA,aAAD;AAAgByC,QAAAA,KAAK,EAAE5B,4BAAiB+C,KAAxC;AAA+CjB,QAAAA,WAAW,EAAElB,oBAAWoC,eAAvE;AAAwFnB,QAAAA,YAAY,EAAG,cAAaiB,QAAS;AAA7H,OAAD,CAAd;AACA,aAAO,KAAP;AACD;;AAED,WAAO,IAAP;AACD;;AAED,iBAAenE,MAAf,CAAsBsD,MAAtB,EAA8B;AAC5BrE,IAAAA,MAAM,CAAC0E,KAAP,CAAc,GAAEW,IAAI,CAACC,SAAL,CAAejB,MAAf,CAAuB,EAAvC;AACArE,IAAAA,MAAM,CAAC8C,IAAP,CAAa,wBAAuB/C,UAAW,SAAQsE,MAAM,CAAC9C,aAAc,EAA5E;AACA,UAAMgE,KAAK,GAAG,4BAASlB,MAAM,CAAC9C,aAAhB,CAAd;AACAvB,IAAAA,MAAM,CAACgF,KAAP,CAAc,wBAAuBK,IAAI,CAACC,SAAL,CAAeC,KAAf,CAAsB,EAA3D;;AAEA,QAAI;AACF;AACA;AACA,YAAMC,SAAS,GAAG,MAAMvE,aAAa,CAACoD,MAAD,CAArC;;AACA,UAAImB,SAAJ,EAAe;AACb,cAAMlF,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B0F,SAA1B,CAAoC;AAAClE,UAAAA,aAAa,EAAEgE;AAAhB,SAApC,CAAN;AACA,eAAO,IAAP;AACD;AACF,KARD,CAQE,OAAOG,GAAP,EAAY;AACZ,UAAIA,GAAG,YAAYC,yBAAnB,EAAqC;AACnC3F,QAAAA,MAAM,CAACkD,KAAP,CAAawC,GAAG,CAACE,OAAjB;;AACA,YAAIF,GAAG,CAACE,OAAJ,CAAYC,OAAZ,CAAoB,uBAApB,MAAiD,CAAC,CAAtD,EAAyD;AACvD7F,UAAAA,MAAM,CAAC0E,KAAP,CAAc,oDAAmDW,IAAI,CAACC,SAAL,CAAeC,KAAf,CAAsB,SAAQxF,UAAW,EAA1G;AACA,gBAAMO,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B0F,SAA1B,CAAoC;AAAClE,YAAAA,aAAa,EAAEgE;AAAhB,WAApC,CAAN;AACA,iBAAO,IAAP;AACD;;AACD,cAAM,IAAIxC,qBAAJ,CAAaC,oBAAWC,qBAAxB,EAA+CyC,GAAG,CAACE,OAAnD,CAAN;AACD;;AACD,YAAM,IAAI7C,qBAAJ,CAAaC,oBAAWC,qBAAxB,EAA+CyC,GAAG,CAACE,OAAnD,CAAN;AACD;AACF;;AAED,iBAAe5E,WAAf,CAA2BO,aAA3B,EAA0C;AACxCvB,IAAAA,MAAM,CAAC8C,IAAP,CAAa,2BAA0BvB,aAAc,OAAMxB,UAAW,EAAtE;AACA,UAAMwF,KAAK,GAAG,4BAAShE,aAAT,CAAd;AACA,UAAMoB,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC;AAACtD,MAAAA,aAAa,EAAEgE;AAAhB,KAAlC,CAArB,CAHwC,CAGwC;;AAEhF,QAAI5C,MAAJ,EAAY;AACV,aAAO;AACLQ,QAAAA,WAAW,EAAER,MAAM,CAACQ,WADf;AAEL2C,QAAAA,UAAU,EAAEvF,YAAY,CAACwF,wBAAb,CAAsCR,KAAtC;AAFP,OAAP;AAID;;AAED,UAAM,IAAIxC,qBAAJ,CAAaC,oBAAWgD,SAAxB,CAAN;AACD;;AAED,iBAAe/E,aAAf,CAA6BoD,MAA7B,EAAqC;AACnCrE,IAAAA,MAAM,CAAC8C,IAAP,CAAa,4BAA2BuB,MAAM,CAAC9C,aAAc,OAAMxB,UAAW,EAA9E;AACA,UAAMwF,KAAK,GAAG,4BAASlB,MAAM,CAAC9C,aAAhB,CAAd;AAEA,UAAMoB,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC;AAACtD,MAAAA,aAAa,EAAEgE;AAAhB,KAAlC,CAArB,CAJmC,CAI6C;;AAChFvF,IAAAA,MAAM,CAAC0E,KAAP,CAAc,+BAA8BW,IAAI,CAACC,SAAL,CAAe3C,MAAf,CAAuB,EAAnE;;AAEA,QAAIA,MAAJ,EAAY;AACV,YAAMpC,YAAY,CAAC0F,MAAb,CAAoBV,KAApB,CAAN;AACA,aAAO,IAAP;AACD;;AAED,WAAO,IAAP;AACD;;AAED,WAASrE,MAAT,CAAgB;AAACQ,IAAAA,SAAD;AAAYS,IAAAA;AAAZ,GAAhB,EAA6C;AAC3C,UAAMoD,KAAK,GAAG;AAACpD,MAAAA,cAAc,EAAE,4BAASA,cAAT;AAAjB,KAAd;;AACA,QAAI;AACF,UAAIT,SAAS,KAAKE,SAAlB,EAA6B;AAC3B5B,QAAAA,MAAM,CAACgF,KAAP,CAAc,eAAcjF,UAAW,QAAOsF,IAAI,CAACC,SAAL,CAAeC,KAAK,CAACpD,cAArB,CAAqC,EAAnF;AACA,eAAO7B,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC,EAAC,GAAGU;AAAJ,SAAlC,CAAP;AACD;;AAED,YAAMW,MAAM,GAAG;AAACxE,QAAAA,SAAS,EAAE,4BAASA,SAAT;AAAZ,OAAf;AACA1B,MAAAA,MAAM,CAACgF,KAAP,CAAc,eAAcjF,UAAW,QAAOwF,KAAK,CAACpD,cAAe,MAAK+D,MAAM,CAACxE,SAAU,EAAzF;AACA,aAAOpB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC,EAAC,GAAGU,KAAJ;AAAW,WAAGW;AAAd,OAAlC,CAAP;AACD,KATD,CASE,OAAOhD,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,WAAS/B,SAAT,CAAmBI,aAAnB,EAAkC;AAChCvB,IAAAA,MAAM,CAAC8C,IAAP,CAAa,2BAA0BvB,aAAc,OAAMxB,UAAW,EAAtE;AACA,UAAMwF,KAAK,GAAG,4BAAShE,aAAT,CAAd;;AACA,QAAI;AACF;AACA,aAAOhB,YAAY,CAACwF,wBAAb,CAAsCR,KAAtC,CAAP;AACD,KAHD,CAGE,OAAOrC,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAe7B,OAAf,CAAuB;AAACE,IAAAA,aAAD;AAAgBkB,IAAAA,UAAhB;AAA4BC,IAAAA;AAA5B,GAAvB,EAAiE;AAC/D1C,IAAAA,MAAM,CAAC8C,IAAP,CAAa,gCAA+BvB,aAAc,OAAMxB,UAAW,EAA3E;AACAC,IAAAA,MAAM,CAAC0E,KAAP,CAAc,QAAOW,IAAI,CAACC,SAAL,CAAe7C,UAAf,CAA2B,kBAAiB4C,IAAI,CAACC,SAAL,CAAe5C,WAAf,CAA4B,EAA7F;AACA,UAAM6C,KAAK,GAAG,4BAAShE,aAAT,CAAd;AACA,UAAMjB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BoG,SAA1B,CAAoC;AACxC5E,MAAAA,aAAa,EAAEgE;AADyB,KAApC,EAEH;AACDa,MAAAA,IAAI,EAAE;AACJ5D,QAAAA,gBAAgB,EAAE,uBAASR,MAAT;AADd,OADL;AAIDqE,MAAAA,KAAK,EAAE;AACL5D,QAAAA,UAAU,EAAE;AAAC6D,UAAAA,KAAK,EAAE7D;AAAR,SADP;AAELC,QAAAA,WAAW,EAAE;AAAC4D,UAAAA,KAAK,EAAE5D;AAAR;AAFR;AAJN,KAFG,CAAN;AAWD;;AAED,iBAAepB,YAAf,CAA4B;AAACC,IAAAA,aAAD;AAAgBgF,IAAAA,QAAhB;AAA0BC,IAAAA,YAAY,GAAG;AAAzC,GAA5B,EAAkF;AAChFxG,IAAAA,MAAM,CAAC8C,IAAP,CAAa,oBAAmB0D,YAAa,IAAGjF,aAAc,OAAMxB,UAAW,EAA/E;AACAC,IAAAA,MAAM,CAAC0E,KAAP,CAAc,aAAYW,IAAI,CAACC,SAAL,CAAeiB,QAAf,CAAyB,GAAnD;AACA,UAAMhB,KAAK,GAAG,4BAAShE,aAAT,CAAd;AACA,UAAMkF,iBAAiB,GAAG,4BAASD,YAAT,CAA1B;AACA,UAAMlG,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BoG,SAA1B,CAAoC;AACxC5E,MAAAA,aAAa,EAAEgE;AADyB,KAApC,EAEH;AACDa,MAAAA,IAAI,EAAE;AACJ5D,QAAAA,gBAAgB,EAAE,uBAASR,MAAT;AADd,OADL;AAIDqE,MAAAA,KAAK,EAAE;AACL,SAACI,iBAAD,GAAqB;AAACH,UAAAA,KAAK,EAAEC;AAAR;AADhB;AAJN,KAFG,CAAN;AAUD;;AAED,WAASnF,QAAT,CAAkB;AAACG,IAAAA,aAAD;AAAgByC,IAAAA,KAAhB;AAAuBC,IAAAA,YAAY,GAAG,EAAtC;AAA0CC,IAAAA,WAAW,GAAG;AAAxD,GAAlB,EAA+E;AAC7E,UAAMwC,WAAW,GAAGzC,YAAY,IAAIC,WAAhB,GAA+B,qBAAoBD,YAAa,qBAAoBC,WAAY,GAAhG,GAAqG,EAAzH;AACAlE,IAAAA,MAAM,CAAC2G,GAAP,CAAW,MAAX,EAAoB,6BAA4BpF,aAAc,GAAEyC,KAAM,GAAE0C,WAAY,OAAM3G,UAAW,EAArG;AACA,UAAMwF,KAAK,GAAG,4BAAShE,aAAT,CAAd;AACA,WAAOjB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B6G,gBAA1B,CAA2C;AAChDrF,MAAAA,aAAa,EAAEgE;AADiC,KAA3C,EAEJ;AACDa,MAAAA,IAAI,EAAE;AACJjE,QAAAA,cAAc,EAAE6B,KADZ;AAEJxB,QAAAA,gBAAgB,EAAE,uBAASR,MAAT,EAFd;AAGJiC,QAAAA,YAHI;AAIJC,QAAAA;AAJI;AADL,KAFI,EASJ;AAACK,MAAAA,UAAU,EAAE;AAACC,QAAAA,GAAG,EAAE;AAAN,OAAb;AAAuBqC,MAAAA,iBAAiB,EAAE;AAA1C,KATI,CAAP;AAUD;AACF","sourcesContent":["/* eslint-disable max-lines */\n/* eslint-disable max-statements */\n/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport {MongoClient, GridFSBucket, MongoDriverError} from 'mongodb';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {Error as ApiError} from '@natlibfi/melinda-commons';\nimport {QUEUE_ITEM_STATE} from './constants';\nimport {logError} from './utils.js';\nimport moment from 'moment';\nimport httpStatus from 'http-status';\nimport sanitize from 'mongo-sanitize';\n\n/* QueueItem:\n{\n  \"correlationId\":\"FOO\",\n  \"cataloger\":\"xxx0000\",\n  \"oCatalogerIn\":\"xxx0000\"\n  \"operation\":\"UPDATE\",\n  \"contentType\":\"application/json\",\n  \"recordLoadParams\": {\n    \"pActiveLibrary\": \"XXX00\",\n    \"pInputFile\": \"filename.seq\",\n    \"pRejectFile\": \"filename.rej\",\n    \"pLogFile\": \"filename.syslog\",\n    \"pOldNew\": \"NEW\"\n  },\n  \"queueItemState\":\"PENDING_QUEUING\",\n  \"creationTime\":\"2020-01-01T00:00:00.000Z\",\n  \"modificationTime\":\"2020-01-01T00:00:01.000Z\",\n  \"handledIds\": []\n}\n*/\n\nexport default async function (MONGO_URI, collection) {\n  const logger = createLogger();\n\n  // Connect to mongo (MONGO)\n  const client = await MongoClient.connect(MONGO_URI, {useNewUrlParser: true, useUnifiedTopology: true});\n  const db = client.db('rest-api');\n  const gridFSBucket = new GridFSBucket(db, {bucketName: collection});\n\n  return {createPrio, createBulk, checkAndSetState, query, queryById, remove, readContent, removeContent, getOne, getStream, setState, pushIds, pushMessages};\n\n  async function createPrio({correlationId, cataloger, oCatalogerIn, operation, noop = undefined, unique = undefined, prio = undefined}) {\n    const time = moment().toDate();\n    const newQueueItem = {\n      correlationId,\n      cataloger,\n      operation,\n      oCatalogerIn,\n      operationSettings: {unique, noop, prio},\n      queueItemState: QUEUE_ITEM_STATE.VALIDATOR.PENDING_VALIDATION,\n      creationTime: time,\n      modificationTime: time,\n      handledIds: [],\n      rejectedIds: []\n    };\n    try {\n      const result = await db.collection(collection).insertOne(newQueueItem);\n      if (result.acknowledged) {\n        logger.info(`New prio queue item for ${correlationId} has been made in ${collection}`);\n        return;\n      }\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    } catch (error) {\n      logError(error);\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    }\n  }\n\n  function createBulk({correlationId, cataloger, oCatalogerIn, operation, contentType, recordLoadParams, stream}) {\n    const time = moment().toDate();\n    const newQueueItem = {\n      correlationId,\n      cataloger,\n      oCatalogerIn,\n      operation,\n      contentType,\n      recordLoadParams,\n      queueItemState: QUEUE_ITEM_STATE.VALIDATOR.UPLOADING,\n      creationTime: time,\n      modificationTime: time,\n      handledIds: [],\n      rejectedIds: []\n    };\n    try {\n      // No await here, promises later\n      db.collection(collection).insertOne(newQueueItem);\n      logger.info(`New bulk queue item for ${correlationId} has been made in ${collection}!`);\n    } catch (error) {\n      logError(error);\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    }\n\n    return new Promise((resolve, reject) => {\n      const outputStream = gridFSBucket.openUploadStream(correlationId);\n\n      stream\n        .on('error', reject)\n        .on('data', chunk => outputStream.write(chunk))\n        .on('end', () => outputStream.end(undefined, undefined, () => {\n          resolve(correlationId);\n        }));\n    });\n  }\n\n  // Check state that the queueItem has not waited too long and set state\n  async function checkAndSetState({correlationId, state, errorMessage = '', errorStatus = ''}) {\n    // checkTimeOut returns true, if queueItem is fresher than 1 minute\n    // otherwise it sets queueItem to state ABORT (408, 'Timeout')\n    const timeOut = await checkTimeOut(correlationId);\n    if (timeOut) {\n      return setState({correlationId, state, errorMessage, errorStatus});\n    }\n    return false;\n  }\n\n  async function query(params) {\n    const result = await db.collection(collection).find(params, {projection: {_id: 0}})\n      .toArray();\n    logger.debug(`Query result: ${result.length > 0 ? 'Found!' : 'Not found!'}`);\n    return result;\n  }\n\n  async function queryById(correlationId, checkModTime) {\n    const result = await db.collection(collection).findOne({correlationId});\n    if (checkModTime) {\n      const timeOut = await checkTimeOut(correlationId);\n      if (timeOut) {\n        return queryById(correlationId);\n      }\n      return result;\n    }\n\n    return result;\n  }\n\n  // Check that if the item has waited too long\n  // If the last modification time for the queueItem is older than 1 minute\n  // set state to ABORT and return false, otherwise return true\n  async function checkTimeOut(correlationId) {\n    const result = await db.collection(collection).findOne({correlationId});\n    const timeoutTime = moment(result.modificationTime).add(1, 'm');\n    logger.silly(`timeOut @ ${timeoutTime}`);\n    if (timeoutTime.isBefore()) {\n      const oldState = result.queueItemState;\n      await setState({correlationId, state: QUEUE_ITEM_STATE.ABORT, errorStatus: httpStatus.REQUEST_TIMEOUT, errorMessage: `Timeout in ${oldState}`});\n      return false;\n    }\n\n    return true;\n  }\n\n  async function remove(params) {\n    logger.debug(`${JSON.stringify(params)}`);\n    logger.info(`Removing from Mongo (${collection}) id: ${params.correlationId}`);\n    const clean = sanitize(params.correlationId);\n    logger.silly(`mongo/remove: clean: ${JSON.stringify(clean)}`);\n\n    try {\n      //const metadataResult = await getFileMetadata({filename: clean});\n      //logger.debug(`mongo/remove: metadataResult: ${JSON.stringify(metadataResult)}`);\n      const noContent = await removeContent(params);\n      if (noContent) {\n        await db.collection(collection).deleteOne({correlationId: clean});\n        return true;\n      }\n    } catch (err) {\n      if (err instanceof MongoDriverError) {\n        logger.error(err.message);\n        if (err.message.indexOf('File not found for id') !== -1) {\n          logger.debug(`mongo/remove: File not found, removing queueItem ${JSON.stringify(clean)} from ${collection}`);\n          await db.collection(collection).deleteOne({correlationId: clean});\n          return true;\n        }\n        throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n      }\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n    }\n  }\n\n  async function readContent(correlationId) {\n    logger.info(`Reading content for id: ${correlationId} in ${collection}`);\n    const clean = sanitize(correlationId);\n    const result = await db.collection(collection).findOne({correlationId: clean}); // njsscan-ignore: node_nosqli_injection\n\n    if (result) {\n      return {\n        contentType: result.contentType,\n        readStream: gridFSBucket.openDownloadStreamByName(clean)\n      };\n    }\n\n    throw new ApiError(httpStatus.NOT_FOUND);\n  }\n\n  async function removeContent(params) {\n    logger.info(`Removing content for id: ${params.correlationId} in ${collection}`);\n    const clean = sanitize(params.correlationId);\n\n    const result = await db.collection(collection).findOne({correlationId: clean}); // njsscan-ignore: node_nosqli_injection\n    logger.debug(`mongo/removeContent: result ${JSON.stringify(result)}`);\n\n    if (result) {\n      await gridFSBucket.delete(clean);\n      return true;\n    }\n\n    return true;\n  }\n\n  function getOne({operation, queueItemState}) {\n    const clean = {queueItemState: sanitize(queueItemState)};\n    try {\n      if (operation === undefined) {\n        logger.silly(`Checking DB ${collection} for ${JSON.stringify(clean.queueItemState)}`);\n        return db.collection(collection).findOne({...clean});\n      }\n\n      const clean2 = {operation: sanitize(operation)};\n      logger.silly(`Checking DB ${collection} for ${clean.queueItemState} + ${clean2.operation}`);\n      return db.collection(collection).findOne({...clean, ...clean2});\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  function getStream(correlationId) {\n    logger.info(`Forming stream from db: ${correlationId} in ${collection}`);\n    const clean = sanitize(correlationId);\n    try {\n      // Return content stream\n      return gridFSBucket.openDownloadStreamByName(clean);\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function pushIds({correlationId, handledIds, rejectedIds}) {\n    logger.info(`Push queue-item ids to list: ${correlationId} to ${collection}`);\n    logger.debug(`ids: ${JSON.stringify(handledIds)}, rejectedIds: ${JSON.stringify(rejectedIds)}`);\n    const clean = sanitize(correlationId);\n    await db.collection(collection).updateOne({\n      correlationId: clean\n    }, {\n      $set: {\n        modificationTime: moment().toDate()\n      },\n      $push: {\n        handledIds: {$each: handledIds},\n        rejectedIds: {$each: rejectedIds}\n      }\n    });\n  }\n\n  async function pushMessages({correlationId, messages, messageField = 'messages'}) {\n    logger.info(`Push messages to ${messageField} ${correlationId} to ${collection}`);\n    logger.debug(`Messages: ${JSON.stringify(messages)}}`);\n    const clean = sanitize(correlationId);\n    const cleanMessageField = sanitize(messageField);\n    await db.collection(collection).updateOne({\n      correlationId: clean\n    }, {\n      $set: {\n        modificationTime: moment().toDate()\n      },\n      $push: {\n        [cleanMessageField]: {$each: messages}\n      }\n    });\n  }\n\n  function setState({correlationId, state, errorMessage = '', errorStatus = ''}) {\n    const errorString = errorMessage || errorStatus ? `, Error message: '${errorMessage}', Error status: '${errorStatus}'` : '';\n    logger.log('info', `Setting queue-item state: ${correlationId}${state}${errorString} to ${collection}`);\n    const clean = sanitize(correlationId);\n    return db.collection(collection).findOneAndUpdate({\n      correlationId: clean\n    }, {\n      $set: {\n        queueItemState: state,\n        modificationTime: moment().toDate(),\n        errorMessage,\n        errorStatus\n      }\n    }, {projection: {_id: 0}, returnNewDocument: true});\n  }\n}\n"],"file":"mongo.js"}

package/package.json

@@ -14,7 +14,7 @@
 		"url": "git@github.com:natlibfi/melinda-rest-api-commons.git"
 	},
 	"license": "AGPL-3.0+",
-	"version": "2.0.0-alpha.12",
+	"version": "2.0.0-alpha.13",
 	"main": "./dist/index.js",
 	"engines": {
 		"node": ">=12"
@@ -33,35 +33,35 @@
 		"build": "babel src --source-maps --copy-files --delete-dir-on-start --out-dir=dist"
 	},
 	"dependencies": {
-		"@natlibfi/fixura": "^2.1.4",
+		"@natlibfi/fixura": "^2.2.1",
 		"@natlibfi/marc-record": "^6.1.2",
-		"@natlibfi/marc-record-serializers": "^7.2.11",
+		"@natlibfi/marc-record-serializers": "^8.0.0-alpha.1",
 		"@natlibfi/marc-record-validate": "^6.0.1",
-		"@natlibfi/marc-record-validators-melinda": "^8.2.10",
-		"@natlibfi/melinda-backend-commons": "^2.0.3",
-		"@natlibfi/melinda-commons": "^11.1.7",
+		"@natlibfi/marc-record-validators-melinda": "^8.4.4",
+		"@natlibfi/melinda-backend-commons": "^2.0.5",
+		"@natlibfi/melinda-commons": "^11.1.10-alpha.1",
 		"amqplib": ">=0.8.0 <1.0.0",
 		"http-status": "^1.5.0",
 		"moment": "^2.29.1",
 		"mongo-sanitize": "^1.1.0",
-		"mongodb": "^4.0.1"
+		"mongodb": "^4.1.3"
 	},
 	"devDependencies": {
-		"@babel/cli": "^7.14.8",
-		"@babel/core": "^7.14.8",
+		"@babel/cli": "^7.15.7",
+		"@babel/core": "^7.15.8",
 		"@babel/plugin-syntax-dynamic-import": "^7.8.3",
-		"@babel/preset-env": "^7.14.8",
-		"@natlibfi/eslint-config-melinda-backend": "^1.0.6",
-		"@babel/eslint-parser": "^7.14.7",
+		"@babel/preset-env": "^7.15.8",
+		"@natlibfi/eslint-config-melinda-backend": "^1.1.1",
+		"@babel/eslint-parser": "^7.15.8",
 		"babel-plugin-dynamic-import-node": "^2.3.3",
-		"babel-plugin-istanbul": "^6.0.0",
+		"babel-plugin-istanbul": "^6.1.1",
 		"babel-plugin-rewire": "^1.2.0",
 		"chai": "^4.3.4",
 		"cross-env": "^7.0.3",
 		"eslint": "^7.32.0",
-		"eslint-plugin-import": "^2.23.4",
-		"mocha": "^9.0.2",
-		"nodemon": "^2.0.12",
+		"eslint-plugin-import": "^2.25.2",
+		"mocha": "^9.1.3",
+		"nodemon": "^2.0.14",
 		"nyc": "^15.1.0"
 	},
 	"eslintConfig": {