@natlibfi/melinda-rest-api-commons 2.0.0-alpha.13 → 2.0.0-alpha.18

package/dist/amqp.js

@@ -70,7 +70,7 @@ async function _default(AMQP_URL) {
   };
 
   async function checkQueue(queue, style = 'basic', purge = false, toRecord = true) {
-    logger.verbose(`checkQueue: ${queue}, Style: ${style}: Purge: ${purge}, toRecord: ${toRecord}`);
+    logger.silly(`checkQueue: ${queue}, Style: ${style}: Purge: ${purge}, toRecord: ${toRecord}`);
 
     try {
       const channelInfo = await channel.assertQueue(queue, {
@@ -79,12 +79,12 @@ async function _default(AMQP_URL) {
 
       if (purge) {
         await purgeQueue(purge);
-        logger.debug(`Queue ${queue} has purged ${channelInfo.messageCount} messages`);
+        logger.verbose(`Queue ${queue} has purged ${channelInfo.messageCount} messages`);
         return checkQueue(queue, style);
       }
 
       if (channelInfo.messageCount < 1) {
-        logger.debug(`checkQueue: ${channelInfo.messageCount} - ${queue} is empty`);
+        logger.silly(`checkQueue: ${channelInfo.messageCount} - ${queue} is empty`);
         return false;
       }
 
@@ -134,7 +134,7 @@ async function _default(AMQP_URL) {
   }
 
   async function consumeChunk(queue) {
-    logger.verbose(`Prepared to consumeChunk from queue: ${queue}`);
+    logger.silly(`Prepared to consumeChunk from queue: ${queue}`);
 
     try {
       await channel.assertQueue(queue, {
@@ -143,7 +143,7 @@ async function _default(AMQP_URL) {
 
       const queMessages = await getData(queue);
       const headers = getHeaderInfo(queMessages[0]);
-      logger.debug(`Filtering messages by cataloger in ${JSON.stringify(headers)}`);
+      logger.silly(`Filtering messages by cataloger in ${JSON.stringify(headers)}`);
       let filterIn = 0; // eslint-disable-line functional/no-let
 
       let filterOut = 0; // eslint-disable-line functional/no-let
@@ -162,8 +162,13 @@ async function _default(AMQP_URL) {
 
         return false;
       });
-      logger.debug(`Filtering result: valid: ${filterIn} non-valid: ${filterOut}`);
-      const records = await messagesToRecords(messages);
+      logger.silly(`Filtering by cataloger result: valid: ${filterIn} non-valid: ${filterOut}`);
+      const records = await messagesToRecords(messages); // eslint-disable-next-line functional/no-conditional-statement
+
+      if (messages) {
+        logger.debug(`consumeChunk (${messages.length} from queue ${queue})`);
+      }
+
       return {
         headers,
         records,
@@ -175,7 +180,7 @@ async function _default(AMQP_URL) {
   }
 
   async function consumeByCorrelationId(queue, correlationId, toRecord = true) {
-    logger.verbose(`Prepared to consumeByCorrelationId from queue: ${queue}`);
+    logger.silly(`Prepared to consumeByCorrelationId from queue: ${queue}`);
 
     try {
       await channel.assertQueue(queue, {
@@ -183,7 +188,7 @@ async function _default(AMQP_URL) {
       }); // get next chunk (100) messages
 
       const queMessages = await getData(queue);
-      logger.debug(`Filtering messages by ${correlationId}`);
+      logger.silly(`Filtering messages by ${correlationId}`);
       let filterIn = 0; // eslint-disable-line functional/no-let
 
       let filterOut = 0; // eslint-disable-line functional/no-let
@@ -202,8 +207,12 @@ async function _default(AMQP_URL) {
 
         return false;
       });
-      logger.debug(`Filtering result: valid: ${filterIn} non-valid: ${filterOut}`);
-      const headers = getHeaderInfo(messages[0]);
+      logger.debug(`Filtering by correlationId result: valid: ${filterIn} non-valid: ${filterOut}`);
+      const headers = getHeaderInfo(messages[0]); // eslint-disable-next-line functional/no-conditional-statement
+
+      if (messages) {
+        logger.verbose(`consumeByCorrelationId (${messages.length}) from queue: ${queue}`);
+      }
 
       if (toRecord) {
         const records = await messagesToRecords(messages);
@@ -224,7 +233,7 @@ async function _default(AMQP_URL) {
   }
 
   async function consumeRawChunk(queue) {
-    logger.verbose(`Prepared to consumeRawChunk from queue: ${queue}`);
+    logger.silly(`Prepared to consumeRawChunk from queue: ${queue}`);
 
     try {
       await channel.assertQueue(queue, {
@@ -233,7 +242,7 @@ async function _default(AMQP_URL) {
 
       const queMessages = await getData(queue);
       const headers = getHeaderInfo(queMessages[0]);
-      logger.debug(`Filtering messages by ${JSON.stringify(headers)}`);
+      logger.silly(`Filtering messages by cataloger in ${JSON.stringify(headers)}`);
       let filterIn = 0; // eslint-disable-line functional/no-let
 
       let filterOut = 0; // eslint-disable-line functional/no-let
@@ -252,7 +261,12 @@ async function _default(AMQP_URL) {
 
         return false;
       });
-      logger.debug(`Filtering result: valid: ${filterIn} non-valid: ${filterOut}`); // return raw (do not convert messages to records)
+      logger.debug(`Filtering by cataloger result: valid: ${filterIn} non-valid: ${filterOut}`); // eslint-disable-next-line functional/no-conditional-statement
+
+      if (messages) {
+        logger.verbose(`consumeRawChunk (${messages.length}) from queue: ${queue}`);
+      } // return raw (do not convert messages to records)
+
 
       return {
         headers,
@@ -264,7 +278,7 @@ async function _default(AMQP_URL) {
   }
 
   async function consumeOne(queue) {
-    logger.verbose(`Prepared to consumeOne from queue: ${queue}`);
+    logger.silly(`Prepared to consumeOne from queue: ${queue}`);
 
     try {
       await channel.assertQueue(queue, {
@@ -276,6 +290,7 @@ async function _default(AMQP_URL) {
       if (message) {
         const headers = getHeaderInfo(message);
         const records = messagesToRecords([message]);
+        logger.verbose(`consumeOne from queue: ${queue}`);
         return {
           headers,
           records,
@@ -290,7 +305,7 @@ async function _default(AMQP_URL) {
   }
 
   async function consumeRaw(queue) {
-    logger.verbose(`Prepared to consumeRaw from queue: ${queue}`);
+    logger.silly(`Prepared to consumeRaw from queue: ${queue}`);
 
     try {
       await channel.assertQueue(queue, {
@@ -323,8 +338,6 @@ async function _default(AMQP_URL) {
     headers,
     data
   }) {
-    logger.verbose(`Send message for ${correlationId} to queue: ${queue}`);
-
     try {
       logger.silly(`Queue ${queue}`);
       logger.silly(`CorrelationId ${correlationId}`);
@@ -340,7 +353,7 @@ async function _default(AMQP_URL) {
         persistent: true,
         headers
       });
-      logger.silly(`Message send to queue ${queue}`);
+      logger.silly(`Send message for ${correlationId} to queue: ${queue}`);
     } catch (error) {
       (0, _utils.logError)(error);
     }
@@ -355,7 +368,7 @@ async function _default(AMQP_URL) {
 
 
   function messagesToRecords(messages) {
-    logger.verbose(`Parsing messages (${messages.length}) to records`);
+    logger.debug(`Parsing messages (${messages.length}) to records`);
     return messages.map(message => {
       const content = JSON.parse(message.content.toString());
       return new _marcRecord.MarcRecord(content.data);
@@ -363,7 +376,7 @@ async function _default(AMQP_URL) {
   }
 
   async function getData(queue) {
-    logger.verbose(`Getting queue data from ${queue}`);
+    logger.debug(`Getting queue data from ${queue}`);
 
     try {
       const {

package/dist/amqp.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/amqp.js"],"names":["AMQP_URL","connection","amqplib","connect","channel","createChannel","logger","checkQueue","consumeChunk","consumeRawChunk","consumeOne","consumeRaw","ackMessages","nackMessages","sendToQueue","removeQueue","messagesToRecords","queue","style","purge","toRecord","verbose","channelInfo","assertQueue","durable","purgeQueue","debug","messageCount","silly","test","consumeByCorrelationId","ApiError","httpStatus","UNPROCESSABLE_ENTITY","error","queMessages","getData","headers","getHeaderInfo","JSON","stringify","filterIn","filterOut","messages","filter","message","properties","cataloger","nack","records","correlationId","get","forEach","ack","data","Buffer","from","persistent","deleteQueue","length","map","content","parse","toString","MarcRecord","messagesToGet","CHUNK_SIZE","pump","count","results","identifiers","identifier","deliveryTag","fields","includes","concat"],"mappings":";;;;;;;AA8BA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AApCA;;AACA;;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAUe,wBAAgBA,QAAhB,EAA0B;AACvC,QAAMC,UAAU,GAAG,MAAMC,iBAAQC,OAAR,CAAgBH,QAAhB,CAAzB;AACA,QAAMI,OAAO,GAAG,MAAMH,UAAU,CAACI,aAAX,EAAtB;AACA,QAAMC,MAAM,GAAG,0CAAf;AAEA,SAAO;AAACC,IAAAA,UAAD;AAAaC,IAAAA,YAAb;AAA2BC,IAAAA,eAA3B;AAA4CC,IAAAA,UAA5C;AAAwDC,IAAAA,UAAxD;AAAoEC,IAAAA,WAApE;AAAiFC,IAAAA,YAAjF;AAA+FC,IAAAA,WAA/F;AAA4GC,IAAAA,WAA5G;AAAyHC,IAAAA;AAAzH,GAAP;;AAEA,iBAAeT,UAAf,CAA0BU,KAA1B,EAAiCC,KAAK,GAAG,OAAzC,EAAkDC,KAAK,GAAG,KAA1D,EAAiEC,QAAQ,GAAG,IAA5E,EAAkF;AAChFd,IAAAA,MAAM,CAACe,OAAP,CAAgB,eAAcJ,KAAM,YAAWC,KAAM,YAAWC,KAAM,eAAcC,QAAS,EAA7F;;AACA,QAAI;AACF,YAAME,WAAW,GAAG,MAAMlB,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAA1B;;AACA,UAAIL,KAAJ,EAAW;AACT,cAAMM,UAAU,CAACN,KAAD,CAAhB;AACAb,QAAAA,MAAM,CAACoB,KAAP,CAAc,SAAQT,KAAM,eAAcK,WAAW,CAACK,YAAa,WAAnE;AACA,eAAOpB,UAAU,CAACU,KAAD,EAAQC,KAAR,CAAjB;AACD;;AAED,UAAII,WAAW,CAACK,YAAZ,GAA2B,CAA/B,EAAkC;AAChCrB,QAAAA,MAAM,CAACoB,KAAP,CAAc,eAAcJ,WAAW,CAACK,YAAa,MAAKV,KAAM,WAAhE;AACA,eAAO,KAAP;AACD;;AACDX,MAAAA,MAAM,CAACsB,KAAP,CAAc,SAAQX,KAAM,QAAOK,WAAW,CAACK,YAAa,WAA5D;;AAEA,UAAIT,KAAK,KAAK,UAAd,EAA0B;AACxB,eAAOI,WAAW,CAACK,YAAnB;AACD,OAhBC,CAkBF;;;AACA,UAAIT,KAAK,KAAK,KAAd,EAAqB;AACnB,eAAOR,UAAU,CAACO,KAAD,CAAjB;AACD,OArBC,CAuBF;;;AACA,UAAIC,KAAK,KAAK,KAAd,EAAqB;AACnB,eAAOP,UAAU,CAACM,KAAD,CAAjB;AACD,OA1BC,CA4BF;;;AACA,UAAIC,KAAK,KAAK,UAAd,EAA0B;AACxB,eAAOT,eAAe,CAACQ,KAAD,CAAtB;AACD,OA/BC,CAiCF;;;AACA,UAAK,8BAAD,CAAiCY,IAAjC,CAAsCX,KAAtC,CAAJ,EAAkD;AAChDZ,QAAAA,MAAM,CAACsB,KAAP,CAAc,4BAA2BV,KAAM,EAA/C;AACA,eAAOY,sBAAsB,CAACb,KAAD,EAAQC,KAAR,EAAeE,QAAf,CAA7B;AACD,OArCC,CAuCF;;;AACA,UAAIF,KAAK,KAAK,OAAd,EAAuB;AACrB,eAAOV,YAAY,CAACS,KAAD,CAAnB;AACD,OA1CC,CA4CF;;;AACA,YAAM,IAAIc,qBAAJ,CAAaC,oBAAWC,oBAAxB,CAAN;AACD,KA9CD,CA8CE,OAAOC,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;;AAED,aAAST,UAAT,CAAoBN,KAApB,EAA2B;AACzB,UAAIA,KAAJ,EAAW;AACT,eAAOf,OAAO,CAACqB,UAAR,CAAmBR,KAAnB,CAAP;AACD;AACF;AACF;;AAED,iBAAeT,YAAf,CAA4BS,KAA5B,EAAmC;AACjCX,IAAAA,MAAM,CAACe,OAAP,CAAgB,wCAAuCJ,KAAM,EAA7D;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAGF;;AACA,YAAMW,WAAW,GAAG,MAAMC,OAAO,CAACnB,KAAD,CAAjC;AAEA,YAAMoB,OAAO,GAAGC,aAAa,CAACH,WAAW,CAAC,CAAD,CAAZ,CAA7B;AACA7B,MAAAA,MAAM,CAACoB,KAAP,CAAc,sCAAqCa,IAAI,CAACC,SAAL,CAAeH,OAAf,CAAwB,EAA3E;AAEA,UAAII,QAAQ,GAAG,CAAf,CATE,CASgB;;AAClB,UAAIC,SAAS,GAAG,CAAhB,CAVE,CAUiB;AAEnB;;AACA,YAAMC,QAAQ,GAAGR,WAAW,CAACS,MAAZ,CAAmBC,OAAO,IAAI;AAC7C,YAAIA,OAAO,CAACC,UAAR,CAAmBT,OAAnB,CAA2BU,SAA3B,KAAyCV,OAAO,CAACU,SAArD,EAAgE;AAC9DN,UAAAA,QAAQ,GADsD,CAClD;;AACZ,iBAAO,IAAP;AACD,SAJ4C,CAM7C;;;AACArC,QAAAA,OAAO,CAAC4C,IAAR,CAAaH,OAAb,EAAsB,KAAtB,EAA6B,IAA7B;AACAH,QAAAA,SAAS,GARoC,CAQhC;;AACb,eAAO,KAAP;AACD,OAVgB,CAAjB;AAYApC,MAAAA,MAAM,CAACoB,KAAP,CAAc,4BAA2Be,QAAS,eAAcC,SAAU,EAA1E;AACA,YAAMO,OAAO,GAAG,MAAMjC,iBAAiB,CAAC2B,QAAD,CAAvC;AAEA,aAAO;AAACN,QAAAA,OAAD;AAAUY,QAAAA,OAAV;AAAmBN,QAAAA;AAAnB,OAAP;AACD,KA7BD,CA6BE,OAAOT,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAeJ,sBAAf,CAAsCb,KAAtC,EAA6CiC,aAA7C,EAA4D9B,QAAQ,GAAG,IAAvE,EAA6E;AAC3Ed,IAAAA,MAAM,CAACe,OAAP,CAAgB,kDAAiDJ,KAAM,EAAvE;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAEF;;AACA,YAAMW,WAAW,GAAG,MAAMC,OAAO,CAACnB,KAAD,CAAjC;AAEAX,MAAAA,MAAM,CAACoB,KAAP,CAAc,yBAAwBwB,aAAc,EAApD;AAEA,UAAIT,QAAQ,GAAG,CAAf,CAPE,CAOgB;;AAClB,UAAIC,SAAS,GAAG,CAAhB,CARE,CAQiB;AAEnB;;AACA,YAAMC,QAAQ,GAAGR,WAAW,CAACS,MAAZ,CAAmBC,OAAO,IAAI;AAC7C,YAAIA,OAAO,CAACC,UAAR,CAAmBI,aAAnB,KAAqCA,aAAzC,EAAwD;AACtDT,UAAAA,QAAQ,GAD8C,CAC1C;;AACZ,iBAAO,IAAP;AACD,SAJ4C,CAM7C;;;AACArC,QAAAA,OAAO,CAAC4C,IAAR,CAAaH,OAAb,EAAsB,KAAtB,EAA6B,IAA7B;AACAH,QAAAA,SAAS,GARoC,CAQhC;;AACb,eAAO,KAAP;AACD,OAVgB,CAAjB;AAWApC,MAAAA,MAAM,CAACoB,KAAP,CAAc,4BAA2Be,QAAS,eAAcC,SAAU,EAA1E;AAEA,YAAML,OAAO,GAAGC,aAAa,CAACK,QAAQ,CAAC,CAAD,CAAT,CAA7B;;AAEA,UAAIvB,QAAJ,EAAc;AACZ,cAAM6B,OAAO,GAAG,MAAMjC,iBAAiB,CAAC2B,QAAD,CAAvC;AAEA,eAAO;AAACN,UAAAA,OAAD;AAAUY,UAAAA,OAAV;AAAmBN,UAAAA;AAAnB,SAAP;AACD;;AACD,aAAO;AAACN,QAAAA,OAAD;AAAUM,QAAAA;AAAV,OAAP;AACD,KAhCD,CAgCE,OAAOT,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAezB,eAAf,CAA+BQ,KAA/B,EAAsC;AACpCX,IAAAA,MAAM,CAACe,OAAP,CAAgB,2CAA0CJ,KAAM,EAAhE;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAEF;;AACA,YAAMW,WAAW,GAAG,MAAMC,OAAO,CAACnB,KAAD,CAAjC;AAEA,YAAMoB,OAAO,GAAGC,aAAa,CAACH,WAAW,CAAC,CAAD,CAAZ,CAA7B;AACA7B,MAAAA,MAAM,CAACoB,KAAP,CAAc,yBAAwBa,IAAI,CAACC,SAAL,CAAeH,OAAf,CAAwB,EAA9D;AAEA,UAAII,QAAQ,GAAG,CAAf,CARE,CAQgB;;AAClB,UAAIC,SAAS,GAAG,CAAhB,CATE,CASiB;AAEnB;;AACA,YAAMC,QAAQ,GAAGR,WAAW,CAACS,MAAZ,CAAmBC,OAAO,IAAI;AAC7C,YAAIA,OAAO,CAACC,UAAR,CAAmBT,OAAnB,CAA2BU,SAA3B,KAAyCV,OAAO,CAACU,SAArD,EAAgE;AAC9DN,UAAAA,QAAQ,GADsD,CAClD;;AAEZ,iBAAO,IAAP;AACD,SAL4C,CAO7C;;;AACArC,QAAAA,OAAO,CAAC4C,IAAR,CAAaH,OAAb,EAAsB,KAAtB,EAA6B,IAA7B;AACAH,QAAAA,SAAS,GAToC,CAShC;;AAEb,eAAO,KAAP;AACD,OAZgB,CAAjB;AAaApC,MAAAA,MAAM,CAACoB,KAAP,CAAc,4BAA2Be,QAAS,eAAcC,SAAU,EAA1E,EAzBE,CA2BF;;AACA,aAAO;AAACL,QAAAA,OAAD;AAAUM,QAAAA;AAAV,OAAP;AACD,KA7BD,CA6BE,OAAOT,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAexB,UAAf,CAA0BO,KAA1B,EAAiC;AAC/BX,IAAAA,MAAM,CAACe,OAAP,CAAgB,sCAAqCJ,KAAM,EAA3D;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAGF;;AACA,YAAMqB,OAAO,GAAG,MAAMzC,OAAO,CAAC+C,GAAR,CAAYlC,KAAZ,CAAtB;;AACA,UAAI4B,OAAJ,EAAa;AACX,cAAMR,OAAO,GAAGC,aAAa,CAACO,OAAD,CAA7B;AACA,cAAMI,OAAO,GAAGjC,iBAAiB,CAAC,CAAC6B,OAAD,CAAD,CAAjC;AACA,eAAO;AAACR,UAAAA,OAAD;AAAUY,UAAAA,OAAV;AAAmBN,UAAAA,QAAQ,EAAE,CAACE,OAAD;AAA7B,SAAP;AACD;;AAED,aAAO,KAAP;AACD,KAZD,CAYE,OAAOX,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAevB,UAAf,CAA0BM,KAA1B,EAAiC;AAC/BX,IAAAA,MAAM,CAACe,OAAP,CAAgB,sCAAqCJ,KAAM,EAA3D;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAEF;;AACA,aAAO,MAAMpB,OAAO,CAAC+C,GAAR,CAAYlC,KAAZ,CAAb;AACD,KAJD,CAIE,OAAOiB,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,WAAStB,WAAT,CAAqB+B,QAArB,EAA+B;AAC7BA,IAAAA,QAAQ,CAACS,OAAT,CAAiBP,OAAO,IAAI;AAC1BvC,MAAAA,MAAM,CAACsB,KAAP,CAAc,eAAciB,OAAO,CAACC,UAAR,CAAmBI,aAAc,EAA7D;AACA9C,MAAAA,OAAO,CAACiD,GAAR,CAAYR,OAAZ;AACD,KAHD;AAID;;AAED,WAAShC,YAAT,CAAsB8B,QAAtB,EAAgC;AAC9BA,IAAAA,QAAQ,CAACS,OAAT,CAAiBP,OAAO,IAAI;AAC1BvC,MAAAA,MAAM,CAACsB,KAAP,CAAc,gBAAeiB,OAAO,CAACC,UAAR,CAAmBI,aAAc,EAA9D;AACA9C,MAAAA,OAAO,CAAC4C,IAAR,CAAaH,OAAb;AACD,KAHD;AAID;;AAED,iBAAe/B,WAAf,CAA2B;AAACG,IAAAA,KAAD;AAAQiC,IAAAA,aAAR;AAAuBb,IAAAA,OAAvB;AAAgCiB,IAAAA;AAAhC,GAA3B,EAAkE;AAChEhD,IAAAA,MAAM,CAACe,OAAP,CAAgB,oBAAmB6B,aAAc,cAAajC,KAAM,EAApE;;AACA,QAAI;AACFX,MAAAA,MAAM,CAACsB,KAAP,CAAc,SAAQX,KAAM,EAA5B;AACAX,MAAAA,MAAM,CAACsB,KAAP,CAAc,iBAAgBsB,aAAc,EAA5C;AACA5C,MAAAA,MAAM,CAACsB,KAAP,CAAc,QAAOW,IAAI,CAACC,SAAL,CAAec,IAAf,CAAqB,EAA1C;AACAhD,MAAAA,MAAM,CAACsB,KAAP,CAAc,WAAUW,IAAI,CAACC,SAAL,CAAeH,OAAf,CAAwB,EAAhD;AAEA,YAAMjC,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN;AAEApB,MAAAA,OAAO,CAACU,WAAR,CACEG,KADF,EAEEsC,MAAM,CAACC,IAAP,CAAYjB,IAAI,CAACC,SAAL,CAAe;AAACc,QAAAA;AAAD,OAAf,CAAZ,CAFF,EAGE;AACEJ,QAAAA,aADF;AAEEO,QAAAA,UAAU,EAAE,IAFd;AAGEpB,QAAAA;AAHF,OAHF;AAUA/B,MAAAA,MAAM,CAACsB,KAAP,CAAc,yBAAwBX,KAAM,EAA5C;AACD,KAnBD,CAmBE,OAAOiB,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAenB,WAAf,CAA2BE,KAA3B,EAAkC;AAChCX,IAAAA,MAAM,CAACe,OAAP,CAAgB,kBAAiBJ,KAAM,GAAvC;AACA,UAAMb,OAAO,CAACsD,WAAR,CAAoBzC,KAApB,CAAN;AACD,GA1PsC,CA4PvC;AACA;AACA;;;AAEA,WAASD,iBAAT,CAA2B2B,QAA3B,EAAqC;AACnCrC,IAAAA,MAAM,CAACe,OAAP,CAAgB,qBAAoBsB,QAAQ,CAACgB,MAAO,cAApD;AAEA,WAAOhB,QAAQ,CAACiB,GAAT,CAAaf,OAAO,IAAI;AAC7B,YAAMgB,OAAO,GAAGtB,IAAI,CAACuB,KAAL,CAAWjB,OAAO,CAACgB,OAAR,CAAgBE,QAAhB,EAAX,CAAhB;AACA,aAAO,IAAIC,sBAAJ,CAAeH,OAAO,CAACP,IAAvB,CAAP;AACD,KAHM,CAAP;AAID;;AAED,iBAAelB,OAAf,CAAuBnB,KAAvB,EAA8B;AAC5BX,IAAAA,MAAM,CAACe,OAAP,CAAgB,2BAA0BJ,KAAM,EAAhD;;AACA,QAAI;AACF,YAAM;AAACU,QAAAA;AAAD,UAAiB,MAAMvB,OAAO,CAACG,UAAR,CAAmBU,KAAnB,CAA7B;AACAX,MAAAA,MAAM,CAACsB,KAAP,CAAc,YAAWD,YAAa,sBAAqBV,KAAM,EAAjE;AACA,YAAMgD,aAAa,GAAGtC,YAAY,IAAIuC,qBAAhB,GAA6BA,qBAA7B,GAA0CvC,YAAhE;AACArB,MAAAA,MAAM,CAACsB,KAAP,CAAc,WAAUqC,aAAc,wBAAuBhD,KAAM,EAAnE;AAEA,YAAM0B,QAAQ,GAAG,MAAMwB,IAAI,CAACF,aAAD,CAA3B;AAEA3D,MAAAA,MAAM,CAACoB,KAAP,CAAc,aAAYiB,QAAQ,CAACgB,MAAO,kBAA1C;AAEA,aAAOhB,QAAP;AACD,KAXD,CAWE,OAAOT,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;;AAED,mBAAeiC,IAAf,CAAoBC,KAApB,EAA2BC,OAAO,GAAG,EAArC,EAAyCC,WAAW,GAAG,EAAvD,EAA2D;AACzD,UAAIF,KAAK,KAAK,CAAd,EAAiB;AACf,eAAOC,OAAP;AACD;;AAED,YAAMxB,OAAO,GAAG,MAAMzC,OAAO,CAAC+C,GAAR,CAAYlC,KAAZ,CAAtB;AACA,YAAMsD,UAAU,GAAG;AACjBrB,QAAAA,aAAa,EAAEL,OAAO,CAACC,UAAR,CAAmBI,aADjB;AAEjBsB,QAAAA,WAAW,EAAE3B,OAAO,CAAC4B,MAAR,CAAeD;AAFX,OAAnB,CANyD,CAUzD;;AACA,UAAIF,WAAW,CAACI,QAAZ,CAAqBH,UAArB,CAAJ,EAAsC;AACpC,eAAOJ,IAAI,CAACC,KAAK,GAAG,CAAT,EAAYC,OAAZ,EAAqBC,WAArB,CAAX;AACD;;AAED,aAAOH,IAAI,CAACC,KAAK,GAAG,CAAT,EAAYC,OAAO,CAACM,MAAR,CAAe9B,OAAf,CAAZ,EAAqCyB,WAAW,CAACK,MAAZ,CAAmBJ,UAAnB,CAArC,CAAX;AACD;AACF;;AAED,WAASjC,aAAT,CAAuBgB,IAAvB,EAA6B;AAC3B,WAAOA,IAAI,CAACR,UAAL,CAAgBT,OAAvB;AACD;AACF","sourcesContent":["/* eslint-disable max-statements */\n/* eslint-disable max-lines */\n/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport amqplib from 'amqplib';\nimport {MarcRecord} from '@natlibfi/marc-record';\nimport {Error as ApiError} from '@natlibfi/melinda-commons';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {CHUNK_SIZE} from './constants';\nimport {logError} from './utils';\nimport httpStatus from 'http-status';\n\nexport default async function (AMQP_URL) {\n  const connection = await amqplib.connect(AMQP_URL);\n  const channel = await connection.createChannel();\n  const logger = createLogger();\n\n  return {checkQueue, consumeChunk, consumeRawChunk, consumeOne, consumeRaw, ackMessages, nackMessages, sendToQueue, removeQueue, messagesToRecords};\n\n  async function checkQueue(queue, style = 'basic', purge = false, toRecord = true) {\n    logger.verbose(`checkQueue: ${queue}, Style: ${style}: Purge: ${purge}, toRecord: ${toRecord}`);\n    try {\n      const channelInfo = await channel.assertQueue(queue, {durable: true});\n      if (purge) {\n        await purgeQueue(purge);\n        logger.debug(`Queue ${queue} has purged ${channelInfo.messageCount} messages`);\n        return checkQueue(queue, style);\n      }\n\n      if (channelInfo.messageCount < 1) {\n        logger.debug(`checkQueue: ${channelInfo.messageCount} - ${queue} is empty`);\n        return false;\n      }\n      logger.silly(`Queue ${queue} has ${channelInfo.messageCount} messages`);\n\n      if (style === 'messages') {\n        return channelInfo.messageCount;\n      }\n\n      // Note: returns a message + a record\n      if (style === 'one') {\n        return consumeOne(queue);\n      }\n\n      // Note: returns just a message\n      if (style === 'raw') {\n        return consumeRaw(queue);\n      }\n\n      // Note: returns a chunk of messages + records\n      if (style === 'rawChunk') {\n        return consumeRawChunk(queue);\n      }\n\n      // Note: returns a chunk of messages or a chunk of messages + records (depending on toRecord)\n      if ((/^.{8}-.{4}-.{4}-.{4}-.{12}$/u).test(style)) {\n        logger.silly(`checkQueue: regexp match ${style}`);\n        return consumeByCorrelationId(queue, style, toRecord);\n      }\n\n      // Note: returns a chunk of messages + records\n      if (style === 'basic') {\n        return consumeChunk(queue);\n      }\n\n      // Defaults:\n      throw new ApiError(httpStatus.UNPROCESSABLE_ENTITY);\n    } catch (error) {\n      logError(error);\n    }\n\n    function purgeQueue(purge) {\n      if (purge) {\n        return channel.purgeQueue(queue);\n      }\n    }\n  }\n\n  async function consumeChunk(queue) {\n    logger.verbose(`Prepared to consumeChunk from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n\n      // getData: next chunk (100) messages\n      const queMessages = await getData(queue);\n\n      const headers = getHeaderInfo(queMessages[0]);\n      logger.debug(`Filtering messages by cataloger in ${JSON.stringify(headers)}`);\n\n      let filterIn = 0; // eslint-disable-line functional/no-let\n      let filterOut = 0; // eslint-disable-line functional/no-let\n\n      // Check that cataloger match! headers\n      const messages = queMessages.filter(message => {\n        if (message.properties.headers.cataloger === headers.cataloger) {\n          filterIn++; // eslint-disable-line no-plusplus\n          return true;\n        }\n\n        // Nack unwanted ones\n        channel.nack(message, false, true);\n        filterOut++; // eslint-disable-line no-plusplus\n        return false;\n      });\n\n      logger.debug(`Filtering result: valid: ${filterIn} non-valid: ${filterOut}`);\n      const records = await messagesToRecords(messages);\n\n      return {headers, records, messages};\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeByCorrelationId(queue, correlationId, toRecord = true) {\n    logger.verbose(`Prepared to consumeByCorrelationId from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n      // get next chunk (100) messages\n      const queMessages = await getData(queue);\n\n      logger.debug(`Filtering messages by ${correlationId}`);\n\n      let filterIn = 0; // eslint-disable-line functional/no-let\n      let filterOut = 0; // eslint-disable-line functional/no-let\n\n      // Check that correlationId matches query\n      const messages = queMessages.filter(message => {\n        if (message.properties.correlationId === correlationId) {\n          filterIn++; // eslint-disable-line no-plusplus\n          return true;\n        }\n\n        // Nack unwanted ones\n        channel.nack(message, false, true);\n        filterOut++; // eslint-disable-line no-plusplus\n        return false;\n      });\n      logger.debug(`Filtering result: valid: ${filterIn} non-valid: ${filterOut}`);\n\n      const headers = getHeaderInfo(messages[0]);\n\n      if (toRecord) {\n        const records = await messagesToRecords(messages);\n\n        return {headers, records, messages};\n      }\n      return {headers, messages};\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeRawChunk(queue) {\n    logger.verbose(`Prepared to consumeRawChunk from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n      // Get next chunk (100) messages\n      const queMessages = await getData(queue);\n\n      const headers = getHeaderInfo(queMessages[0]);\n      logger.debug(`Filtering messages by ${JSON.stringify(headers)}`);\n\n      let filterIn = 0; // eslint-disable-line functional/no-let\n      let filterOut = 0; // eslint-disable-line functional/no-let\n\n      // Check that cataloger match! headers\n      const messages = queMessages.filter(message => {\n        if (message.properties.headers.cataloger === headers.cataloger) {\n          filterIn++; // eslint-disable-line no-plusplus\n\n          return true;\n        }\n\n        // Nack unwanted ones\n        channel.nack(message, false, true);\n        filterOut++; // eslint-disable-line no-plusplus\n\n        return false;\n      });\n      logger.debug(`Filtering result: valid: ${filterIn} non-valid: ${filterOut}`);\n\n      // return raw (do not convert messages to records)\n      return {headers, messages};\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeOne(queue) {\n    logger.verbose(`Prepared to consumeOne from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n\n      // Returns false if 0 items in queue\n      const message = await channel.get(queue);\n      if (message) {\n        const headers = getHeaderInfo(message);\n        const records = messagesToRecords([message]);\n        return {headers, records, messages: [message]};\n      }\n\n      return false;\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeRaw(queue) {\n    logger.verbose(`Prepared to consumeRaw from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n      // Returns false if 0 items in queue\n      return await channel.get(queue);\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  function ackMessages(messages) {\n    messages.forEach(message => {\n      logger.silly(`Ack message ${message.properties.correlationId}`);\n      channel.ack(message);\n    });\n  }\n\n  function nackMessages(messages) {\n    messages.forEach(message => {\n      logger.silly(`Nack message ${message.properties.correlationId}`);\n      channel.nack(message);\n    });\n  }\n\n  async function sendToQueue({queue, correlationId, headers, data}) {\n    logger.verbose(`Send message for ${correlationId} to queue: ${queue}`);\n    try {\n      logger.silly(`Queue ${queue}`);\n      logger.silly(`CorrelationId ${correlationId}`);\n      logger.silly(`Data ${JSON.stringify(data)}`);\n      logger.silly(`Headers ${JSON.stringify(headers)}`);\n\n      await channel.assertQueue(queue, {durable: true});\n\n      channel.sendToQueue(\n        queue,\n        Buffer.from(JSON.stringify({data})),\n        {\n          correlationId,\n          persistent: true,\n          headers\n        }\n      );\n\n      logger.silly(`Message send to queue ${queue}`);\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function removeQueue(queue) {\n    logger.verbose(`Removing queue ${queue}.`);\n    await channel.deleteQueue(queue);\n  }\n\n  // ----------------\n  // Helper functions\n  // ----------------\n\n  function messagesToRecords(messages) {\n    logger.verbose(`Parsing messages (${messages.length}) to records`);\n\n    return messages.map(message => {\n      const content = JSON.parse(message.content.toString());\n      return new MarcRecord(content.data);\n    });\n  }\n\n  async function getData(queue) {\n    logger.verbose(`Getting queue data from ${queue}`);\n    try {\n      const {messageCount} = await channel.checkQueue(queue);\n      logger.silly(`There is ${messageCount} messages in queue ${queue}`);\n      const messagesToGet = messageCount >= CHUNK_SIZE ? CHUNK_SIZE : messageCount;\n      logger.silly(`Getting ${messagesToGet} messages from queue ${queue}`);\n\n      const messages = await pump(messagesToGet);\n\n      logger.debug(`Returning ${messages.length} unique messages`);\n\n      return messages;\n    } catch (error) {\n      logError(error);\n    }\n\n    async function pump(count, results = [], identifiers = []) {\n      if (count === 0) {\n        return results;\n      }\n\n      const message = await channel.get(queue);\n      const identifier = {\n        correlationId: message.properties.correlationId,\n        deliveryTag: message.fields.deliveryTag\n      };\n      // Filter not unique messages\n      if (identifiers.includes(identifier)) {\n        return pump(count - 1, results, identifiers);\n      }\n\n      return pump(count - 1, results.concat(message), identifiers.concat(identifier));\n    }\n  }\n\n  function getHeaderInfo(data) {\n    return data.properties.headers;\n  }\n}\n"],"file":"amqp.js"}
+{"version":3,"sources":["../src/amqp.js"],"names":["AMQP_URL","connection","amqplib","connect","channel","createChannel","logger","checkQueue","consumeChunk","consumeRawChunk","consumeOne","consumeRaw","ackMessages","nackMessages","sendToQueue","removeQueue","messagesToRecords","queue","style","purge","toRecord","silly","channelInfo","assertQueue","durable","purgeQueue","verbose","messageCount","test","consumeByCorrelationId","ApiError","httpStatus","UNPROCESSABLE_ENTITY","error","queMessages","getData","headers","getHeaderInfo","JSON","stringify","filterIn","filterOut","messages","filter","message","properties","cataloger","nack","records","debug","length","correlationId","get","forEach","ack","data","Buffer","from","persistent","deleteQueue","map","content","parse","toString","MarcRecord","messagesToGet","CHUNK_SIZE","pump","count","results","identifiers","identifier","deliveryTag","fields","includes","concat"],"mappings":";;;;;;;AA8BA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AApCA;;AACA;;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAUe,wBAAgBA,QAAhB,EAA0B;AACvC,QAAMC,UAAU,GAAG,MAAMC,iBAAQC,OAAR,CAAgBH,QAAhB,CAAzB;AACA,QAAMI,OAAO,GAAG,MAAMH,UAAU,CAACI,aAAX,EAAtB;AACA,QAAMC,MAAM,GAAG,0CAAf;AAEA,SAAO;AAACC,IAAAA,UAAD;AAAaC,IAAAA,YAAb;AAA2BC,IAAAA,eAA3B;AAA4CC,IAAAA,UAA5C;AAAwDC,IAAAA,UAAxD;AAAoEC,IAAAA,WAApE;AAAiFC,IAAAA,YAAjF;AAA+FC,IAAAA,WAA/F;AAA4GC,IAAAA,WAA5G;AAAyHC,IAAAA;AAAzH,GAAP;;AAEA,iBAAeT,UAAf,CAA0BU,KAA1B,EAAiCC,KAAK,GAAG,OAAzC,EAAkDC,KAAK,GAAG,KAA1D,EAAiEC,QAAQ,GAAG,IAA5E,EAAkF;AAChFd,IAAAA,MAAM,CAACe,KAAP,CAAc,eAAcJ,KAAM,YAAWC,KAAM,YAAWC,KAAM,eAAcC,QAAS,EAA3F;;AACA,QAAI;AACF,YAAME,WAAW,GAAG,MAAMlB,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAA1B;;AACA,UAAIL,KAAJ,EAAW;AACT,cAAMM,UAAU,CAACN,KAAD,CAAhB;AACAb,QAAAA,MAAM,CAACoB,OAAP,CAAgB,SAAQT,KAAM,eAAcK,WAAW,CAACK,YAAa,WAArE;AACA,eAAOpB,UAAU,CAACU,KAAD,EAAQC,KAAR,CAAjB;AACD;;AAED,UAAII,WAAW,CAACK,YAAZ,GAA2B,CAA/B,EAAkC;AAChCrB,QAAAA,MAAM,CAACe,KAAP,CAAc,eAAcC,WAAW,CAACK,YAAa,MAAKV,KAAM,WAAhE;AACA,eAAO,KAAP;AACD;;AACDX,MAAAA,MAAM,CAACe,KAAP,CAAc,SAAQJ,KAAM,QAAOK,WAAW,CAACK,YAAa,WAA5D;;AAEA,UAAIT,KAAK,KAAK,UAAd,EAA0B;AACxB,eAAOI,WAAW,CAACK,YAAnB;AACD,OAhBC,CAkBF;;;AACA,UAAIT,KAAK,KAAK,KAAd,EAAqB;AACnB,eAAOR,UAAU,CAACO,KAAD,CAAjB;AACD,OArBC,CAuBF;;;AACA,UAAIC,KAAK,KAAK,KAAd,EAAqB;AACnB,eAAOP,UAAU,CAACM,KAAD,CAAjB;AACD,OA1BC,CA4BF;;;AACA,UAAIC,KAAK,KAAK,UAAd,EAA0B;AACxB,eAAOT,eAAe,CAACQ,KAAD,CAAtB;AACD,OA/BC,CAiCF;;;AACA,UAAK,8BAAD,CAAiCW,IAAjC,CAAsCV,KAAtC,CAAJ,EAAkD;AAChDZ,QAAAA,MAAM,CAACe,KAAP,CAAc,4BAA2BH,KAAM,EAA/C;AACA,eAAOW,sBAAsB,CAACZ,KAAD,EAAQC,KAAR,EAAeE,QAAf,CAA7B;AACD,OArCC,CAuCF;;;AACA,UAAIF,KAAK,KAAK,OAAd,EAAuB;AACrB,eAAOV,YAAY,CAACS,KAAD,CAAnB;AACD,OA1CC,CA4CF;;;AACA,YAAM,IAAIa,qBAAJ,CAAaC,oBAAWC,oBAAxB,CAAN;AACD,KA9CD,CA8CE,OAAOC,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;;AAED,aAASR,UAAT,CAAoBN,KAApB,EAA2B;AACzB,UAAIA,KAAJ,EAAW;AACT,eAAOf,OAAO,CAACqB,UAAR,CAAmBR,KAAnB,CAAP;AACD;AACF;AACF;;AAED,iBAAeT,YAAf,CAA4BS,KAA5B,EAAmC;AACjCX,IAAAA,MAAM,CAACe,KAAP,CAAc,wCAAuCJ,KAAM,EAA3D;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAGF;;AACA,YAAMU,WAAW,GAAG,MAAMC,OAAO,CAAClB,KAAD,CAAjC;AAEA,YAAMmB,OAAO,GAAGC,aAAa,CAACH,WAAW,CAAC,CAAD,CAAZ,CAA7B;AACA5B,MAAAA,MAAM,CAACe,KAAP,CAAc,sCAAqCiB,IAAI,CAACC,SAAL,CAAeH,OAAf,CAAwB,EAA3E;AAEA,UAAII,QAAQ,GAAG,CAAf,CATE,CASgB;;AAClB,UAAIC,SAAS,GAAG,CAAhB,CAVE,CAUiB;AAEnB;;AACA,YAAMC,QAAQ,GAAGR,WAAW,CAACS,MAAZ,CAAmBC,OAAO,IAAI;AAC7C,YAAIA,OAAO,CAACC,UAAR,CAAmBT,OAAnB,CAA2BU,SAA3B,KAAyCV,OAAO,CAACU,SAArD,EAAgE;AAC9DN,UAAAA,QAAQ,GADsD,CAClD;;AACZ,iBAAO,IAAP;AACD,SAJ4C,CAM7C;;;AACApC,QAAAA,OAAO,CAAC2C,IAAR,CAAaH,OAAb,EAAsB,KAAtB,EAA6B,IAA7B;AACAH,QAAAA,SAAS,GARoC,CAQhC;;AACb,eAAO,KAAP;AACD,OAVgB,CAAjB;AAYAnC,MAAAA,MAAM,CAACe,KAAP,CAAc,yCAAwCmB,QAAS,eAAcC,SAAU,EAAvF;AACA,YAAMO,OAAO,GAAG,MAAMhC,iBAAiB,CAAC0B,QAAD,CAAvC,CA1BE,CA4BF;;AACA,UAAIA,QAAJ,EAAc;AACZpC,QAAAA,MAAM,CAAC2C,KAAP,CAAc,iBAAgBP,QAAQ,CAACQ,MAAO,eAAcjC,KAAM,GAAlE;AACD;;AAED,aAAO;AAACmB,QAAAA,OAAD;AAAUY,QAAAA,OAAV;AAAmBN,QAAAA;AAAnB,OAAP;AACD,KAlCD,CAkCE,OAAOT,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAeJ,sBAAf,CAAsCZ,KAAtC,EAA6CkC,aAA7C,EAA4D/B,QAAQ,GAAG,IAAvE,EAA6E;AAC3Ed,IAAAA,MAAM,CAACe,KAAP,CAAc,kDAAiDJ,KAAM,EAArE;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAEF;;AACA,YAAMU,WAAW,GAAG,MAAMC,OAAO,CAAClB,KAAD,CAAjC;AAEAX,MAAAA,MAAM,CAACe,KAAP,CAAc,yBAAwB8B,aAAc,EAApD;AAEA,UAAIX,QAAQ,GAAG,CAAf,CAPE,CAOgB;;AAClB,UAAIC,SAAS,GAAG,CAAhB,CARE,CAQiB;AAEnB;;AACA,YAAMC,QAAQ,GAAGR,WAAW,CAACS,MAAZ,CAAmBC,OAAO,IAAI;AAC7C,YAAIA,OAAO,CAACC,UAAR,CAAmBM,aAAnB,KAAqCA,aAAzC,EAAwD;AACtDX,UAAAA,QAAQ,GAD8C,CAC1C;;AACZ,iBAAO,IAAP;AACD,SAJ4C,CAM7C;;;AACApC,QAAAA,OAAO,CAAC2C,IAAR,CAAaH,OAAb,EAAsB,KAAtB,EAA6B,IAA7B;AACAH,QAAAA,SAAS,GARoC,CAQhC;;AACb,eAAO,KAAP;AACD,OAVgB,CAAjB;AAWAnC,MAAAA,MAAM,CAAC2C,KAAP,CAAc,6CAA4CT,QAAS,eAAcC,SAAU,EAA3F;AAEA,YAAML,OAAO,GAAGC,aAAa,CAACK,QAAQ,CAAC,CAAD,CAAT,CAA7B,CAxBE,CA0BF;;AACA,UAAIA,QAAJ,EAAc;AACZpC,QAAAA,MAAM,CAACoB,OAAP,CAAgB,2BAA0BgB,QAAQ,CAACQ,MAAO,iBAAgBjC,KAAM,EAAhF;AACD;;AAED,UAAIG,QAAJ,EAAc;AACZ,cAAM4B,OAAO,GAAG,MAAMhC,iBAAiB,CAAC0B,QAAD,CAAvC;AAEA,eAAO;AAACN,UAAAA,OAAD;AAAUY,UAAAA,OAAV;AAAmBN,UAAAA;AAAnB,SAAP;AACD;;AACD,aAAO;AAACN,QAAAA,OAAD;AAAUM,QAAAA;AAAV,OAAP;AACD,KArCD,CAqCE,OAAOT,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAexB,eAAf,CAA+BQ,KAA/B,EAAsC;AACpCX,IAAAA,MAAM,CAACe,KAAP,CAAc,2CAA0CJ,KAAM,EAA9D;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAEF;;AACA,YAAMU,WAAW,GAAG,MAAMC,OAAO,CAAClB,KAAD,CAAjC;AAEA,YAAMmB,OAAO,GAAGC,aAAa,CAACH,WAAW,CAAC,CAAD,CAAZ,CAA7B;AACA5B,MAAAA,MAAM,CAACe,KAAP,CAAc,sCAAqCiB,IAAI,CAACC,SAAL,CAAeH,OAAf,CAAwB,EAA3E;AAEA,UAAII,QAAQ,GAAG,CAAf,CARE,CAQgB;;AAClB,UAAIC,SAAS,GAAG,CAAhB,CATE,CASiB;AAEnB;;AACA,YAAMC,QAAQ,GAAGR,WAAW,CAACS,MAAZ,CAAmBC,OAAO,IAAI;AAC7C,YAAIA,OAAO,CAACC,UAAR,CAAmBT,OAAnB,CAA2BU,SAA3B,KAAyCV,OAAO,CAACU,SAArD,EAAgE;AAC9DN,UAAAA,QAAQ,GADsD,CAClD;;AAEZ,iBAAO,IAAP;AACD,SAL4C,CAO7C;;;AACApC,QAAAA,OAAO,CAAC2C,IAAR,CAAaH,OAAb,EAAsB,KAAtB,EAA6B,IAA7B;AACAH,QAAAA,SAAS,GAToC,CAShC;;AAEb,eAAO,KAAP;AACD,OAZgB,CAAjB;AAaAnC,MAAAA,MAAM,CAAC2C,KAAP,CAAc,yCAAwCT,QAAS,eAAcC,SAAU,EAAvF,EAzBE,CA2BF;;AACA,UAAIC,QAAJ,EAAc;AACZpC,QAAAA,MAAM,CAACoB,OAAP,CAAgB,oBAAmBgB,QAAQ,CAACQ,MAAO,iBAAgBjC,KAAM,EAAzE;AACD,OA9BC,CAgCF;;;AACA,aAAO;AAACmB,QAAAA,OAAD;AAAUM,QAAAA;AAAV,OAAP;AACD,KAlCD,CAkCE,OAAOT,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAevB,UAAf,CAA0BO,KAA1B,EAAiC;AAC/BX,IAAAA,MAAM,CAACe,KAAP,CAAc,sCAAqCJ,KAAM,EAAzD;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAGF;;AACA,YAAMoB,OAAO,GAAG,MAAMxC,OAAO,CAACgD,GAAR,CAAYnC,KAAZ,CAAtB;;AACA,UAAI2B,OAAJ,EAAa;AACX,cAAMR,OAAO,GAAGC,aAAa,CAACO,OAAD,CAA7B;AACA,cAAMI,OAAO,GAAGhC,iBAAiB,CAAC,CAAC4B,OAAD,CAAD,CAAjC;AACAtC,QAAAA,MAAM,CAACoB,OAAP,CAAgB,0BAAyBT,KAAM,EAA/C;AACA,eAAO;AAACmB,UAAAA,OAAD;AAAUY,UAAAA,OAAV;AAAmBN,UAAAA,QAAQ,EAAE,CAACE,OAAD;AAA7B,SAAP;AACD;;AAED,aAAO,KAAP;AACD,KAbD,CAaE,OAAOX,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAetB,UAAf,CAA0BM,KAA1B,EAAiC;AAC/BX,IAAAA,MAAM,CAACe,KAAP,CAAc,sCAAqCJ,KAAM,EAAzD;;AACA,QAAI;AACF,YAAMb,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAEF;;AACA,aAAO,MAAMpB,OAAO,CAACgD,GAAR,CAAYnC,KAAZ,CAAb;AACD,KAJD,CAIE,OAAOgB,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,WAASrB,WAAT,CAAqB8B,QAArB,EAA+B;AAC7BA,IAAAA,QAAQ,CAACW,OAAT,CAAiBT,OAAO,IAAI;AAC1BtC,MAAAA,MAAM,CAACe,KAAP,CAAc,eAAcuB,OAAO,CAACC,UAAR,CAAmBM,aAAc,EAA7D;AACA/C,MAAAA,OAAO,CAACkD,GAAR,CAAYV,OAAZ;AACD,KAHD;AAID;;AAED,WAAS/B,YAAT,CAAsB6B,QAAtB,EAAgC;AAC9BA,IAAAA,QAAQ,CAACW,OAAT,CAAiBT,OAAO,IAAI;AAC1BtC,MAAAA,MAAM,CAACe,KAAP,CAAc,gBAAeuB,OAAO,CAACC,UAAR,CAAmBM,aAAc,EAA9D;AACA/C,MAAAA,OAAO,CAAC2C,IAAR,CAAaH,OAAb;AACD,KAHD;AAID;;AAED,iBAAe9B,WAAf,CAA2B;AAACG,IAAAA,KAAD;AAAQkC,IAAAA,aAAR;AAAuBf,IAAAA,OAAvB;AAAgCmB,IAAAA;AAAhC,GAA3B,EAAkE;AAChE,QAAI;AACFjD,MAAAA,MAAM,CAACe,KAAP,CAAc,SAAQJ,KAAM,EAA5B;AACAX,MAAAA,MAAM,CAACe,KAAP,CAAc,iBAAgB8B,aAAc,EAA5C;AACA7C,MAAAA,MAAM,CAACe,KAAP,CAAc,QAAOiB,IAAI,CAACC,SAAL,CAAegB,IAAf,CAAqB,EAA1C;AACAjD,MAAAA,MAAM,CAACe,KAAP,CAAc,WAAUiB,IAAI,CAACC,SAAL,CAAeH,OAAf,CAAwB,EAAhD;AAEA,YAAMhC,OAAO,CAACmB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN;AAEApB,MAAAA,OAAO,CAACU,WAAR,CACEG,KADF,EAEEuC,MAAM,CAACC,IAAP,CAAYnB,IAAI,CAACC,SAAL,CAAe;AAACgB,QAAAA;AAAD,OAAf,CAAZ,CAFF,EAGE;AACEJ,QAAAA,aADF;AAEEO,QAAAA,UAAU,EAAE,IAFd;AAGEtB,QAAAA;AAHF,OAHF;AASA9B,MAAAA,MAAM,CAACe,KAAP,CAAc,oBAAmB8B,aAAc,cAAalC,KAAM,EAAlE;AACD,KAlBD,CAkBE,OAAOgB,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAelB,WAAf,CAA2BE,KAA3B,EAAkC;AAChCX,IAAAA,MAAM,CAACoB,OAAP,CAAgB,kBAAiBT,KAAM,GAAvC;AACA,UAAMb,OAAO,CAACuD,WAAR,CAAoB1C,KAApB,CAAN;AACD,GAxQsC,CA0QvC;AACA;AACA;;;AAEA,WAASD,iBAAT,CAA2B0B,QAA3B,EAAqC;AACnCpC,IAAAA,MAAM,CAAC2C,KAAP,CAAc,qBAAoBP,QAAQ,CAACQ,MAAO,cAAlD;AAEA,WAAOR,QAAQ,CAACkB,GAAT,CAAahB,OAAO,IAAI;AAC7B,YAAMiB,OAAO,GAAGvB,IAAI,CAACwB,KAAL,CAAWlB,OAAO,CAACiB,OAAR,CAAgBE,QAAhB,EAAX,CAAhB;AACA,aAAO,IAAIC,sBAAJ,CAAeH,OAAO,CAACN,IAAvB,CAAP;AACD,KAHM,CAAP;AAID;;AAED,iBAAepB,OAAf,CAAuBlB,KAAvB,EAA8B;AAC5BX,IAAAA,MAAM,CAAC2C,KAAP,CAAc,2BAA0BhC,KAAM,EAA9C;;AACA,QAAI;AACF,YAAM;AAACU,QAAAA;AAAD,UAAiB,MAAMvB,OAAO,CAACG,UAAR,CAAmBU,KAAnB,CAA7B;AACAX,MAAAA,MAAM,CAACe,KAAP,CAAc,YAAWM,YAAa,sBAAqBV,KAAM,EAAjE;AACA,YAAMgD,aAAa,GAAGtC,YAAY,IAAIuC,qBAAhB,GAA6BA,qBAA7B,GAA0CvC,YAAhE;AACArB,MAAAA,MAAM,CAACe,KAAP,CAAc,WAAU4C,aAAc,wBAAuBhD,KAAM,EAAnE;AAEA,YAAMyB,QAAQ,GAAG,MAAMyB,IAAI,CAACF,aAAD,CAA3B;AAEA3D,MAAAA,MAAM,CAAC2C,KAAP,CAAc,aAAYP,QAAQ,CAACQ,MAAO,kBAA1C;AAEA,aAAOR,QAAP;AACD,KAXD,CAWE,OAAOT,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;;AAED,mBAAekC,IAAf,CAAoBC,KAApB,EAA2BC,OAAO,GAAG,EAArC,EAAyCC,WAAW,GAAG,EAAvD,EAA2D;AACzD,UAAIF,KAAK,KAAK,CAAd,EAAiB;AACf,eAAOC,OAAP;AACD;;AAED,YAAMzB,OAAO,GAAG,MAAMxC,OAAO,CAACgD,GAAR,CAAYnC,KAAZ,CAAtB;AACA,YAAMsD,UAAU,GAAG;AACjBpB,QAAAA,aAAa,EAAEP,OAAO,CAACC,UAAR,CAAmBM,aADjB;AAEjBqB,QAAAA,WAAW,EAAE5B,OAAO,CAAC6B,MAAR,CAAeD;AAFX,OAAnB,CANyD,CAUzD;;AACA,UAAIF,WAAW,CAACI,QAAZ,CAAqBH,UAArB,CAAJ,EAAsC;AACpC,eAAOJ,IAAI,CAACC,KAAK,GAAG,CAAT,EAAYC,OAAZ,EAAqBC,WAArB,CAAX;AACD;;AAED,aAAOH,IAAI,CAACC,KAAK,GAAG,CAAT,EAAYC,OAAO,CAACM,MAAR,CAAe/B,OAAf,CAAZ,EAAqC0B,WAAW,CAACK,MAAZ,CAAmBJ,UAAnB,CAArC,CAAX;AACD;AACF;;AAED,WAASlC,aAAT,CAAuBkB,IAAvB,EAA6B;AAC3B,WAAOA,IAAI,CAACV,UAAL,CAAgBT,OAAvB;AACD;AACF","sourcesContent":["/* eslint-disable max-statements */\n/* eslint-disable max-lines */\n/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport amqplib from 'amqplib';\nimport {MarcRecord} from '@natlibfi/marc-record';\nimport {Error as ApiError} from '@natlibfi/melinda-commons';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {CHUNK_SIZE} from './constants';\nimport {logError} from './utils';\nimport httpStatus from 'http-status';\n\nexport default async function (AMQP_URL) {\n  const connection = await amqplib.connect(AMQP_URL);\n  const channel = await connection.createChannel();\n  const logger = createLogger();\n\n  return {checkQueue, consumeChunk, consumeRawChunk, consumeOne, consumeRaw, ackMessages, nackMessages, sendToQueue, removeQueue, messagesToRecords};\n\n  async function checkQueue(queue, style = 'basic', purge = false, toRecord = true) {\n    logger.silly(`checkQueue: ${queue}, Style: ${style}: Purge: ${purge}, toRecord: ${toRecord}`);\n    try {\n      const channelInfo = await channel.assertQueue(queue, {durable: true});\n      if (purge) {\n        await purgeQueue(purge);\n        logger.verbose(`Queue ${queue} has purged ${channelInfo.messageCount} messages`);\n        return checkQueue(queue, style);\n      }\n\n      if (channelInfo.messageCount < 1) {\n        logger.silly(`checkQueue: ${channelInfo.messageCount} - ${queue} is empty`);\n        return false;\n      }\n      logger.silly(`Queue ${queue} has ${channelInfo.messageCount} messages`);\n\n      if (style === 'messages') {\n        return channelInfo.messageCount;\n      }\n\n      // Note: returns a message + a record\n      if (style === 'one') {\n        return consumeOne(queue);\n      }\n\n      // Note: returns just a message\n      if (style === 'raw') {\n        return consumeRaw(queue);\n      }\n\n      // Note: returns a chunk of messages + records\n      if (style === 'rawChunk') {\n        return consumeRawChunk(queue);\n      }\n\n      // Note: returns a chunk of messages or a chunk of messages + records (depending on toRecord)\n      if ((/^.{8}-.{4}-.{4}-.{4}-.{12}$/u).test(style)) {\n        logger.silly(`checkQueue: regexp match ${style}`);\n        return consumeByCorrelationId(queue, style, toRecord);\n      }\n\n      // Note: returns a chunk of messages + records\n      if (style === 'basic') {\n        return consumeChunk(queue);\n      }\n\n      // Defaults:\n      throw new ApiError(httpStatus.UNPROCESSABLE_ENTITY);\n    } catch (error) {\n      logError(error);\n    }\n\n    function purgeQueue(purge) {\n      if (purge) {\n        return channel.purgeQueue(queue);\n      }\n    }\n  }\n\n  async function consumeChunk(queue) {\n    logger.silly(`Prepared to consumeChunk from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n\n      // getData: next chunk (100) messages\n      const queMessages = await getData(queue);\n\n      const headers = getHeaderInfo(queMessages[0]);\n      logger.silly(`Filtering messages by cataloger in ${JSON.stringify(headers)}`);\n\n      let filterIn = 0; // eslint-disable-line functional/no-let\n      let filterOut = 0; // eslint-disable-line functional/no-let\n\n      // Check that cataloger match! headers\n      const messages = queMessages.filter(message => {\n        if (message.properties.headers.cataloger === headers.cataloger) {\n          filterIn++; // eslint-disable-line no-plusplus\n          return true;\n        }\n\n        // Nack unwanted ones\n        channel.nack(message, false, true);\n        filterOut++; // eslint-disable-line no-plusplus\n        return false;\n      });\n\n      logger.silly(`Filtering by cataloger result: valid: ${filterIn} non-valid: ${filterOut}`);\n      const records = await messagesToRecords(messages);\n\n      // eslint-disable-next-line functional/no-conditional-statement\n      if (messages) {\n        logger.debug(`consumeChunk (${messages.length} from queue ${queue})`);\n      }\n\n      return {headers, records, messages};\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeByCorrelationId(queue, correlationId, toRecord = true) {\n    logger.silly(`Prepared to consumeByCorrelationId from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n      // get next chunk (100) messages\n      const queMessages = await getData(queue);\n\n      logger.silly(`Filtering messages by ${correlationId}`);\n\n      let filterIn = 0; // eslint-disable-line functional/no-let\n      let filterOut = 0; // eslint-disable-line functional/no-let\n\n      // Check that correlationId matches query\n      const messages = queMessages.filter(message => {\n        if (message.properties.correlationId === correlationId) {\n          filterIn++; // eslint-disable-line no-plusplus\n          return true;\n        }\n\n        // Nack unwanted ones\n        channel.nack(message, false, true);\n        filterOut++; // eslint-disable-line no-plusplus\n        return false;\n      });\n      logger.debug(`Filtering by correlationId result: valid: ${filterIn} non-valid: ${filterOut}`);\n\n      const headers = getHeaderInfo(messages[0]);\n\n      // eslint-disable-next-line functional/no-conditional-statement\n      if (messages) {\n        logger.verbose(`consumeByCorrelationId (${messages.length}) from queue: ${queue}`);\n      }\n\n      if (toRecord) {\n        const records = await messagesToRecords(messages);\n\n        return {headers, records, messages};\n      }\n      return {headers, messages};\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeRawChunk(queue) {\n    logger.silly(`Prepared to consumeRawChunk from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n      // Get next chunk (100) messages\n      const queMessages = await getData(queue);\n\n      const headers = getHeaderInfo(queMessages[0]);\n      logger.silly(`Filtering messages by cataloger in ${JSON.stringify(headers)}`);\n\n      let filterIn = 0; // eslint-disable-line functional/no-let\n      let filterOut = 0; // eslint-disable-line functional/no-let\n\n      // Check that cataloger match! headers\n      const messages = queMessages.filter(message => {\n        if (message.properties.headers.cataloger === headers.cataloger) {\n          filterIn++; // eslint-disable-line no-plusplus\n\n          return true;\n        }\n\n        // Nack unwanted ones\n        channel.nack(message, false, true);\n        filterOut++; // eslint-disable-line no-plusplus\n\n        return false;\n      });\n      logger.debug(`Filtering by cataloger result: valid: ${filterIn} non-valid: ${filterOut}`);\n\n      // eslint-disable-next-line functional/no-conditional-statement\n      if (messages) {\n        logger.verbose(`consumeRawChunk (${messages.length}) from queue: ${queue}`);\n      }\n\n      // return raw (do not convert messages to records)\n      return {headers, messages};\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeOne(queue) {\n    logger.silly(`Prepared to consumeOne from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n\n      // Returns false if 0 items in queue\n      const message = await channel.get(queue);\n      if (message) {\n        const headers = getHeaderInfo(message);\n        const records = messagesToRecords([message]);\n        logger.verbose(`consumeOne from queue: ${queue}`);\n        return {headers, records, messages: [message]};\n      }\n\n      return false;\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function consumeRaw(queue) {\n    logger.silly(`Prepared to consumeRaw from queue: ${queue}`);\n    try {\n      await channel.assertQueue(queue, {durable: true});\n      // Returns false if 0 items in queue\n      return await channel.get(queue);\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  function ackMessages(messages) {\n    messages.forEach(message => {\n      logger.silly(`Ack message ${message.properties.correlationId}`);\n      channel.ack(message);\n    });\n  }\n\n  function nackMessages(messages) {\n    messages.forEach(message => {\n      logger.silly(`Nack message ${message.properties.correlationId}`);\n      channel.nack(message);\n    });\n  }\n\n  async function sendToQueue({queue, correlationId, headers, data}) {\n    try {\n      logger.silly(`Queue ${queue}`);\n      logger.silly(`CorrelationId ${correlationId}`);\n      logger.silly(`Data ${JSON.stringify(data)}`);\n      logger.silly(`Headers ${JSON.stringify(headers)}`);\n\n      await channel.assertQueue(queue, {durable: true});\n\n      channel.sendToQueue(\n        queue,\n        Buffer.from(JSON.stringify({data})),\n        {\n          correlationId,\n          persistent: true,\n          headers\n        }\n      );\n      logger.silly(`Send message for ${correlationId} to queue: ${queue}`);\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function removeQueue(queue) {\n    logger.verbose(`Removing queue ${queue}.`);\n    await channel.deleteQueue(queue);\n  }\n\n  // ----------------\n  // Helper functions\n  // ----------------\n\n  function messagesToRecords(messages) {\n    logger.debug(`Parsing messages (${messages.length}) to records`);\n\n    return messages.map(message => {\n      const content = JSON.parse(message.content.toString());\n      return new MarcRecord(content.data);\n    });\n  }\n\n  async function getData(queue) {\n    logger.debug(`Getting queue data from ${queue}`);\n    try {\n      const {messageCount} = await channel.checkQueue(queue);\n      logger.silly(`There is ${messageCount} messages in queue ${queue}`);\n      const messagesToGet = messageCount >= CHUNK_SIZE ? CHUNK_SIZE : messageCount;\n      logger.silly(`Getting ${messagesToGet} messages from queue ${queue}`);\n\n      const messages = await pump(messagesToGet);\n\n      logger.debug(`Returning ${messages.length} unique messages`);\n\n      return messages;\n    } catch (error) {\n      logError(error);\n    }\n\n    async function pump(count, results = [], identifiers = []) {\n      if (count === 0) {\n        return results;\n      }\n\n      const message = await channel.get(queue);\n      const identifier = {\n        correlationId: message.properties.correlationId,\n        deliveryTag: message.fields.deliveryTag\n      };\n      // Filter not unique messages\n      if (identifiers.includes(identifier)) {\n        return pump(count - 1, results, identifiers);\n      }\n\n      return pump(count - 1, results.concat(message), identifiers.concat(identifier));\n    }\n  }\n\n  function getHeaderInfo(data) {\n    return data.properties.headers;\n  }\n}\n"],"file":"amqp.js"}

package/dist/helpers/conversion.js

@@ -56,7 +56,7 @@ function _default() {
   };
 
   function serialize(record, format) {
-    logger.verbose('Serializing record');
+    logger.verbose(`Serializing record to ${format}`);
 
     try {
       if (format === _constants.conversionFormats.MARCXML) {
@@ -87,24 +87,24 @@ function _default() {
   function unserialize(data, format, validationOptions = {
     subfieldValues: false
   }) {
-    logger.verbose('Unserializing record');
+    logger.verbose(`Unserializing record from ${format}`);
     logger.silly(`Format: ${format}`);
-    logger.silly(`Validation options: ${JSON.stringify(validationOptions)}`);
+    logger.debug(`Validation options: ${JSON.stringify(validationOptions)}`);
     logger.silly(`Data: ${JSON.stringify(data)}`);
 
     try {
       if (format === _constants.conversionFormats.MARCXML) {
-        logger.verbose('Unserialize format marcxml');
+        logger.silly('Unserialize format marcxml');
         return _marcRecordSerializers.MARCXML.from(data, validationOptions);
       }
 
       if (format === _constants.conversionFormats.ISO2709) {
-        logger.verbose('Unserialize format iso2709');
+        logger.silly('Unserialize format iso2709');
         return _marcRecordSerializers.ISO2709.from(data, validationOptions);
       }
 
       if (format === _constants.conversionFormats.JSON) {
-        logger.verbose('Unserialize format json');
+        logger.silly('Unserialize format json');
         return _marcRecordSerializers.Json.from(data, validationOptions);
       }
 

package/dist/helpers/conversion.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/helpers/conversion.js"],"names":["logger","serialize","unserialize","record","format","verbose","conversionFormats","MARCXML","to","ISO2709","JSON","Json","ConversionError","httpStatus","UNSUPPORTED_MEDIA_TYPE","err","BAD_REQUEST","data","validationOptions","subfieldValues","silly","stringify","from"],"mappings":";;;;;;;AA6BA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AAlCA;;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AASe,oBAAY;AACzB,QAAMA,MAAM,GAAG,0CAAf;AAEA,SAAO;AAACC,IAAAA,SAAD;AAAYC,IAAAA;AAAZ,GAAP;;AAEA,WAASD,SAAT,CAAmBE,MAAnB,EAA2BC,MAA3B,EAAmC;AACjCJ,IAAAA,MAAM,CAACK,OAAP,CAAe,oBAAf;;AACA,QAAI;AACF,UAAID,MAAM,KAAKE,6BAAkBC,OAAjC,EAA0C;AACxC,eAAOA,+BAAQC,EAAR,CAAWL,MAAX,CAAP;AACD;;AAED,UAAIC,MAAM,KAAKE,6BAAkBG,OAAjC,EAA0C;AACxC,eAAOA,+BAAQD,EAAR,CAAWL,MAAX,CAAP;AACD;;AAED,UAAIC,MAAM,KAAKE,6BAAkBI,IAAjC,EAAuC;AACrC,eAAOC,4BAAKH,EAAL,CAAQL,MAAR,CAAP;AACD;;AAED,YAAM,IAAIS,qBAAJ,CAAoBC,oBAAWC,sBAA/B,CAAN;AACD,KAdD,CAcE,OAAOC,GAAP,EAAY;AACZ,2BAASA,GAAT;;AACA,UAAIA,GAAG,YAAYH,qBAAnB,EAAoC;AAAE;AACpC,cAAMG,GAAN;AACD;;AACD,YAAM,IAAIH,qBAAJ,CAAoBC,oBAAWG,WAA/B,EAA4C,gCAA5C,CAAN;AACD;AACF;;AAED,WAASd,WAAT,CAAqBe,IAArB,EAA2Bb,MAA3B,EAAmCc,iBAAiB,GAAG;AAACC,IAAAA,cAAc,EAAE;AAAjB,GAAvD,EAAgF;AAC9EnB,IAAAA,MAAM,CAACK,OAAP,CAAe,sBAAf;AACAL,IAAAA,MAAM,CAACoB,KAAP,CAAc,WAAUhB,MAAO,EAA/B;AACAJ,IAAAA,MAAM,CAACoB,KAAP,CAAc,uBAAsBV,IAAI,CAACW,SAAL,CAAeH,iBAAf,CAAkC,EAAtE;AACAlB,IAAAA,MAAM,CAACoB,KAAP,CAAc,SAAQV,IAAI,CAACW,SAAL,CAAeJ,IAAf,CAAqB,EAA3C;;AACA,QAAI;AACF,UAAIb,MAAM,KAAKE,6BAAkBC,OAAjC,EAA0C;AACxCP,QAAAA,MAAM,CAACK,OAAP,CAAe,4BAAf;AACA,eAAOE,+BAAQe,IAAR,CAAaL,IAAb,EAAmBC,iBAAnB,CAAP;AACD;;AAED,UAAId,MAAM,KAAKE,6BAAkBG,OAAjC,EAA0C;AACxCT,QAAAA,MAAM,CAACK,OAAP,CAAe,4BAAf;AACA,eAAOI,+BAAQa,IAAR,CAAaL,IAAb,EAAmBC,iBAAnB,CAAP;AACD;;AAED,UAAId,MAAM,KAAKE,6BAAkBI,IAAjC,EAAuC;AACrCV,QAAAA,MAAM,CAACK,OAAP,CAAe,yBAAf;AACA,eAAOM,4BAAKW,IAAL,CAAUL,IAAV,EAAgBC,iBAAhB,CAAP;AACD;;AAED,YAAM,IAAIN,qBAAJ,CAAoBC,oBAAWC,sBAA/B,CAAN;AACD,KAjBD,CAiBE,OAAOC,GAAP,EAAY;AACZ,2BAASA,GAAT;;AACA,UAAIA,GAAG,YAAYH,qBAAnB,EAAoC;AAAE;AACpC,cAAMG,GAAN;AACD;;AACD,YAAM,IAAIH,qBAAJ,CAAoBC,oBAAWG,WAA/B,EAA4C,kCAA5C,CAAN;AACD;AACF;AACF","sourcesContent":["/* eslint-disable max-statements */\n/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport httpStatus from 'http-status';\nimport {MARCXML, ISO2709, Json} from '@natlibfi/marc-record-serializers';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {Error as ConversionError} from '@natlibfi/melinda-commons';\nimport {conversionFormats} from '../constants';\nimport {logError} from '../utils';\n\nexport default function () {\n  const logger = createLogger();\n\n  return {serialize, unserialize};\n\n  function serialize(record, format) {\n    logger.verbose('Serializing record');\n    try {\n      if (format === conversionFormats.MARCXML) {\n        return MARCXML.to(record);\n      }\n\n      if (format === conversionFormats.ISO2709) {\n        return ISO2709.to(record);\n      }\n\n      if (format === conversionFormats.JSON) {\n        return Json.to(record);\n      }\n\n      throw new ConversionError(httpStatus.UNSUPPORTED_MEDIA_TYPE);\n    } catch (err) {\n      logError(err);\n      if (err instanceof ConversionError) { // eslint-disable-line functional/no-conditional-statement\n        throw err;\n      }\n      throw new ConversionError(httpStatus.BAD_REQUEST, 'Error while serializing record');\n    }\n  }\n\n  function unserialize(data, format, validationOptions = {subfieldValues: false}) {\n    logger.verbose('Unserializing record');\n    logger.silly(`Format: ${format}`);\n    logger.silly(`Validation options: ${JSON.stringify(validationOptions)}`);\n    logger.silly(`Data: ${JSON.stringify(data)}`);\n    try {\n      if (format === conversionFormats.MARCXML) {\n        logger.verbose('Unserialize format marcxml');\n        return MARCXML.from(data, validationOptions);\n      }\n\n      if (format === conversionFormats.ISO2709) {\n        logger.verbose('Unserialize format iso2709');\n        return ISO2709.from(data, validationOptions);\n      }\n\n      if (format === conversionFormats.JSON) {\n        logger.verbose('Unserialize format json');\n        return Json.from(data, validationOptions);\n      }\n\n      throw new ConversionError(httpStatus.UNSUPPORTED_MEDIA_TYPE);\n    } catch (err) {\n      logError(err);\n      if (err instanceof ConversionError) { // eslint-disable-line functional/no-conditional-statement\n        throw err;\n      }\n      throw new ConversionError(httpStatus.BAD_REQUEST, 'Error while unserializing record');\n    }\n  }\n}\n"],"file":"conversion.js"}
+{"version":3,"sources":["../../src/helpers/conversion.js"],"names":["logger","serialize","unserialize","record","format","verbose","conversionFormats","MARCXML","to","ISO2709","JSON","Json","ConversionError","httpStatus","UNSUPPORTED_MEDIA_TYPE","err","BAD_REQUEST","data","validationOptions","subfieldValues","silly","debug","stringify","from"],"mappings":";;;;;;;AA6BA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AAlCA;;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AASe,oBAAY;AACzB,QAAMA,MAAM,GAAG,0CAAf;AAEA,SAAO;AAACC,IAAAA,SAAD;AAAYC,IAAAA;AAAZ,GAAP;;AAEA,WAASD,SAAT,CAAmBE,MAAnB,EAA2BC,MAA3B,EAAmC;AACjCJ,IAAAA,MAAM,CAACK,OAAP,CAAgB,yBAAwBD,MAAO,EAA/C;;AACA,QAAI;AACF,UAAIA,MAAM,KAAKE,6BAAkBC,OAAjC,EAA0C;AACxC,eAAOA,+BAAQC,EAAR,CAAWL,MAAX,CAAP;AACD;;AAED,UAAIC,MAAM,KAAKE,6BAAkBG,OAAjC,EAA0C;AACxC,eAAOA,+BAAQD,EAAR,CAAWL,MAAX,CAAP;AACD;;AAED,UAAIC,MAAM,KAAKE,6BAAkBI,IAAjC,EAAuC;AACrC,eAAOC,4BAAKH,EAAL,CAAQL,MAAR,CAAP;AACD;;AAED,YAAM,IAAIS,qBAAJ,CAAoBC,oBAAWC,sBAA/B,CAAN;AACD,KAdD,CAcE,OAAOC,GAAP,EAAY;AACZ,2BAASA,GAAT;;AACA,UAAIA,GAAG,YAAYH,qBAAnB,EAAoC;AAAE;AACpC,cAAMG,GAAN;AACD;;AACD,YAAM,IAAIH,qBAAJ,CAAoBC,oBAAWG,WAA/B,EAA4C,gCAA5C,CAAN;AACD;AACF;;AAED,WAASd,WAAT,CAAqBe,IAArB,EAA2Bb,MAA3B,EAAmCc,iBAAiB,GAAG;AAACC,IAAAA,cAAc,EAAE;AAAjB,GAAvD,EAAgF;AAC9EnB,IAAAA,MAAM,CAACK,OAAP,CAAgB,6BAA4BD,MAAO,EAAnD;AACAJ,IAAAA,MAAM,CAACoB,KAAP,CAAc,WAAUhB,MAAO,EAA/B;AACAJ,IAAAA,MAAM,CAACqB,KAAP,CAAc,uBAAsBX,IAAI,CAACY,SAAL,CAAeJ,iBAAf,CAAkC,EAAtE;AACAlB,IAAAA,MAAM,CAACoB,KAAP,CAAc,SAAQV,IAAI,CAACY,SAAL,CAAeL,IAAf,CAAqB,EAA3C;;AACA,QAAI;AACF,UAAIb,MAAM,KAAKE,6BAAkBC,OAAjC,EAA0C;AACxCP,QAAAA,MAAM,CAACoB,KAAP,CAAa,4BAAb;AACA,eAAOb,+BAAQgB,IAAR,CAAaN,IAAb,EAAmBC,iBAAnB,CAAP;AACD;;AAED,UAAId,MAAM,KAAKE,6BAAkBG,OAAjC,EAA0C;AACxCT,QAAAA,MAAM,CAACoB,KAAP,CAAa,4BAAb;AACA,eAAOX,+BAAQc,IAAR,CAAaN,IAAb,EAAmBC,iBAAnB,CAAP;AACD;;AAED,UAAId,MAAM,KAAKE,6BAAkBI,IAAjC,EAAuC;AACrCV,QAAAA,MAAM,CAACoB,KAAP,CAAa,yBAAb;AACA,eAAOT,4BAAKY,IAAL,CAAUN,IAAV,EAAgBC,iBAAhB,CAAP;AACD;;AAED,YAAM,IAAIN,qBAAJ,CAAoBC,oBAAWC,sBAA/B,CAAN;AACD,KAjBD,CAiBE,OAAOC,GAAP,EAAY;AACZ,2BAASA,GAAT;;AACA,UAAIA,GAAG,YAAYH,qBAAnB,EAAoC;AAAE;AACpC,cAAMG,GAAN;AACD;;AACD,YAAM,IAAIH,qBAAJ,CAAoBC,oBAAWG,WAA/B,EAA4C,kCAA5C,CAAN;AACD;AACF;AACF","sourcesContent":["/* eslint-disable max-statements */\n/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport httpStatus from 'http-status';\nimport {MARCXML, ISO2709, Json} from '@natlibfi/marc-record-serializers';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {Error as ConversionError} from '@natlibfi/melinda-commons';\nimport {conversionFormats} from '../constants';\nimport {logError} from '../utils';\n\nexport default function () {\n  const logger = createLogger();\n\n  return {serialize, unserialize};\n\n  function serialize(record, format) {\n    logger.verbose(`Serializing record to ${format}`);\n    try {\n      if (format === conversionFormats.MARCXML) {\n        return MARCXML.to(record);\n      }\n\n      if (format === conversionFormats.ISO2709) {\n        return ISO2709.to(record);\n      }\n\n      if (format === conversionFormats.JSON) {\n        return Json.to(record);\n      }\n\n      throw new ConversionError(httpStatus.UNSUPPORTED_MEDIA_TYPE);\n    } catch (err) {\n      logError(err);\n      if (err instanceof ConversionError) { // eslint-disable-line functional/no-conditional-statement\n        throw err;\n      }\n      throw new ConversionError(httpStatus.BAD_REQUEST, 'Error while serializing record');\n    }\n  }\n\n  function unserialize(data, format, validationOptions = {subfieldValues: false}) {\n    logger.verbose(`Unserializing record from ${format}`);\n    logger.silly(`Format: ${format}`);\n    logger.debug(`Validation options: ${JSON.stringify(validationOptions)}`);\n    logger.silly(`Data: ${JSON.stringify(data)}`);\n    try {\n      if (format === conversionFormats.MARCXML) {\n        logger.silly('Unserialize format marcxml');\n        return MARCXML.from(data, validationOptions);\n      }\n\n      if (format === conversionFormats.ISO2709) {\n        logger.silly('Unserialize format iso2709');\n        return ISO2709.from(data, validationOptions);\n      }\n\n      if (format === conversionFormats.JSON) {\n        logger.silly('Unserialize format json');\n        return Json.from(data, validationOptions);\n      }\n\n      throw new ConversionError(httpStatus.UNSUPPORTED_MEDIA_TYPE);\n    } catch (err) {\n      logError(err);\n      if (err instanceof ConversionError) { // eslint-disable-line functional/no-conditional-statement\n        throw err;\n      }\n      throw new ConversionError(httpStatus.BAD_REQUEST, 'Error while unserializing record');\n    }\n  }\n}\n"],"file":"conversion.js"}

package/dist/helpers/validation.js

@@ -42,7 +42,7 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
 */
 var _default = async () => {
   const logger = (0, _melindaBackendCommons.createLogger)();
-  logger.verbose('Running validations');
+  logger.verbose('Running inbuilt record validations');
   const validate = (0, _marcRecordValidate.default)([await (0, _marcRecordValidatorsMelinda.FieldStructure)([{
     tag: /^003$/u,
     valuePattern: /^FI-MELINDA$/u

package/dist/helpers/validation.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/helpers/validation.js"],"names":["logger","verbose","validate","tag","valuePattern","unvalidRecord","record","valid","report","fix","validateFixes","subfieldValues","failed","messages"],"mappings":";;;;;;;AA4BA;;AACA;;AAGA;;;;AAhCA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;eAQe,YAAY;AACzB,QAAMA,MAAM,GAAG,0CAAf;AAEAA,EAAAA,MAAM,CAACC,OAAP,CAAe,qBAAf;AACA,QAAMC,QAAQ,GAAG,iCAAgB,CAAC,MAAM,iDAAe,CAAC;AAACC,IAAAA,GAAG,EAAE,QAAN;AAAgBC,IAAAA,YAAY,EAAE;AAA9B,GAAD,CAAf,CAAP,CAAhB,CAAjB;AAEA,SAAO,MAAMC,aAAN,IAAuB;AAC5B,UAAM;AAACC,MAAAA,MAAD;AAASC,MAAAA,KAAT;AAAgBC,MAAAA;AAAhB,QAA0B,MAAMN,QAAQ,CAACG,aAAD,EAAgB;AAACI,MAAAA,GAAG,EAAE,KAAN;AAAaC,MAAAA,aAAa,EAAE;AAA5B,KAAhB,EAAoD;AAACC,MAAAA,cAAc,EAAE;AAAjB,KAApD,CAA9C;AAEA,WAAO;AACLL,MAAAA,MADK;AAELM,MAAAA,MAAM,EAAEL,KAAK,KAAK,KAFb;AAGLM,MAAAA,QAAQ,EAAEL;AAHL,KAAP;AAKD,GARD;AASD,C","sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport validateFactory from '@natlibfi/marc-record-validate';\nimport {\n  FieldStructure as fieldStructure\n} from '@natlibfi/marc-record-validators-melinda';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\n\nexport default async () => {\n  const logger = createLogger();\n\n  logger.verbose('Running validations');\n  const validate = validateFactory([await fieldStructure([{tag: /^003$/u, valuePattern: /^FI-MELINDA$/u}])]);\n\n  return async unvalidRecord => {\n    const {record, valid, report} = await validate(unvalidRecord, {fix: false, validateFixes: false}, {subfieldValues: false});\n\n    return {\n      record,\n      failed: valid === false,\n      messages: report\n    };\n  };\n};\n"],"file":"validation.js"}
+{"version":3,"sources":["../../src/helpers/validation.js"],"names":["logger","verbose","validate","tag","valuePattern","unvalidRecord","record","valid","report","fix","validateFixes","subfieldValues","failed","messages"],"mappings":";;;;;;;AA4BA;;AACA;;AAGA;;;;AAhCA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;eAQe,YAAY;AACzB,QAAMA,MAAM,GAAG,0CAAf;AAEAA,EAAAA,MAAM,CAACC,OAAP,CAAe,oCAAf;AACA,QAAMC,QAAQ,GAAG,iCAAgB,CAAC,MAAM,iDAAe,CAAC;AAACC,IAAAA,GAAG,EAAE,QAAN;AAAgBC,IAAAA,YAAY,EAAE;AAA9B,GAAD,CAAf,CAAP,CAAhB,CAAjB;AAEA,SAAO,MAAMC,aAAN,IAAuB;AAC5B,UAAM;AAACC,MAAAA,MAAD;AAASC,MAAAA,KAAT;AAAgBC,MAAAA;AAAhB,QAA0B,MAAMN,QAAQ,CAACG,aAAD,EAAgB;AAACI,MAAAA,GAAG,EAAE,KAAN;AAAaC,MAAAA,aAAa,EAAE;AAA5B,KAAhB,EAAoD;AAACC,MAAAA,cAAc,EAAE;AAAjB,KAApD,CAA9C;AAEA,WAAO;AACLL,MAAAA,MADK;AAELM,MAAAA,MAAM,EAAEL,KAAK,KAAK,KAFb;AAGLM,MAAAA,QAAQ,EAAEL;AAHL,KAAP;AAKD,GARD;AASD,C","sourcesContent":["/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport validateFactory from '@natlibfi/marc-record-validate';\nimport {\n  FieldStructure as fieldStructure\n} from '@natlibfi/marc-record-validators-melinda';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\n\nexport default async () => {\n  const logger = createLogger();\n\n  logger.verbose('Running inbuilt record validations');\n  const validate = validateFactory([await fieldStructure([{tag: /^003$/u, valuePattern: /^FI-MELINDA$/u}])]);\n\n  return async unvalidRecord => {\n    const {record, valid, report} = await validate(unvalidRecord, {fix: false, validateFixes: false}, {subfieldValues: false});\n\n    return {\n      record,\n      failed: valid === false,\n      messages: report\n    };\n  };\n};\n"],"file":"validation.js"}

package/dist/mongo.js

@@ -133,7 +133,7 @@ async function _default(MONGO_URI, collection) {
       const result = await db.collection(collection).insertOne(newQueueItem);
 
       if (result.acknowledged) {
-        logger.info(`New prio queue item for ${correlationId} has been made in ${collection}`);
+        logger.info(`New PRIO queue item for ${operation} ${correlationId} has been made in ${collection}`);
         return;
       }
 
@@ -171,7 +171,7 @@ async function _default(MONGO_URI, collection) {
     try {
       // No await here, promises later
       db.collection(collection).insertOne(newQueueItem);
-      logger.info(`New bulk queue item for ${correlationId} has been made in ${collection}!`);
+      logger.info(`New BULK queue item for ${operation} ${correlationId} has been made in ${collection}!`);
     } catch (error) {
       (0, _utils.logError)(error);
       throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR);
@@ -261,7 +261,7 @@ async function _default(MONGO_URI, collection) {
   }
 
   async function remove(params) {
-    logger.debug(`${JSON.stringify(params)}`);
+    logger.silly(`${JSON.stringify(params)}`);
     logger.info(`Removing from Mongo (${collection}) id: ${params.correlationId}`);
     const clean = (0, _mongoSanitize.default)(params.correlationId);
     logger.silly(`mongo/remove: clean: ${JSON.stringify(clean)}`);
@@ -279,16 +279,15 @@ async function _default(MONGO_URI, collection) {
       }
     } catch (err) {
       if (err instanceof _mongodb.MongoDriverError) {
-        logger.error(err.message);
-
         if (err.message.indexOf('File not found for id') !== -1) {
-          logger.debug(`mongo/remove: File not found, removing queueItem ${JSON.stringify(clean)} from ${collection}`);
+          logger.silly(`mongo/remove: File not found, removing queueItem ${JSON.stringify(clean)} from ${collection}`);
           await db.collection(collection).deleteOne({
             correlationId: clean
           });
           return true;
         }
 
+        logger.error(err.message);
         throw new _melindaCommons.Error(_httpStatus.default.INTERNAL_SERVER_ERROR, err.message);
       }
 
@@ -297,7 +296,7 @@ async function _default(MONGO_URI, collection) {
   }
 
   async function readContent(correlationId) {
-    logger.info(`Reading content for id: ${correlationId} in ${collection}`);
+    logger.info(`Reading content from mongo for id: ${correlationId} in ${collection}`);
     const clean = (0, _mongoSanitize.default)(correlationId);
     const result = await db.collection(collection).findOne({
       correlationId: clean
@@ -314,13 +313,13 @@ async function _default(MONGO_URI, collection) {
   }
 
   async function removeContent(params) {
-    logger.info(`Removing content for id: ${params.correlationId} in ${collection}`);
+    logger.info(`Removing content from mongo for id: ${params.correlationId} in ${collection}`);
     const clean = (0, _mongoSanitize.default)(params.correlationId);
     const result = await db.collection(collection).findOne({
       correlationId: clean
     }); // njsscan-ignore: node_nosqli_injection
 
-    logger.debug(`mongo/removeContent: result ${JSON.stringify(result)}`);
+    logger.silly(`mongo/removeContent: result ${JSON.stringify(result)}`);
 
     if (result) {
       await gridFSBucket.delete(clean);
@@ -358,7 +357,7 @@ async function _default(MONGO_URI, collection) {
   }
 
   function getStream(correlationId) {
-    logger.info(`Forming stream from db: ${correlationId} in ${collection}`);
+    logger.info(`Forming stream from mongo for ${correlationId} in ${collection}`);
     const clean = (0, _mongoSanitize.default)(correlationId);
 
     try {
@@ -374,8 +373,8 @@ async function _default(MONGO_URI, collection) {
     handledIds,
     rejectedIds
   }) {
-    logger.info(`Push queue-item ids to list: ${correlationId} to ${collection}`);
-    logger.debug(`ids: ${JSON.stringify(handledIds)}, rejectedIds: ${JSON.stringify(rejectedIds)}`);
+    logger.verbose(`Push ids (${handledIds.length}) and rejectedIds (${rejectedIds.length}) ${correlationId} to ${collection}`);
+    logger.debug(`ids (${handledIds.length}): ${JSON.stringify(handledIds)}, rejectedIds ${rejectedIds.length}: ${JSON.stringify(rejectedIds)}`);
     const clean = (0, _mongoSanitize.default)(correlationId);
     await db.collection(collection).updateOne({
       correlationId: clean
@@ -399,8 +398,8 @@ async function _default(MONGO_URI, collection) {
     messages,
     messageField = 'messages'
   }) {
-    logger.info(`Push messages to ${messageField} ${correlationId} to ${collection}`);
-    logger.debug(`Messages: ${JSON.stringify(messages)}}`);
+    logger.verbose(`Push messages (${messages.length}) to ${messageField} ${correlationId} to ${collection}`);
+    logger.silly(`Messages (${messages.length}): ${JSON.stringify(messages)}}`);
     const clean = (0, _mongoSanitize.default)(correlationId);
     const cleanMessageField = (0, _mongoSanitize.default)(messageField);
     await db.collection(collection).updateOne({
@@ -424,7 +423,7 @@ async function _default(MONGO_URI, collection) {
     errorStatus = ''
   }) {
     const errorString = errorMessage || errorStatus ? `, Error message: '${errorMessage}', Error status: '${errorStatus}'` : '';
-    logger.log('info', `Setting queue-item state: ${correlationId}${state}${errorString} to ${collection}`);
+    logger.info(`Setting queue-item state ${state} for ${correlationId}${errorString} to ${collection}`);
     const clean = (0, _mongoSanitize.default)(correlationId);
     return db.collection(collection).findOneAndUpdate({
       correlationId: clean

package/dist/mongo.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/mongo.js"],"names":["MONGO_URI","collection","logger","client","MongoClient","connect","useNewUrlParser","useUnifiedTopology","db","gridFSBucket","GridFSBucket","bucketName","createPrio","createBulk","checkAndSetState","query","queryById","remove","readContent","removeContent","getOne","getStream","setState","pushIds","pushMessages","correlationId","cataloger","oCatalogerIn","operation","noop","undefined","unique","prio","time","toDate","newQueueItem","operationSettings","queueItemState","QUEUE_ITEM_STATE","VALIDATOR","PENDING_VALIDATION","creationTime","modificationTime","handledIds","rejectedIds","result","insertOne","acknowledged","info","ApiError","httpStatus","INTERNAL_SERVER_ERROR","error","contentType","recordLoadParams","stream","UPLOADING","Promise","resolve","reject","outputStream","openUploadStream","on","chunk","write","end","state","errorMessage","errorStatus","timeOut","checkTimeOut","params","find","projection","_id","toArray","debug","length","checkModTime","findOne","timeoutTime","add","silly","isBefore","oldState","ABORT","REQUEST_TIMEOUT","JSON","stringify","clean","noContent","deleteOne","err","MongoDriverError","message","indexOf","readStream","openDownloadStreamByName","NOT_FOUND","delete","clean2","updateOne","$set","$push","$each","messages","messageField","cleanMessageField","errorString","log","findOneAndUpdate","returnNewDocument"],"mappings":";;;;;;;AA8BA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AArCA;;AACA;;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAWA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAEe,wBAAgBA,SAAhB,EAA2BC,UAA3B,EAAuC;AACpD,QAAMC,MAAM,GAAG,0CAAf,CADoD,CAGpD;;AACA,QAAMC,MAAM,GAAG,MAAMC,qBAAYC,OAAZ,CAAoBL,SAApB,EAA+B;AAACM,IAAAA,eAAe,EAAE,IAAlB;AAAwBC,IAAAA,kBAAkB,EAAE;AAA5C,GAA/B,CAArB;AACA,QAAMC,EAAE,GAAGL,MAAM,CAACK,EAAP,CAAU,UAAV,CAAX;AACA,QAAMC,YAAY,GAAG,IAAIC,qBAAJ,CAAiBF,EAAjB,EAAqB;AAACG,IAAAA,UAAU,EAAEV;AAAb,GAArB,CAArB;AAEA,SAAO;AAACW,IAAAA,UAAD;AAAaC,IAAAA,UAAb;AAAyBC,IAAAA,gBAAzB;AAA2CC,IAAAA,KAA3C;AAAkDC,IAAAA,SAAlD;AAA6DC,IAAAA,MAA7D;AAAqEC,IAAAA,WAArE;AAAkFC,IAAAA,aAAlF;AAAiGC,IAAAA,MAAjG;AAAyGC,IAAAA,SAAzG;AAAoHC,IAAAA,QAApH;AAA8HC,IAAAA,OAA9H;AAAuIC,IAAAA;AAAvI,GAAP;;AAEA,iBAAeZ,UAAf,CAA0B;AAACa,IAAAA,aAAD;AAAgBC,IAAAA,SAAhB;AAA2BC,IAAAA,YAA3B;AAAyCC,IAAAA,SAAzC;AAAoDC,IAAAA,IAAI,GAAGC,SAA3D;AAAsEC,IAAAA,MAAM,GAAGD,SAA/E;AAA0FE,IAAAA,IAAI,GAAGF;AAAjG,GAA1B,EAAuI;AACrI,UAAMG,IAAI,GAAG,uBAASC,MAAT,EAAb;AACA,UAAMC,YAAY,GAAG;AACnBV,MAAAA,aADmB;AAEnBC,MAAAA,SAFmB;AAGnBE,MAAAA,SAHmB;AAInBD,MAAAA,YAJmB;AAKnBS,MAAAA,iBAAiB,EAAE;AAACL,QAAAA,MAAD;AAASF,QAAAA,IAAT;AAAeG,QAAAA;AAAf,OALA;AAMnBK,MAAAA,cAAc,EAAEC,4BAAiBC,SAAjB,CAA2BC,kBANxB;AAOnBC,MAAAA,YAAY,EAAER,IAPK;AAQnBS,MAAAA,gBAAgB,EAAET,IARC;AASnBU,MAAAA,UAAU,EAAE,EATO;AAUnBC,MAAAA,WAAW,EAAE;AAVM,KAArB;;AAYA,QAAI;AACF,YAAMC,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B6C,SAA1B,CAAoCX,YAApC,CAArB;;AACA,UAAIU,MAAM,CAACE,YAAX,EAAyB;AACvB7C,QAAAA,MAAM,CAAC8C,IAAP,CAAa,2BAA0BvB,aAAc,qBAAoBxB,UAAW,EAApF;AACA;AACD;;AACD,YAAM,IAAIgD,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD,KAPD,CAOE,OAAOC,KAAP,EAAc;AACd,2BAASA,KAAT;AACA,YAAM,IAAIH,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD;AACF;;AAED,WAAStC,UAAT,CAAoB;AAACY,IAAAA,aAAD;AAAgBC,IAAAA,SAAhB;AAA2BC,IAAAA,YAA3B;AAAyCC,IAAAA,SAAzC;AAAoDyB,IAAAA,WAApD;AAAiEC,IAAAA,gBAAjE;AAAmFC,IAAAA;AAAnF,GAApB,EAAgH;AAC9G,UAAMtB,IAAI,GAAG,uBAASC,MAAT,EAAb;AACA,UAAMC,YAAY,GAAG;AACnBV,MAAAA,aADmB;AAEnBC,MAAAA,SAFmB;AAGnBC,MAAAA,YAHmB;AAInBC,MAAAA,SAJmB;AAKnByB,MAAAA,WALmB;AAMnBC,MAAAA,gBANmB;AAOnBjB,MAAAA,cAAc,EAAEC,4BAAiBC,SAAjB,CAA2BiB,SAPxB;AAQnBf,MAAAA,YAAY,EAAER,IARK;AASnBS,MAAAA,gBAAgB,EAAET,IATC;AAUnBU,MAAAA,UAAU,EAAE,EAVO;AAWnBC,MAAAA,WAAW,EAAE;AAXM,KAArB;;AAaA,QAAI;AACF;AACApC,MAAAA,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B6C,SAA1B,CAAoCX,YAApC;AACAjC,MAAAA,MAAM,CAAC8C,IAAP,CAAa,2BAA0BvB,aAAc,qBAAoBxB,UAAW,GAApF;AACD,KAJD,CAIE,OAAOmD,KAAP,EAAc;AACd,2BAASA,KAAT;AACA,YAAM,IAAIH,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD;;AAED,WAAO,IAAIM,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtC,YAAMC,YAAY,GAAGnD,YAAY,CAACoD,gBAAb,CAA8BpC,aAA9B,CAArB;AAEA8B,MAAAA,MAAM,CACHO,EADH,CACM,OADN,EACeH,MADf,EAEGG,EAFH,CAEM,MAFN,EAEcC,KAAK,IAAIH,YAAY,CAACI,KAAb,CAAmBD,KAAnB,CAFvB,EAGGD,EAHH,CAGM,KAHN,EAGa,MAAMF,YAAY,CAACK,GAAb,CAAiBnC,SAAjB,EAA4BA,SAA5B,EAAuC,MAAM;AAC5D4B,QAAAA,OAAO,CAACjC,aAAD,CAAP;AACD,OAFgB,CAHnB;AAMD,KATM,CAAP;AAUD,GAvEmD,CAyEpD;;;AACA,iBAAeX,gBAAf,CAAgC;AAACW,IAAAA,aAAD;AAAgByC,IAAAA,KAAhB;AAAuBC,IAAAA,YAAY,GAAG,EAAtC;AAA0CC,IAAAA,WAAW,GAAG;AAAxD,GAAhC,EAA6F;AAC3F;AACA;AACA,UAAMC,OAAO,GAAG,MAAMC,YAAY,CAAC7C,aAAD,CAAlC;;AACA,QAAI4C,OAAJ,EAAa;AACX,aAAO/C,QAAQ,CAAC;AAACG,QAAAA,aAAD;AAAgByC,QAAAA,KAAhB;AAAuBC,QAAAA,YAAvB;AAAqCC,QAAAA;AAArC,OAAD,CAAf;AACD;;AACD,WAAO,KAAP;AACD;;AAED,iBAAerD,KAAf,CAAqBwD,MAArB,EAA6B;AAC3B,UAAM1B,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BuE,IAA1B,CAA+BD,MAA/B,EAAuC;AAACE,MAAAA,UAAU,EAAE;AAACC,QAAAA,GAAG,EAAE;AAAN;AAAb,KAAvC,EAClBC,OADkB,EAArB;AAEAzE,IAAAA,MAAM,CAAC0E,KAAP,CAAc,iBAAgB/B,MAAM,CAACgC,MAAP,GAAgB,CAAhB,GAAoB,QAApB,GAA+B,YAAa,EAA1E;AACA,WAAOhC,MAAP;AACD;;AAED,iBAAe7B,SAAf,CAAyBS,aAAzB,EAAwCqD,YAAxC,EAAsD;AACpD,UAAMjC,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC;AAACtD,MAAAA;AAAD,KAAlC,CAArB;;AACA,QAAIqD,YAAJ,EAAkB;AAChB,YAAMT,OAAO,GAAG,MAAMC,YAAY,CAAC7C,aAAD,CAAlC;;AACA,UAAI4C,OAAJ,EAAa;AACX,eAAOrD,SAAS,CAACS,aAAD,CAAhB;AACD;;AACD,aAAOoB,MAAP;AACD;;AAED,WAAOA,MAAP;AACD,GAtGmD,CAwGpD;AACA;AACA;;;AACA,iBAAeyB,YAAf,CAA4B7C,aAA5B,EAA2C;AACzC,UAAMoB,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC;AAACtD,MAAAA;AAAD,KAAlC,CAArB;AACA,UAAMuD,WAAW,GAAG,qBAAOnC,MAAM,CAACH,gBAAd,EAAgCuC,GAAhC,CAAoC,CAApC,EAAuC,GAAvC,CAApB;AACA/E,IAAAA,MAAM,CAACgF,KAAP,CAAc,aAAYF,WAAY,EAAtC;;AACA,QAAIA,WAAW,CAACG,QAAZ,EAAJ,EAA4B;AAC1B,YAAMC,QAAQ,GAAGvC,MAAM,CAACR,cAAxB;AACA,YAAMf,QAAQ,CAAC;AAACG,QAAAA,aAAD;AAAgByC,QAAAA,KAAK,EAAE5B,4BAAiB+C,KAAxC;AAA+CjB,QAAAA,WAAW,EAAElB,oBAAWoC,eAAvE;AAAwFnB,QAAAA,YAAY,EAAG,cAAaiB,QAAS;AAA7H,OAAD,CAAd;AACA,aAAO,KAAP;AACD;;AAED,WAAO,IAAP;AACD;;AAED,iBAAenE,MAAf,CAAsBsD,MAAtB,EAA8B;AAC5BrE,IAAAA,MAAM,CAAC0E,KAAP,CAAc,GAAEW,IAAI,CAACC,SAAL,CAAejB,MAAf,CAAuB,EAAvC;AACArE,IAAAA,MAAM,CAAC8C,IAAP,CAAa,wBAAuB/C,UAAW,SAAQsE,MAAM,CAAC9C,aAAc,EAA5E;AACA,UAAMgE,KAAK,GAAG,4BAASlB,MAAM,CAAC9C,aAAhB,CAAd;AACAvB,IAAAA,MAAM,CAACgF,KAAP,CAAc,wBAAuBK,IAAI,CAACC,SAAL,CAAeC,KAAf,CAAsB,EAA3D;;AAEA,QAAI;AACF;AACA;AACA,YAAMC,SAAS,GAAG,MAAMvE,aAAa,CAACoD,MAAD,CAArC;;AACA,UAAImB,SAAJ,EAAe;AACb,cAAMlF,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B0F,SAA1B,CAAoC;AAAClE,UAAAA,aAAa,EAAEgE;AAAhB,SAApC,CAAN;AACA,eAAO,IAAP;AACD;AACF,KARD,CAQE,OAAOG,GAAP,EAAY;AACZ,UAAIA,GAAG,YAAYC,yBAAnB,EAAqC;AACnC3F,QAAAA,MAAM,CAACkD,KAAP,CAAawC,GAAG,CAACE,OAAjB;;AACA,YAAIF,GAAG,CAACE,OAAJ,CAAYC,OAAZ,CAAoB,uBAApB,MAAiD,CAAC,CAAtD,EAAyD;AACvD7F,UAAAA,MAAM,CAAC0E,KAAP,CAAc,oDAAmDW,IAAI,CAACC,SAAL,CAAeC,KAAf,CAAsB,SAAQxF,UAAW,EAA1G;AACA,gBAAMO,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B0F,SAA1B,CAAoC;AAAClE,YAAAA,aAAa,EAAEgE;AAAhB,WAApC,CAAN;AACA,iBAAO,IAAP;AACD;;AACD,cAAM,IAAIxC,qBAAJ,CAAaC,oBAAWC,qBAAxB,EAA+CyC,GAAG,CAACE,OAAnD,CAAN;AACD;;AACD,YAAM,IAAI7C,qBAAJ,CAAaC,oBAAWC,qBAAxB,EAA+CyC,GAAG,CAACE,OAAnD,CAAN;AACD;AACF;;AAED,iBAAe5E,WAAf,CAA2BO,aAA3B,EAA0C;AACxCvB,IAAAA,MAAM,CAAC8C,IAAP,CAAa,2BAA0BvB,aAAc,OAAMxB,UAAW,EAAtE;AACA,UAAMwF,KAAK,GAAG,4BAAShE,aAAT,CAAd;AACA,UAAMoB,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC;AAACtD,MAAAA,aAAa,EAAEgE;AAAhB,KAAlC,CAArB,CAHwC,CAGwC;;AAEhF,QAAI5C,MAAJ,EAAY;AACV,aAAO;AACLQ,QAAAA,WAAW,EAAER,MAAM,CAACQ,WADf;AAEL2C,QAAAA,UAAU,EAAEvF,YAAY,CAACwF,wBAAb,CAAsCR,KAAtC;AAFP,OAAP;AAID;;AAED,UAAM,IAAIxC,qBAAJ,CAAaC,oBAAWgD,SAAxB,CAAN;AACD;;AAED,iBAAe/E,aAAf,CAA6BoD,MAA7B,EAAqC;AACnCrE,IAAAA,MAAM,CAAC8C,IAAP,CAAa,4BAA2BuB,MAAM,CAAC9C,aAAc,OAAMxB,UAAW,EAA9E;AACA,UAAMwF,KAAK,GAAG,4BAASlB,MAAM,CAAC9C,aAAhB,CAAd;AAEA,UAAMoB,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC;AAACtD,MAAAA,aAAa,EAAEgE;AAAhB,KAAlC,CAArB,CAJmC,CAI6C;;AAChFvF,IAAAA,MAAM,CAAC0E,KAAP,CAAc,+BAA8BW,IAAI,CAACC,SAAL,CAAe3C,MAAf,CAAuB,EAAnE;;AAEA,QAAIA,MAAJ,EAAY;AACV,YAAMpC,YAAY,CAAC0F,MAAb,CAAoBV,KAApB,CAAN;AACA,aAAO,IAAP;AACD;;AAED,WAAO,IAAP;AACD;;AAED,WAASrE,MAAT,CAAgB;AAACQ,IAAAA,SAAD;AAAYS,IAAAA;AAAZ,GAAhB,EAA6C;AAC3C,UAAMoD,KAAK,GAAG;AAACpD,MAAAA,cAAc,EAAE,4BAASA,cAAT;AAAjB,KAAd;;AACA,QAAI;AACF,UAAIT,SAAS,KAAKE,SAAlB,EAA6B;AAC3B5B,QAAAA,MAAM,CAACgF,KAAP,CAAc,eAAcjF,UAAW,QAAOsF,IAAI,CAACC,SAAL,CAAeC,KAAK,CAACpD,cAArB,CAAqC,EAAnF;AACA,eAAO7B,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC,EAAC,GAAGU;AAAJ,SAAlC,CAAP;AACD;;AAED,YAAMW,MAAM,GAAG;AAACxE,QAAAA,SAAS,EAAE,4BAASA,SAAT;AAAZ,OAAf;AACA1B,MAAAA,MAAM,CAACgF,KAAP,CAAc,eAAcjF,UAAW,QAAOwF,KAAK,CAACpD,cAAe,MAAK+D,MAAM,CAACxE,SAAU,EAAzF;AACA,aAAOpB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC,EAAC,GAAGU,KAAJ;AAAW,WAAGW;AAAd,OAAlC,CAAP;AACD,KATD,CASE,OAAOhD,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,WAAS/B,SAAT,CAAmBI,aAAnB,EAAkC;AAChCvB,IAAAA,MAAM,CAAC8C,IAAP,CAAa,2BAA0BvB,aAAc,OAAMxB,UAAW,EAAtE;AACA,UAAMwF,KAAK,GAAG,4BAAShE,aAAT,CAAd;;AACA,QAAI;AACF;AACA,aAAOhB,YAAY,CAACwF,wBAAb,CAAsCR,KAAtC,CAAP;AACD,KAHD,CAGE,OAAOrC,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAe7B,OAAf,CAAuB;AAACE,IAAAA,aAAD;AAAgBkB,IAAAA,UAAhB;AAA4BC,IAAAA;AAA5B,GAAvB,EAAiE;AAC/D1C,IAAAA,MAAM,CAAC8C,IAAP,CAAa,gCAA+BvB,aAAc,OAAMxB,UAAW,EAA3E;AACAC,IAAAA,MAAM,CAAC0E,KAAP,CAAc,QAAOW,IAAI,CAACC,SAAL,CAAe7C,UAAf,CAA2B,kBAAiB4C,IAAI,CAACC,SAAL,CAAe5C,WAAf,CAA4B,EAA7F;AACA,UAAM6C,KAAK,GAAG,4BAAShE,aAAT,CAAd;AACA,UAAMjB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BoG,SAA1B,CAAoC;AACxC5E,MAAAA,aAAa,EAAEgE;AADyB,KAApC,EAEH;AACDa,MAAAA,IAAI,EAAE;AACJ5D,QAAAA,gBAAgB,EAAE,uBAASR,MAAT;AADd,OADL;AAIDqE,MAAAA,KAAK,EAAE;AACL5D,QAAAA,UAAU,EAAE;AAAC6D,UAAAA,KAAK,EAAE7D;AAAR,SADP;AAELC,QAAAA,WAAW,EAAE;AAAC4D,UAAAA,KAAK,EAAE5D;AAAR;AAFR;AAJN,KAFG,CAAN;AAWD;;AAED,iBAAepB,YAAf,CAA4B;AAACC,IAAAA,aAAD;AAAgBgF,IAAAA,QAAhB;AAA0BC,IAAAA,YAAY,GAAG;AAAzC,GAA5B,EAAkF;AAChFxG,IAAAA,MAAM,CAAC8C,IAAP,CAAa,oBAAmB0D,YAAa,IAAGjF,aAAc,OAAMxB,UAAW,EAA/E;AACAC,IAAAA,MAAM,CAAC0E,KAAP,CAAc,aAAYW,IAAI,CAACC,SAAL,CAAeiB,QAAf,CAAyB,GAAnD;AACA,UAAMhB,KAAK,GAAG,4BAAShE,aAAT,CAAd;AACA,UAAMkF,iBAAiB,GAAG,4BAASD,YAAT,CAA1B;AACA,UAAMlG,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BoG,SAA1B,CAAoC;AACxC5E,MAAAA,aAAa,EAAEgE;AADyB,KAApC,EAEH;AACDa,MAAAA,IAAI,EAAE;AACJ5D,QAAAA,gBAAgB,EAAE,uBAASR,MAAT;AADd,OADL;AAIDqE,MAAAA,KAAK,EAAE;AACL,SAACI,iBAAD,GAAqB;AAACH,UAAAA,KAAK,EAAEC;AAAR;AADhB;AAJN,KAFG,CAAN;AAUD;;AAED,WAASnF,QAAT,CAAkB;AAACG,IAAAA,aAAD;AAAgByC,IAAAA,KAAhB;AAAuBC,IAAAA,YAAY,GAAG,EAAtC;AAA0CC,IAAAA,WAAW,GAAG;AAAxD,GAAlB,EAA+E;AAC7E,UAAMwC,WAAW,GAAGzC,YAAY,IAAIC,WAAhB,GAA+B,qBAAoBD,YAAa,qBAAoBC,WAAY,GAAhG,GAAqG,EAAzH;AACAlE,IAAAA,MAAM,CAAC2G,GAAP,CAAW,MAAX,EAAoB,6BAA4BpF,aAAc,GAAEyC,KAAM,GAAE0C,WAAY,OAAM3G,UAAW,EAArG;AACA,UAAMwF,KAAK,GAAG,4BAAShE,aAAT,CAAd;AACA,WAAOjB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B6G,gBAA1B,CAA2C;AAChDrF,MAAAA,aAAa,EAAEgE;AADiC,KAA3C,EAEJ;AACDa,MAAAA,IAAI,EAAE;AACJjE,QAAAA,cAAc,EAAE6B,KADZ;AAEJxB,QAAAA,gBAAgB,EAAE,uBAASR,MAAT,EAFd;AAGJiC,QAAAA,YAHI;AAIJC,QAAAA;AAJI;AADL,KAFI,EASJ;AAACK,MAAAA,UAAU,EAAE;AAACC,QAAAA,GAAG,EAAE;AAAN,OAAb;AAAuBqC,MAAAA,iBAAiB,EAAE;AAA1C,KATI,CAAP;AAUD;AACF","sourcesContent":["/* eslint-disable max-lines */\n/* eslint-disable max-statements */\n/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport {MongoClient, GridFSBucket, MongoDriverError} from 'mongodb';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {Error as ApiError} from '@natlibfi/melinda-commons';\nimport {QUEUE_ITEM_STATE} from './constants';\nimport {logError} from './utils.js';\nimport moment from 'moment';\nimport httpStatus from 'http-status';\nimport sanitize from 'mongo-sanitize';\n\n/* QueueItem:\n{\n  \"correlationId\":\"FOO\",\n  \"cataloger\":\"xxx0000\",\n  \"oCatalogerIn\":\"xxx0000\"\n  \"operation\":\"UPDATE\",\n  \"contentType\":\"application/json\",\n  \"recordLoadParams\": {\n    \"pActiveLibrary\": \"XXX00\",\n    \"pInputFile\": \"filename.seq\",\n    \"pRejectFile\": \"filename.rej\",\n    \"pLogFile\": \"filename.syslog\",\n    \"pOldNew\": \"NEW\"\n  },\n  \"queueItemState\":\"PENDING_QUEUING\",\n  \"creationTime\":\"2020-01-01T00:00:00.000Z\",\n  \"modificationTime\":\"2020-01-01T00:00:01.000Z\",\n  \"handledIds\": []\n}\n*/\n\nexport default async function (MONGO_URI, collection) {\n  const logger = createLogger();\n\n  // Connect to mongo (MONGO)\n  const client = await MongoClient.connect(MONGO_URI, {useNewUrlParser: true, useUnifiedTopology: true});\n  const db = client.db('rest-api');\n  const gridFSBucket = new GridFSBucket(db, {bucketName: collection});\n\n  return {createPrio, createBulk, checkAndSetState, query, queryById, remove, readContent, removeContent, getOne, getStream, setState, pushIds, pushMessages};\n\n  async function createPrio({correlationId, cataloger, oCatalogerIn, operation, noop = undefined, unique = undefined, prio = undefined}) {\n    const time = moment().toDate();\n    const newQueueItem = {\n      correlationId,\n      cataloger,\n      operation,\n      oCatalogerIn,\n      operationSettings: {unique, noop, prio},\n      queueItemState: QUEUE_ITEM_STATE.VALIDATOR.PENDING_VALIDATION,\n      creationTime: time,\n      modificationTime: time,\n      handledIds: [],\n      rejectedIds: []\n    };\n    try {\n      const result = await db.collection(collection).insertOne(newQueueItem);\n      if (result.acknowledged) {\n        logger.info(`New prio queue item for ${correlationId} has been made in ${collection}`);\n        return;\n      }\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    } catch (error) {\n      logError(error);\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    }\n  }\n\n  function createBulk({correlationId, cataloger, oCatalogerIn, operation, contentType, recordLoadParams, stream}) {\n    const time = moment().toDate();\n    const newQueueItem = {\n      correlationId,\n      cataloger,\n      oCatalogerIn,\n      operation,\n      contentType,\n      recordLoadParams,\n      queueItemState: QUEUE_ITEM_STATE.VALIDATOR.UPLOADING,\n      creationTime: time,\n      modificationTime: time,\n      handledIds: [],\n      rejectedIds: []\n    };\n    try {\n      // No await here, promises later\n      db.collection(collection).insertOne(newQueueItem);\n      logger.info(`New bulk queue item for ${correlationId} has been made in ${collection}!`);\n    } catch (error) {\n      logError(error);\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    }\n\n    return new Promise((resolve, reject) => {\n      const outputStream = gridFSBucket.openUploadStream(correlationId);\n\n      stream\n        .on('error', reject)\n        .on('data', chunk => outputStream.write(chunk))\n        .on('end', () => outputStream.end(undefined, undefined, () => {\n          resolve(correlationId);\n        }));\n    });\n  }\n\n  // Check state that the queueItem has not waited too long and set state\n  async function checkAndSetState({correlationId, state, errorMessage = '', errorStatus = ''}) {\n    // checkTimeOut returns true, if queueItem is fresher than 1 minute\n    // otherwise it sets queueItem to state ABORT (408, 'Timeout')\n    const timeOut = await checkTimeOut(correlationId);\n    if (timeOut) {\n      return setState({correlationId, state, errorMessage, errorStatus});\n    }\n    return false;\n  }\n\n  async function query(params) {\n    const result = await db.collection(collection).find(params, {projection: {_id: 0}})\n      .toArray();\n    logger.debug(`Query result: ${result.length > 0 ? 'Found!' : 'Not found!'}`);\n    return result;\n  }\n\n  async function queryById(correlationId, checkModTime) {\n    const result = await db.collection(collection).findOne({correlationId});\n    if (checkModTime) {\n      const timeOut = await checkTimeOut(correlationId);\n      if (timeOut) {\n        return queryById(correlationId);\n      }\n      return result;\n    }\n\n    return result;\n  }\n\n  // Check that if the item has waited too long\n  // If the last modification time for the queueItem is older than 1 minute\n  // set state to ABORT and return false, otherwise return true\n  async function checkTimeOut(correlationId) {\n    const result = await db.collection(collection).findOne({correlationId});\n    const timeoutTime = moment(result.modificationTime).add(1, 'm');\n    logger.silly(`timeOut @ ${timeoutTime}`);\n    if (timeoutTime.isBefore()) {\n      const oldState = result.queueItemState;\n      await setState({correlationId, state: QUEUE_ITEM_STATE.ABORT, errorStatus: httpStatus.REQUEST_TIMEOUT, errorMessage: `Timeout in ${oldState}`});\n      return false;\n    }\n\n    return true;\n  }\n\n  async function remove(params) {\n    logger.debug(`${JSON.stringify(params)}`);\n    logger.info(`Removing from Mongo (${collection}) id: ${params.correlationId}`);\n    const clean = sanitize(params.correlationId);\n    logger.silly(`mongo/remove: clean: ${JSON.stringify(clean)}`);\n\n    try {\n      //const metadataResult = await getFileMetadata({filename: clean});\n      //logger.debug(`mongo/remove: metadataResult: ${JSON.stringify(metadataResult)}`);\n      const noContent = await removeContent(params);\n      if (noContent) {\n        await db.collection(collection).deleteOne({correlationId: clean});\n        return true;\n      }\n    } catch (err) {\n      if (err instanceof MongoDriverError) {\n        logger.error(err.message);\n        if (err.message.indexOf('File not found for id') !== -1) {\n          logger.debug(`mongo/remove: File not found, removing queueItem ${JSON.stringify(clean)} from ${collection}`);\n          await db.collection(collection).deleteOne({correlationId: clean});\n          return true;\n        }\n        throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n      }\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n    }\n  }\n\n  async function readContent(correlationId) {\n    logger.info(`Reading content for id: ${correlationId} in ${collection}`);\n    const clean = sanitize(correlationId);\n    const result = await db.collection(collection).findOne({correlationId: clean}); // njsscan-ignore: node_nosqli_injection\n\n    if (result) {\n      return {\n        contentType: result.contentType,\n        readStream: gridFSBucket.openDownloadStreamByName(clean)\n      };\n    }\n\n    throw new ApiError(httpStatus.NOT_FOUND);\n  }\n\n  async function removeContent(params) {\n    logger.info(`Removing content for id: ${params.correlationId} in ${collection}`);\n    const clean = sanitize(params.correlationId);\n\n    const result = await db.collection(collection).findOne({correlationId: clean}); // njsscan-ignore: node_nosqli_injection\n    logger.debug(`mongo/removeContent: result ${JSON.stringify(result)}`);\n\n    if (result) {\n      await gridFSBucket.delete(clean);\n      return true;\n    }\n\n    return true;\n  }\n\n  function getOne({operation, queueItemState}) {\n    const clean = {queueItemState: sanitize(queueItemState)};\n    try {\n      if (operation === undefined) {\n        logger.silly(`Checking DB ${collection} for ${JSON.stringify(clean.queueItemState)}`);\n        return db.collection(collection).findOne({...clean});\n      }\n\n      const clean2 = {operation: sanitize(operation)};\n      logger.silly(`Checking DB ${collection} for ${clean.queueItemState} + ${clean2.operation}`);\n      return db.collection(collection).findOne({...clean, ...clean2});\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  function getStream(correlationId) {\n    logger.info(`Forming stream from db: ${correlationId} in ${collection}`);\n    const clean = sanitize(correlationId);\n    try {\n      // Return content stream\n      return gridFSBucket.openDownloadStreamByName(clean);\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function pushIds({correlationId, handledIds, rejectedIds}) {\n    logger.info(`Push queue-item ids to list: ${correlationId} to ${collection}`);\n    logger.debug(`ids: ${JSON.stringify(handledIds)}, rejectedIds: ${JSON.stringify(rejectedIds)}`);\n    const clean = sanitize(correlationId);\n    await db.collection(collection).updateOne({\n      correlationId: clean\n    }, {\n      $set: {\n        modificationTime: moment().toDate()\n      },\n      $push: {\n        handledIds: {$each: handledIds},\n        rejectedIds: {$each: rejectedIds}\n      }\n    });\n  }\n\n  async function pushMessages({correlationId, messages, messageField = 'messages'}) {\n    logger.info(`Push messages to ${messageField} ${correlationId} to ${collection}`);\n    logger.debug(`Messages: ${JSON.stringify(messages)}}`);\n    const clean = sanitize(correlationId);\n    const cleanMessageField = sanitize(messageField);\n    await db.collection(collection).updateOne({\n      correlationId: clean\n    }, {\n      $set: {\n        modificationTime: moment().toDate()\n      },\n      $push: {\n        [cleanMessageField]: {$each: messages}\n      }\n    });\n  }\n\n  function setState({correlationId, state, errorMessage = '', errorStatus = ''}) {\n    const errorString = errorMessage || errorStatus ? `, Error message: '${errorMessage}', Error status: '${errorStatus}'` : '';\n    logger.log('info', `Setting queue-item state: ${correlationId}${state}${errorString} to ${collection}`);\n    const clean = sanitize(correlationId);\n    return db.collection(collection).findOneAndUpdate({\n      correlationId: clean\n    }, {\n      $set: {\n        queueItemState: state,\n        modificationTime: moment().toDate(),\n        errorMessage,\n        errorStatus\n      }\n    }, {projection: {_id: 0}, returnNewDocument: true});\n  }\n}\n"],"file":"mongo.js"}
+{"version":3,"sources":["../src/mongo.js"],"names":["MONGO_URI","collection","logger","client","MongoClient","connect","useNewUrlParser","useUnifiedTopology","db","gridFSBucket","GridFSBucket","bucketName","createPrio","createBulk","checkAndSetState","query","queryById","remove","readContent","removeContent","getOne","getStream","setState","pushIds","pushMessages","correlationId","cataloger","oCatalogerIn","operation","noop","undefined","unique","prio","time","toDate","newQueueItem","operationSettings","queueItemState","QUEUE_ITEM_STATE","VALIDATOR","PENDING_VALIDATION","creationTime","modificationTime","handledIds","rejectedIds","result","insertOne","acknowledged","info","ApiError","httpStatus","INTERNAL_SERVER_ERROR","error","contentType","recordLoadParams","stream","UPLOADING","Promise","resolve","reject","outputStream","openUploadStream","on","chunk","write","end","state","errorMessage","errorStatus","timeOut","checkTimeOut","params","find","projection","_id","toArray","debug","length","checkModTime","findOne","timeoutTime","add","silly","isBefore","oldState","ABORT","REQUEST_TIMEOUT","JSON","stringify","clean","noContent","deleteOne","err","MongoDriverError","message","indexOf","readStream","openDownloadStreamByName","NOT_FOUND","delete","clean2","verbose","updateOne","$set","$push","$each","messages","messageField","cleanMessageField","errorString","findOneAndUpdate","returnNewDocument"],"mappings":";;;;;;;AA8BA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AArCA;;AACA;;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAWA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAEe,wBAAgBA,SAAhB,EAA2BC,UAA3B,EAAuC;AACpD,QAAMC,MAAM,GAAG,0CAAf,CADoD,CAGpD;;AACA,QAAMC,MAAM,GAAG,MAAMC,qBAAYC,OAAZ,CAAoBL,SAApB,EAA+B;AAACM,IAAAA,eAAe,EAAE,IAAlB;AAAwBC,IAAAA,kBAAkB,EAAE;AAA5C,GAA/B,CAArB;AACA,QAAMC,EAAE,GAAGL,MAAM,CAACK,EAAP,CAAU,UAAV,CAAX;AACA,QAAMC,YAAY,GAAG,IAAIC,qBAAJ,CAAiBF,EAAjB,EAAqB;AAACG,IAAAA,UAAU,EAAEV;AAAb,GAArB,CAArB;AAEA,SAAO;AAACW,IAAAA,UAAD;AAAaC,IAAAA,UAAb;AAAyBC,IAAAA,gBAAzB;AAA2CC,IAAAA,KAA3C;AAAkDC,IAAAA,SAAlD;AAA6DC,IAAAA,MAA7D;AAAqEC,IAAAA,WAArE;AAAkFC,IAAAA,aAAlF;AAAiGC,IAAAA,MAAjG;AAAyGC,IAAAA,SAAzG;AAAoHC,IAAAA,QAApH;AAA8HC,IAAAA,OAA9H;AAAuIC,IAAAA;AAAvI,GAAP;;AAEA,iBAAeZ,UAAf,CAA0B;AAACa,IAAAA,aAAD;AAAgBC,IAAAA,SAAhB;AAA2BC,IAAAA,YAA3B;AAAyCC,IAAAA,SAAzC;AAAoDC,IAAAA,IAAI,GAAGC,SAA3D;AAAsEC,IAAAA,MAAM,GAAGD,SAA/E;AAA0FE,IAAAA,IAAI,GAAGF;AAAjG,GAA1B,EAAuI;AACrI,UAAMG,IAAI,GAAG,uBAASC,MAAT,EAAb;AACA,UAAMC,YAAY,GAAG;AACnBV,MAAAA,aADmB;AAEnBC,MAAAA,SAFmB;AAGnBE,MAAAA,SAHmB;AAInBD,MAAAA,YAJmB;AAKnBS,MAAAA,iBAAiB,EAAE;AAACL,QAAAA,MAAD;AAASF,QAAAA,IAAT;AAAeG,QAAAA;AAAf,OALA;AAMnBK,MAAAA,cAAc,EAAEC,4BAAiBC,SAAjB,CAA2BC,kBANxB;AAOnBC,MAAAA,YAAY,EAAER,IAPK;AAQnBS,MAAAA,gBAAgB,EAAET,IARC;AASnBU,MAAAA,UAAU,EAAE,EATO;AAUnBC,MAAAA,WAAW,EAAE;AAVM,KAArB;;AAYA,QAAI;AACF,YAAMC,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B6C,SAA1B,CAAoCX,YAApC,CAArB;;AACA,UAAIU,MAAM,CAACE,YAAX,EAAyB;AACvB7C,QAAAA,MAAM,CAAC8C,IAAP,CAAa,2BAA0BpB,SAAU,IAAGH,aAAc,qBAAoBxB,UAAW,EAAjG;AACA;AACD;;AACD,YAAM,IAAIgD,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD,KAPD,CAOE,OAAOC,KAAP,EAAc;AACd,2BAASA,KAAT;AACA,YAAM,IAAIH,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD;AACF;;AAED,WAAStC,UAAT,CAAoB;AAACY,IAAAA,aAAD;AAAgBC,IAAAA,SAAhB;AAA2BC,IAAAA,YAA3B;AAAyCC,IAAAA,SAAzC;AAAoDyB,IAAAA,WAApD;AAAiEC,IAAAA,gBAAjE;AAAmFC,IAAAA;AAAnF,GAApB,EAAgH;AAC9G,UAAMtB,IAAI,GAAG,uBAASC,MAAT,EAAb;AACA,UAAMC,YAAY,GAAG;AACnBV,MAAAA,aADmB;AAEnBC,MAAAA,SAFmB;AAGnBC,MAAAA,YAHmB;AAInBC,MAAAA,SAJmB;AAKnByB,MAAAA,WALmB;AAMnBC,MAAAA,gBANmB;AAOnBjB,MAAAA,cAAc,EAAEC,4BAAiBC,SAAjB,CAA2BiB,SAPxB;AAQnBf,MAAAA,YAAY,EAAER,IARK;AASnBS,MAAAA,gBAAgB,EAAET,IATC;AAUnBU,MAAAA,UAAU,EAAE,EAVO;AAWnBC,MAAAA,WAAW,EAAE;AAXM,KAArB;;AAaA,QAAI;AACF;AACApC,MAAAA,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B6C,SAA1B,CAAoCX,YAApC;AACAjC,MAAAA,MAAM,CAAC8C,IAAP,CAAa,2BAA0BpB,SAAU,IAAGH,aAAc,qBAAoBxB,UAAW,GAAjG;AACD,KAJD,CAIE,OAAOmD,KAAP,EAAc;AACd,2BAASA,KAAT;AACA,YAAM,IAAIH,qBAAJ,CAAaC,oBAAWC,qBAAxB,CAAN;AACD;;AAED,WAAO,IAAIM,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtC,YAAMC,YAAY,GAAGnD,YAAY,CAACoD,gBAAb,CAA8BpC,aAA9B,CAArB;AAEA8B,MAAAA,MAAM,CACHO,EADH,CACM,OADN,EACeH,MADf,EAEGG,EAFH,CAEM,MAFN,EAEcC,KAAK,IAAIH,YAAY,CAACI,KAAb,CAAmBD,KAAnB,CAFvB,EAGGD,EAHH,CAGM,KAHN,EAGa,MAAMF,YAAY,CAACK,GAAb,CAAiBnC,SAAjB,EAA4BA,SAA5B,EAAuC,MAAM;AAC5D4B,QAAAA,OAAO,CAACjC,aAAD,CAAP;AACD,OAFgB,CAHnB;AAMD,KATM,CAAP;AAUD,GAvEmD,CAyEpD;;;AACA,iBAAeX,gBAAf,CAAgC;AAACW,IAAAA,aAAD;AAAgByC,IAAAA,KAAhB;AAAuBC,IAAAA,YAAY,GAAG,EAAtC;AAA0CC,IAAAA,WAAW,GAAG;AAAxD,GAAhC,EAA6F;AAC3F;AACA;AACA,UAAMC,OAAO,GAAG,MAAMC,YAAY,CAAC7C,aAAD,CAAlC;;AACA,QAAI4C,OAAJ,EAAa;AACX,aAAO/C,QAAQ,CAAC;AAACG,QAAAA,aAAD;AAAgByC,QAAAA,KAAhB;AAAuBC,QAAAA,YAAvB;AAAqCC,QAAAA;AAArC,OAAD,CAAf;AACD;;AACD,WAAO,KAAP;AACD;;AAED,iBAAerD,KAAf,CAAqBwD,MAArB,EAA6B;AAC3B,UAAM1B,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BuE,IAA1B,CAA+BD,MAA/B,EAAuC;AAACE,MAAAA,UAAU,EAAE;AAACC,QAAAA,GAAG,EAAE;AAAN;AAAb,KAAvC,EAClBC,OADkB,EAArB;AAEAzE,IAAAA,MAAM,CAAC0E,KAAP,CAAc,iBAAgB/B,MAAM,CAACgC,MAAP,GAAgB,CAAhB,GAAoB,QAApB,GAA+B,YAAa,EAA1E;AACA,WAAOhC,MAAP;AACD;;AAED,iBAAe7B,SAAf,CAAyBS,aAAzB,EAAwCqD,YAAxC,EAAsD;AACpD,UAAMjC,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC;AAACtD,MAAAA;AAAD,KAAlC,CAArB;;AACA,QAAIqD,YAAJ,EAAkB;AAChB,YAAMT,OAAO,GAAG,MAAMC,YAAY,CAAC7C,aAAD,CAAlC;;AACA,UAAI4C,OAAJ,EAAa;AACX,eAAOrD,SAAS,CAACS,aAAD,CAAhB;AACD;;AACD,aAAOoB,MAAP;AACD;;AAED,WAAOA,MAAP;AACD,GAtGmD,CAwGpD;AACA;AACA;;;AACA,iBAAeyB,YAAf,CAA4B7C,aAA5B,EAA2C;AACzC,UAAMoB,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC;AAACtD,MAAAA;AAAD,KAAlC,CAArB;AACA,UAAMuD,WAAW,GAAG,qBAAOnC,MAAM,CAACH,gBAAd,EAAgCuC,GAAhC,CAAoC,CAApC,EAAuC,GAAvC,CAApB;AACA/E,IAAAA,MAAM,CAACgF,KAAP,CAAc,aAAYF,WAAY,EAAtC;;AACA,QAAIA,WAAW,CAACG,QAAZ,EAAJ,EAA4B;AAC1B,YAAMC,QAAQ,GAAGvC,MAAM,CAACR,cAAxB;AACA,YAAMf,QAAQ,CAAC;AAACG,QAAAA,aAAD;AAAgByC,QAAAA,KAAK,EAAE5B,4BAAiB+C,KAAxC;AAA+CjB,QAAAA,WAAW,EAAElB,oBAAWoC,eAAvE;AAAwFnB,QAAAA,YAAY,EAAG,cAAaiB,QAAS;AAA7H,OAAD,CAAd;AACA,aAAO,KAAP;AACD;;AAED,WAAO,IAAP;AACD;;AAED,iBAAenE,MAAf,CAAsBsD,MAAtB,EAA8B;AAC5BrE,IAAAA,MAAM,CAACgF,KAAP,CAAc,GAAEK,IAAI,CAACC,SAAL,CAAejB,MAAf,CAAuB,EAAvC;AACArE,IAAAA,MAAM,CAAC8C,IAAP,CAAa,wBAAuB/C,UAAW,SAAQsE,MAAM,CAAC9C,aAAc,EAA5E;AACA,UAAMgE,KAAK,GAAG,4BAASlB,MAAM,CAAC9C,aAAhB,CAAd;AACAvB,IAAAA,MAAM,CAACgF,KAAP,CAAc,wBAAuBK,IAAI,CAACC,SAAL,CAAeC,KAAf,CAAsB,EAA3D;;AAEA,QAAI;AACF;AACA;AACA,YAAMC,SAAS,GAAG,MAAMvE,aAAa,CAACoD,MAAD,CAArC;;AACA,UAAImB,SAAJ,EAAe;AACb,cAAMlF,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B0F,SAA1B,CAAoC;AAAClE,UAAAA,aAAa,EAAEgE;AAAhB,SAApC,CAAN;AACA,eAAO,IAAP;AACD;AACF,KARD,CAQE,OAAOG,GAAP,EAAY;AACZ,UAAIA,GAAG,YAAYC,yBAAnB,EAAqC;AACnC,YAAID,GAAG,CAACE,OAAJ,CAAYC,OAAZ,CAAoB,uBAApB,MAAiD,CAAC,CAAtD,EAAyD;AACvD7F,UAAAA,MAAM,CAACgF,KAAP,CAAc,oDAAmDK,IAAI,CAACC,SAAL,CAAeC,KAAf,CAAsB,SAAQxF,UAAW,EAA1G;AACA,gBAAMO,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B0F,SAA1B,CAAoC;AAAClE,YAAAA,aAAa,EAAEgE;AAAhB,WAApC,CAAN;AACA,iBAAO,IAAP;AACD;;AACDvF,QAAAA,MAAM,CAACkD,KAAP,CAAawC,GAAG,CAACE,OAAjB;AACA,cAAM,IAAI7C,qBAAJ,CAAaC,oBAAWC,qBAAxB,EAA+CyC,GAAG,CAACE,OAAnD,CAAN;AACD;;AACD,YAAM,IAAI7C,qBAAJ,CAAaC,oBAAWC,qBAAxB,EAA+CyC,GAAG,CAACE,OAAnD,CAAN;AACD;AACF;;AAED,iBAAe5E,WAAf,CAA2BO,aAA3B,EAA0C;AACxCvB,IAAAA,MAAM,CAAC8C,IAAP,CAAa,sCAAqCvB,aAAc,OAAMxB,UAAW,EAAjF;AACA,UAAMwF,KAAK,GAAG,4BAAShE,aAAT,CAAd;AACA,UAAMoB,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC;AAACtD,MAAAA,aAAa,EAAEgE;AAAhB,KAAlC,CAArB,CAHwC,CAGwC;;AAEhF,QAAI5C,MAAJ,EAAY;AACV,aAAO;AACLQ,QAAAA,WAAW,EAAER,MAAM,CAACQ,WADf;AAEL2C,QAAAA,UAAU,EAAEvF,YAAY,CAACwF,wBAAb,CAAsCR,KAAtC;AAFP,OAAP;AAID;;AAED,UAAM,IAAIxC,qBAAJ,CAAaC,oBAAWgD,SAAxB,CAAN;AACD;;AAED,iBAAe/E,aAAf,CAA6BoD,MAA7B,EAAqC;AACnCrE,IAAAA,MAAM,CAAC8C,IAAP,CAAa,uCAAsCuB,MAAM,CAAC9C,aAAc,OAAMxB,UAAW,EAAzF;AACA,UAAMwF,KAAK,GAAG,4BAASlB,MAAM,CAAC9C,aAAhB,CAAd;AAEA,UAAMoB,MAAM,GAAG,MAAMrC,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC;AAACtD,MAAAA,aAAa,EAAEgE;AAAhB,KAAlC,CAArB,CAJmC,CAI6C;;AAChFvF,IAAAA,MAAM,CAACgF,KAAP,CAAc,+BAA8BK,IAAI,CAACC,SAAL,CAAe3C,MAAf,CAAuB,EAAnE;;AAEA,QAAIA,MAAJ,EAAY;AACV,YAAMpC,YAAY,CAAC0F,MAAb,CAAoBV,KAApB,CAAN;AACA,aAAO,IAAP;AACD;;AAED,WAAO,IAAP;AACD;;AAED,WAASrE,MAAT,CAAgB;AAACQ,IAAAA,SAAD;AAAYS,IAAAA;AAAZ,GAAhB,EAA6C;AAC3C,UAAMoD,KAAK,GAAG;AAACpD,MAAAA,cAAc,EAAE,4BAASA,cAAT;AAAjB,KAAd;;AACA,QAAI;AACF,UAAIT,SAAS,KAAKE,SAAlB,EAA6B;AAC3B5B,QAAAA,MAAM,CAACgF,KAAP,CAAc,eAAcjF,UAAW,QAAOsF,IAAI,CAACC,SAAL,CAAeC,KAAK,CAACpD,cAArB,CAAqC,EAAnF;AACA,eAAO7B,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC,EAAC,GAAGU;AAAJ,SAAlC,CAAP;AACD;;AAED,YAAMW,MAAM,GAAG;AAACxE,QAAAA,SAAS,EAAE,4BAASA,SAAT;AAAZ,OAAf;AACA1B,MAAAA,MAAM,CAACgF,KAAP,CAAc,eAAcjF,UAAW,QAAOwF,KAAK,CAACpD,cAAe,MAAK+D,MAAM,CAACxE,SAAU,EAAzF;AACA,aAAOpB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B8E,OAA1B,CAAkC,EAAC,GAAGU,KAAJ;AAAW,WAAGW;AAAd,OAAlC,CAAP;AACD,KATD,CASE,OAAOhD,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,WAAS/B,SAAT,CAAmBI,aAAnB,EAAkC;AAChCvB,IAAAA,MAAM,CAAC8C,IAAP,CAAa,iCAAgCvB,aAAc,OAAMxB,UAAW,EAA5E;AACA,UAAMwF,KAAK,GAAG,4BAAShE,aAAT,CAAd;;AACA,QAAI;AACF;AACA,aAAOhB,YAAY,CAACwF,wBAAb,CAAsCR,KAAtC,CAAP;AACD,KAHD,CAGE,OAAOrC,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAe7B,OAAf,CAAuB;AAACE,IAAAA,aAAD;AAAgBkB,IAAAA,UAAhB;AAA4BC,IAAAA;AAA5B,GAAvB,EAAiE;AAC/D1C,IAAAA,MAAM,CAACmG,OAAP,CAAgB,aAAY1D,UAAU,CAACkC,MAAO,sBAAqBjC,WAAW,CAACiC,MAAO,KAAIpD,aAAc,OAAMxB,UAAW,EAAzH;AACAC,IAAAA,MAAM,CAAC0E,KAAP,CAAc,QAAOjC,UAAU,CAACkC,MAAO,MAAKU,IAAI,CAACC,SAAL,CAAe7C,UAAf,CAA2B,iBAAgBC,WAAW,CAACiC,MAAO,KAAIU,IAAI,CAACC,SAAL,CAAe5C,WAAf,CAA4B,EAA1I;AACA,UAAM6C,KAAK,GAAG,4BAAShE,aAAT,CAAd;AACA,UAAMjB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BqG,SAA1B,CAAoC;AACxC7E,MAAAA,aAAa,EAAEgE;AADyB,KAApC,EAEH;AACDc,MAAAA,IAAI,EAAE;AACJ7D,QAAAA,gBAAgB,EAAE,uBAASR,MAAT;AADd,OADL;AAIDsE,MAAAA,KAAK,EAAE;AACL7D,QAAAA,UAAU,EAAE;AAAC8D,UAAAA,KAAK,EAAE9D;AAAR,SADP;AAELC,QAAAA,WAAW,EAAE;AAAC6D,UAAAA,KAAK,EAAE7D;AAAR;AAFR;AAJN,KAFG,CAAN;AAWD;;AAED,iBAAepB,YAAf,CAA4B;AAACC,IAAAA,aAAD;AAAgBiF,IAAAA,QAAhB;AAA0BC,IAAAA,YAAY,GAAG;AAAzC,GAA5B,EAAkF;AAChFzG,IAAAA,MAAM,CAACmG,OAAP,CAAgB,kBAAiBK,QAAQ,CAAC7B,MAAO,QAAO8B,YAAa,IAAGlF,aAAc,OAAMxB,UAAW,EAAvG;AACAC,IAAAA,MAAM,CAACgF,KAAP,CAAc,aAAYwB,QAAQ,CAAC7B,MAAO,MAAKU,IAAI,CAACC,SAAL,CAAekB,QAAf,CAAyB,GAAxE;AACA,UAAMjB,KAAK,GAAG,4BAAShE,aAAT,CAAd;AACA,UAAMmF,iBAAiB,GAAG,4BAASD,YAAT,CAA1B;AACA,UAAMnG,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0BqG,SAA1B,CAAoC;AACxC7E,MAAAA,aAAa,EAAEgE;AADyB,KAApC,EAEH;AACDc,MAAAA,IAAI,EAAE;AACJ7D,QAAAA,gBAAgB,EAAE,uBAASR,MAAT;AADd,OADL;AAIDsE,MAAAA,KAAK,EAAE;AACL,SAACI,iBAAD,GAAqB;AAACH,UAAAA,KAAK,EAAEC;AAAR;AADhB;AAJN,KAFG,CAAN;AAUD;;AAED,WAASpF,QAAT,CAAkB;AAACG,IAAAA,aAAD;AAAgByC,IAAAA,KAAhB;AAAuBC,IAAAA,YAAY,GAAG,EAAtC;AAA0CC,IAAAA,WAAW,GAAG;AAAxD,GAAlB,EAA+E;AAC7E,UAAMyC,WAAW,GAAG1C,YAAY,IAAIC,WAAhB,GAA+B,qBAAoBD,YAAa,qBAAoBC,WAAY,GAAhG,GAAqG,EAAzH;AACAlE,IAAAA,MAAM,CAAC8C,IAAP,CAAa,4BAA2BkB,KAAM,QAAOzC,aAAc,GAAEoF,WAAY,OAAM5G,UAAW,EAAlG;AACA,UAAMwF,KAAK,GAAG,4BAAShE,aAAT,CAAd;AACA,WAAOjB,EAAE,CAACP,UAAH,CAAcA,UAAd,EAA0B6G,gBAA1B,CAA2C;AAChDrF,MAAAA,aAAa,EAAEgE;AADiC,KAA3C,EAEJ;AACDc,MAAAA,IAAI,EAAE;AACJlE,QAAAA,cAAc,EAAE6B,KADZ;AAEJxB,QAAAA,gBAAgB,EAAE,uBAASR,MAAT,EAFd;AAGJiC,QAAAA,YAHI;AAIJC,QAAAA;AAJI;AADL,KAFI,EASJ;AAACK,MAAAA,UAAU,EAAE;AAACC,QAAAA,GAAG,EAAE;AAAN,OAAb;AAAuBqC,MAAAA,iBAAiB,EAAE;AAA1C,KATI,CAAP;AAUD;AACF","sourcesContent":["/* eslint-disable max-lines */\n/* eslint-disable max-statements */\n/**\n*\n* @licstart  The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program.  If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend  The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport {MongoClient, GridFSBucket, MongoDriverError} from 'mongodb';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {Error as ApiError} from '@natlibfi/melinda-commons';\nimport {QUEUE_ITEM_STATE} from './constants';\nimport {logError} from './utils.js';\nimport moment from 'moment';\nimport httpStatus from 'http-status';\nimport sanitize from 'mongo-sanitize';\n\n/* QueueItem:\n{\n  \"correlationId\":\"FOO\",\n  \"cataloger\":\"xxx0000\",\n  \"oCatalogerIn\":\"xxx0000\"\n  \"operation\":\"UPDATE\",\n  \"contentType\":\"application/json\",\n  \"recordLoadParams\": {\n    \"pActiveLibrary\": \"XXX00\",\n    \"pInputFile\": \"filename.seq\",\n    \"pRejectFile\": \"filename.rej\",\n    \"pLogFile\": \"filename.syslog\",\n    \"pOldNew\": \"NEW\"\n  },\n  \"queueItemState\":\"PENDING_QUEUING\",\n  \"creationTime\":\"2020-01-01T00:00:00.000Z\",\n  \"modificationTime\":\"2020-01-01T00:00:01.000Z\",\n  \"handledIds\": []\n}\n*/\n\nexport default async function (MONGO_URI, collection) {\n  const logger = createLogger();\n\n  // Connect to mongo (MONGO)\n  const client = await MongoClient.connect(MONGO_URI, {useNewUrlParser: true, useUnifiedTopology: true});\n  const db = client.db('rest-api');\n  const gridFSBucket = new GridFSBucket(db, {bucketName: collection});\n\n  return {createPrio, createBulk, checkAndSetState, query, queryById, remove, readContent, removeContent, getOne, getStream, setState, pushIds, pushMessages};\n\n  async function createPrio({correlationId, cataloger, oCatalogerIn, operation, noop = undefined, unique = undefined, prio = undefined}) {\n    const time = moment().toDate();\n    const newQueueItem = {\n      correlationId,\n      cataloger,\n      operation,\n      oCatalogerIn,\n      operationSettings: {unique, noop, prio},\n      queueItemState: QUEUE_ITEM_STATE.VALIDATOR.PENDING_VALIDATION,\n      creationTime: time,\n      modificationTime: time,\n      handledIds: [],\n      rejectedIds: []\n    };\n    try {\n      const result = await db.collection(collection).insertOne(newQueueItem);\n      if (result.acknowledged) {\n        logger.info(`New PRIO queue item for ${operation} ${correlationId} has been made in ${collection}`);\n        return;\n      }\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    } catch (error) {\n      logError(error);\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    }\n  }\n\n  function createBulk({correlationId, cataloger, oCatalogerIn, operation, contentType, recordLoadParams, stream}) {\n    const time = moment().toDate();\n    const newQueueItem = {\n      correlationId,\n      cataloger,\n      oCatalogerIn,\n      operation,\n      contentType,\n      recordLoadParams,\n      queueItemState: QUEUE_ITEM_STATE.VALIDATOR.UPLOADING,\n      creationTime: time,\n      modificationTime: time,\n      handledIds: [],\n      rejectedIds: []\n    };\n    try {\n      // No await here, promises later\n      db.collection(collection).insertOne(newQueueItem);\n      logger.info(`New BULK queue item for ${operation} ${correlationId} has been made in ${collection}!`);\n    } catch (error) {\n      logError(error);\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    }\n\n    return new Promise((resolve, reject) => {\n      const outputStream = gridFSBucket.openUploadStream(correlationId);\n\n      stream\n        .on('error', reject)\n        .on('data', chunk => outputStream.write(chunk))\n        .on('end', () => outputStream.end(undefined, undefined, () => {\n          resolve(correlationId);\n        }));\n    });\n  }\n\n  // Check state that the queueItem has not waited too long and set state\n  async function checkAndSetState({correlationId, state, errorMessage = '', errorStatus = ''}) {\n    // checkTimeOut returns true, if queueItem is fresher than 1 minute\n    // otherwise it sets queueItem to state ABORT (408, 'Timeout')\n    const timeOut = await checkTimeOut(correlationId);\n    if (timeOut) {\n      return setState({correlationId, state, errorMessage, errorStatus});\n    }\n    return false;\n  }\n\n  async function query(params) {\n    const result = await db.collection(collection).find(params, {projection: {_id: 0}})\n      .toArray();\n    logger.debug(`Query result: ${result.length > 0 ? 'Found!' : 'Not found!'}`);\n    return result;\n  }\n\n  async function queryById(correlationId, checkModTime) {\n    const result = await db.collection(collection).findOne({correlationId});\n    if (checkModTime) {\n      const timeOut = await checkTimeOut(correlationId);\n      if (timeOut) {\n        return queryById(correlationId);\n      }\n      return result;\n    }\n\n    return result;\n  }\n\n  // Check that if the item has waited too long\n  // If the last modification time for the queueItem is older than 1 minute\n  // set state to ABORT and return false, otherwise return true\n  async function checkTimeOut(correlationId) {\n    const result = await db.collection(collection).findOne({correlationId});\n    const timeoutTime = moment(result.modificationTime).add(1, 'm');\n    logger.silly(`timeOut @ ${timeoutTime}`);\n    if (timeoutTime.isBefore()) {\n      const oldState = result.queueItemState;\n      await setState({correlationId, state: QUEUE_ITEM_STATE.ABORT, errorStatus: httpStatus.REQUEST_TIMEOUT, errorMessage: `Timeout in ${oldState}`});\n      return false;\n    }\n\n    return true;\n  }\n\n  async function remove(params) {\n    logger.silly(`${JSON.stringify(params)}`);\n    logger.info(`Removing from Mongo (${collection}) id: ${params.correlationId}`);\n    const clean = sanitize(params.correlationId);\n    logger.silly(`mongo/remove: clean: ${JSON.stringify(clean)}`);\n\n    try {\n      //const metadataResult = await getFileMetadata({filename: clean});\n      //logger.debug(`mongo/remove: metadataResult: ${JSON.stringify(metadataResult)}`);\n      const noContent = await removeContent(params);\n      if (noContent) {\n        await db.collection(collection).deleteOne({correlationId: clean});\n        return true;\n      }\n    } catch (err) {\n      if (err instanceof MongoDriverError) {\n        if (err.message.indexOf('File not found for id') !== -1) {\n          logger.silly(`mongo/remove: File not found, removing queueItem ${JSON.stringify(clean)} from ${collection}`);\n          await db.collection(collection).deleteOne({correlationId: clean});\n          return true;\n        }\n        logger.error(err.message);\n        throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n      }\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n    }\n  }\n\n  async function readContent(correlationId) {\n    logger.info(`Reading content from mongo for id: ${correlationId} in ${collection}`);\n    const clean = sanitize(correlationId);\n    const result = await db.collection(collection).findOne({correlationId: clean}); // njsscan-ignore: node_nosqli_injection\n\n    if (result) {\n      return {\n        contentType: result.contentType,\n        readStream: gridFSBucket.openDownloadStreamByName(clean)\n      };\n    }\n\n    throw new ApiError(httpStatus.NOT_FOUND);\n  }\n\n  async function removeContent(params) {\n    logger.info(`Removing content from mongo for id: ${params.correlationId} in ${collection}`);\n    const clean = sanitize(params.correlationId);\n\n    const result = await db.collection(collection).findOne({correlationId: clean}); // njsscan-ignore: node_nosqli_injection\n    logger.silly(`mongo/removeContent: result ${JSON.stringify(result)}`);\n\n    if (result) {\n      await gridFSBucket.delete(clean);\n      return true;\n    }\n\n    return true;\n  }\n\n  function getOne({operation, queueItemState}) {\n    const clean = {queueItemState: sanitize(queueItemState)};\n    try {\n      if (operation === undefined) {\n        logger.silly(`Checking DB ${collection} for ${JSON.stringify(clean.queueItemState)}`);\n        return db.collection(collection).findOne({...clean});\n      }\n\n      const clean2 = {operation: sanitize(operation)};\n      logger.silly(`Checking DB ${collection} for ${clean.queueItemState} + ${clean2.operation}`);\n      return db.collection(collection).findOne({...clean, ...clean2});\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  function getStream(correlationId) {\n    logger.info(`Forming stream from mongo for ${correlationId} in ${collection}`);\n    const clean = sanitize(correlationId);\n    try {\n      // Return content stream\n      return gridFSBucket.openDownloadStreamByName(clean);\n    } catch (error) {\n      logError(error);\n    }\n  }\n\n  async function pushIds({correlationId, handledIds, rejectedIds}) {\n    logger.verbose(`Push ids (${handledIds.length}) and rejectedIds (${rejectedIds.length}) ${correlationId} to ${collection}`);\n    logger.debug(`ids (${handledIds.length}): ${JSON.stringify(handledIds)}, rejectedIds ${rejectedIds.length}: ${JSON.stringify(rejectedIds)}`);\n    const clean = sanitize(correlationId);\n    await db.collection(collection).updateOne({\n      correlationId: clean\n    }, {\n      $set: {\n        modificationTime: moment().toDate()\n      },\n      $push: {\n        handledIds: {$each: handledIds},\n        rejectedIds: {$each: rejectedIds}\n      }\n    });\n  }\n\n  async function pushMessages({correlationId, messages, messageField = 'messages'}) {\n    logger.verbose(`Push messages (${messages.length}) to ${messageField} ${correlationId} to ${collection}`);\n    logger.silly(`Messages (${messages.length}): ${JSON.stringify(messages)}}`);\n    const clean = sanitize(correlationId);\n    const cleanMessageField = sanitize(messageField);\n    await db.collection(collection).updateOne({\n      correlationId: clean\n    }, {\n      $set: {\n        modificationTime: moment().toDate()\n      },\n      $push: {\n        [cleanMessageField]: {$each: messages}\n      }\n    });\n  }\n\n  function setState({correlationId, state, errorMessage = '', errorStatus = ''}) {\n    const errorString = errorMessage || errorStatus ? `, Error message: '${errorMessage}', Error status: '${errorStatus}'` : '';\n    logger.info(`Setting queue-item state ${state} for ${correlationId}${errorString} to ${collection}`);\n    const clean = sanitize(correlationId);\n    return db.collection(collection).findOneAndUpdate({\n      correlationId: clean\n    }, {\n      $set: {\n        queueItemState: state,\n        modificationTime: moment().toDate(),\n        errorMessage,\n        errorStatus\n      }\n    }, {projection: {_id: 0}, returnNewDocument: true});\n  }\n}\n"],"file":"mongo.js"}

package/package.json

@@ -14,10 +14,10 @@
 		"url": "git@github.com:natlibfi/melinda-rest-api-commons.git"
 	},
 	"license": "AGPL-3.0+",
-	"version": "2.0.0-alpha.13",
+	"version": "2.0.0-alpha.18",
 	"main": "./dist/index.js",
 	"engines": {
-		"node": ">=12"
+		"node": ">=14"
 	},
 	"publishConfig": {
 		"access": "public"