@natlibfi/melinda-rest-api-commons 2.0.1-alpha.6 → 2.0.1-alpha.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/amqp.js +10 -1
- package/dist/amqp.js.map +1 -1
- package/package.json +1 -1
package/dist/amqp.js
CHANGED
|
@@ -234,8 +234,17 @@ async function _default(AMQP_URL) {
|
|
|
234
234
|
}
|
|
235
235
|
|
|
236
236
|
async function removeQueue(queue) {
|
|
237
|
+
// deleteQueue borks the channel if the queue does not exist
|
|
238
|
+
// -> use throwaway tempChannel to avoid killing actual channel in use
|
|
239
|
+
// this might be doable also with assertQueue before deleteQueue
|
|
240
|
+
const tempChannel = await connection.createChannel();
|
|
237
241
|
logger.verbose(`Removing queue ${queue}.`);
|
|
238
|
-
await
|
|
242
|
+
await tempChannel.deleteQueue(queue);
|
|
243
|
+
|
|
244
|
+
if (tempChannel) {
|
|
245
|
+
// eslint-disable-line functional/no-conditional-statement
|
|
246
|
+
await tempChannel.close();
|
|
247
|
+
}
|
|
239
248
|
} // ----------------
|
|
240
249
|
// Helper functions
|
|
241
250
|
// ----------------
|
package/dist/amqp.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/amqp.js"],"names":["AMQP_URL","connection","amqplib","connect","channel","createChannel","logger","checkQueue","consumeChunk","consumeOne","ackMessages","nackMessages","sendToQueue","removeQueue","messagesToRecords","queue","style","toRecord","purge","silly","channelInfo","assertQueue","durable","purgeQueue","verbose","messageCount","ApiError","httpStatus","UNPROCESSABLE_ENTITY","error","messages","getData","headers","getHeaderInfo","debug","length","records","message","get","colors","maxArrayLength","depth","forEach","properties","correlationId","ack","nack","data","JSON","stringify","Buffer","from","persistent","deleteQueue","map","content","parse","toString","MarcRecord","messagesToGet","CHUNK_SIZE","pump","count","results","identifiers","identifier","deliveryTag","fields","includes","concat"],"mappings":";;;;;;;AA6BA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AAnCA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAWe,wBAAgBA,QAAhB,EAA0B;AACvC,QAAMC,UAAU,GAAG,MAAMC,iBAAQC,OAAR,CAAgBH,QAAhB,CAAzB;AACA,QAAMI,OAAO,GAAG,MAAMH,UAAU,CAACI,aAAX,EAAtB;AACA,QAAMC,MAAM,GAAG,0CAAf;AAEA,SAAO;AAACC,IAAAA,UAAD;AAAaC,IAAAA,YAAb;AAA2BC,IAAAA,UAA3B;AAAuCC,IAAAA,WAAvC;AAAoDC,IAAAA,YAApD;AAAkEC,IAAAA,WAAlE;AAA+EC,IAAAA,WAA/E;AAA4FC,IAAAA;AAA5F,GAAP;;AAEA,iBAAeP,UAAf,CAA0B;AAACQ,IAAAA,KAAD;AAAQC,IAAAA,KAAK,GAAG,OAAhB;AAAyBC,IAAAA,QAAQ,GAAG,IAApC;AAA0CC,IAAAA,KAAK,GAAG;AAAlD,GAA1B,EAAoF;AAClFZ,IAAAA,MAAM,CAACa,KAAP,CAAc,eAAcJ,KAAM,YAAWC,KAAM,eAAcC,QAAS,YAAWC,KAAM,IAA3F;;AACA,QAAI;AACF,YAAME,WAAW,GAAG,MAAMhB,OAAO,CAACiB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAA1B;;AACA,UAAIJ,KAAJ,EAAW;AACT,cAAMK,UAAU,CAACL,KAAD,CAAhB;AACAZ,QAAAA,MAAM,CAACkB,OAAP,CAAgB,SAAQT,KAAM,eAAcK,WAAW,CAACK,YAAa,WAArE;AACA,eAAOlB,UAAU,CAAC;AAACQ,UAAAA,KAAD;AAAQC,UAAAA;AAAR,SAAD,CAAjB;AACD;;AAED,UAAII,WAAW,CAACK,YAAZ,GAA2B,CAA/B,EAAkC;AAChCnB,QAAAA,MAAM,CAACa,KAAP,CAAc,eAAcC,WAAW,CAACK,YAAa,MAAKV,KAAM,WAAhE;AACA,eAAO,KAAP;AACD;;AACDT,MAAAA,MAAM,CAACa,KAAP,CAAc,SAAQJ,KAAM,QAAOK,WAAW,CAACK,YAAa,WAA5D;;AAEA,UAAIT,KAAK,KAAK,UAAd,EAA0B;AACxB,eAAOI,WAAW,CAACK,YAAnB;AACD,OAhBC,CAkBF;AACA;AACA;AACA;;;AACA,UAAIT,KAAK,KAAK,KAAd,EAAqB;AACnB,eAAOP,UAAU,CAACM,KAAD,EAAQE,QAAR,CAAjB;AACD,OAxBC,CA0BF;AACA;;;AACA,UAAID,KAAK,KAAK,OAAd,EAAuB;AACrB,eAAOR,YAAY,CAACO,KAAD,EAAQE,QAAR,CAAnB;AACD,OA9BC,CAgCF;;;AACA,YAAM,IAAIS,qBAAJ,CAAaC,oBAAWC,oBAAxB,CAAN;AACD,KAlCD,CAkCE,OAAOC,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;;AAED,aAASN,UAAT,CAAoBL,KAApB,EAA2B;AACzB,UAAIA,KAAJ,EAAW;AACT,eAAOd,OAAO,CAACmB,UAAR,CAAmBR,KAAnB,CAAP;AACD;AACF;AACF;;AAED,iBAAeP,YAAf,CAA4BO,KAA5B,EAAmCE,QAAnC,EAA6C;AAC3CX,IAAAA,MAAM,CAACa,KAAP,CAAc,wCAAuCJ,KAAM,EAA3D;;AACA,QAAI;AACF,YAAMX,OAAO,CAACiB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAGF;;AACA,YAAMQ,QAAQ,GAAG,MAAMC,OAAO,CAAChB,KAAD,CAA9B;AACA,YAAMiB,OAAO,GAAGC,aAAa,CAACH,QAAQ,CAAC,CAAD,CAAT,CAA7B;AACAxB,MAAAA,MAAM,CAAC4B,KAAP,CAAc,iBAAgBJ,QAAQ,GAAGA,QAAQ,CAACK,MAAZ,GAAqB,GAAI,eAAcpB,KAAM,KAAIE,QAAQ,GAAG,YAAH,GAAkB,eAAgB,EAAjI;;AAEA,UAAIA,QAAJ,EAAc;AACZ,cAAMmB,OAAO,GAAG,MAAMtB,iBAAiB,CAACgB,QAAD,CAAvC;AACA,eAAO;AAACE,UAAAA,OAAD;AAAUI,UAAAA,OAAV;AAAmBN,UAAAA;AAAnB,SAAP;AACD;;AAED,aAAO;AAACE,QAAAA,OAAD;AAAUF,QAAAA;AAAV,OAAP;AACD,KAdD,CAcE,OAAOD,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAepB,UAAf,CAA0BM,KAA1B,EAAiCE,QAAjC,EAA2C;AACzCX,IAAAA,MAAM,CAACa,KAAP,CAAc,sCAAqCJ,KAAM,EAAzD;;AACA,QAAI;AACF,YAAMX,OAAO,CAACiB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAGF;;AACA,YAAMe,OAAO,GAAG,MAAMjC,OAAO,CAACkC,GAAR,CAAYvB,KAAZ,CAAtB;AAEAT,MAAAA,MAAM,CAACa,KAAP,CAAc,YAAW,mBAAQkB,OAAR,EAAiB;AAACE,QAAAA,MAAM,EAAE,IAAT;AAAeC,QAAAA,cAAc,EAAE,CAA/B;AAAkCC,QAAAA,KAAK,EAAE;AAAzC,OAAjB,CAA8D,EAAvF,EANE,CAOF;;AACAnC,MAAAA,MAAM,CAAC4B,KAAP,CAAc,0BAAyBnB,KAAM,IAAGE,QAAQ,GAAG,YAAH,GAAkB,kBAAmB,EAA7F;;AAEA,UAAIoB,OAAJ,EAAa;AACX,YAAIpB,QAAJ,EAAc;AACZ,gBAAMe,OAAO,GAAGC,aAAa,CAACI,OAAD,CAA7B;AACA,gBAAMD,OAAO,GAAGtB,iBAAiB,CAAC,CAACuB,OAAD,CAAD,CAAjC;AACA,iBAAO;AAACL,YAAAA,OAAD;AAAUI,YAAAA,OAAV;AAAmBN,YAAAA,QAAQ,EAAE,CAACO,OAAD;AAA7B,WAAP;AACD;;AAED,eAAOA,OAAP;AACD;;AAED,aAAO,KAAP;AACD,KArBD,CAqBE,OAAOR,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,WAASnB,WAAT,CAAqBoB,QAArB,EAA+B;AAC7BA,IAAAA,QAAQ,CAACY,OAAT,CAAiBL,OAAO,IAAI;AAC1B/B,MAAAA,MAAM,CAACa,KAAP,CAAc,eAAckB,OAAO,CAACM,UAAR,CAAmBC,aAAc,EAA7D;AACAxC,MAAAA,OAAO,CAACyC,GAAR,CAAYR,OAAZ;AACD,KAHD;AAID;;AAED,WAAS1B,YAAT,CAAsBmB,QAAtB,EAAgC;AAC9BA,IAAAA,QAAQ,CAACY,OAAT,CAAiBL,OAAO,IAAI;AAC1B/B,MAAAA,MAAM,CAACa,KAAP,CAAc,gBAAekB,OAAO,CAACM,UAAR,CAAmBC,aAAc,EAA9D;AACAxC,MAAAA,OAAO,CAAC0C,IAAR,CAAaT,OAAb;AACD,KAHD;AAID;;AAED,iBAAezB,WAAf,CAA2B;AAACG,IAAAA,KAAD;AAAQ6B,IAAAA,aAAR;AAAuBZ,IAAAA,OAAvB;AAAgCe,IAAAA;AAAhC,GAA3B,EAAkE;AAChE,QAAI;AACFzC,MAAAA,MAAM,CAACa,KAAP,CAAc,SAAQJ,KAAM,EAA5B;AACAT,MAAAA,MAAM,CAACa,KAAP,CAAc,iBAAgByB,aAAc,EAA5C;AACAtC,MAAAA,MAAM,CAACa,KAAP,CAAc,QAAO6B,IAAI,CAACC,SAAL,CAAeF,IAAf,CAAqB,EAA1C;AACAzC,MAAAA,MAAM,CAACa,KAAP,CAAc,WAAU6B,IAAI,CAACC,SAAL,CAAejB,OAAf,CAAwB,EAAhD;AAEA,YAAM5B,OAAO,CAACiB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN;AAEAlB,MAAAA,OAAO,CAACQ,WAAR,CACEG,KADF,EAEEmC,MAAM,CAACC,IAAP,CAAYH,IAAI,CAACC,SAAL,CAAe;AAACF,QAAAA;AAAD,OAAf,CAAZ,CAFF,EAGE;AACEH,QAAAA,aADF;AAEEQ,QAAAA,UAAU,EAAE,IAFd;AAGEpB,QAAAA;AAHF,OAHF;AASA1B,MAAAA,MAAM,CAACa,KAAP,CAAc,oBAAmByB,aAAc,cAAa7B,KAAM,EAAlE;AACD,KAlBD,CAkBE,OAAOc,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAehB,WAAf,CAA2BE,KAA3B,EAAkC;AAChCT,IAAAA,MAAM,CAACkB,OAAP,CAAgB,kBAAiBT,KAAM,GAAvC;AACA,UAAMX,OAAO,CAACiD,WAAR,CAAoBtC,KAApB,CAAN;AACD,GAhJsC,CAkJvC;AACA;AACA;;;AAEA,WAASD,iBAAT,CAA2BgB,QAA3B,EAAqC;AACnCxB,IAAAA,MAAM,CAAC4B,KAAP,CAAc,qBAAoBJ,QAAQ,CAACK,MAAO,cAAlD;AAEA,WAAOL,QAAQ,CAACwB,GAAT,CAAajB,OAAO,IAAI;AAC7B,YAAMkB,OAAO,GAAGP,IAAI,CAACQ,KAAL,CAAWnB,OAAO,CAACkB,OAAR,CAAgBE,QAAhB,EAAX,CAAhB;AACA,aAAO,IAAIC,sBAAJ,CAAeH,OAAO,CAACR,IAAvB,CAAP;AACD,KAHM,CAAP;AAID;;AAED,iBAAehB,OAAf,CAAuBhB,KAAvB,EAA8B;AAC5BT,IAAAA,MAAM,CAAC4B,KAAP,CAAc,2BAA0BnB,KAAM,EAA9C;;AACA,QAAI;AACF,YAAM;AAACU,QAAAA;AAAD,UAAiB,MAAMrB,OAAO,CAACG,UAAR,CAAmBQ,KAAnB,CAA7B;AACAT,MAAAA,MAAM,CAACa,KAAP,CAAc,YAAWM,YAAa,sBAAqBV,KAAM,EAAjE;AACA,YAAM4C,aAAa,GAAGlC,YAAY,IAAImC,qBAAhB,GAA6BA,qBAA7B,GAA0CnC,YAAhE;AACAnB,MAAAA,MAAM,CAACa,KAAP,CAAc,WAAUwC,aAAc,wBAAuB5C,KAAM,EAAnE;AAEA,YAAMe,QAAQ,GAAG,MAAM+B,IAAI,CAACF,aAAD,CAA3B;AAEArD,MAAAA,MAAM,CAAC4B,KAAP,CAAc,aAAYJ,QAAQ,CAACK,MAAO,kBAA1C;AAEA,aAAOL,QAAP;AACD,KAXD,CAWE,OAAOD,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;;AAED,mBAAegC,IAAf,CAAoBC,KAApB,EAA2BC,OAAO,GAAG,EAArC,EAAyCC,WAAW,GAAG,EAAvD,EAA2D;AACzD,UAAIF,KAAK,KAAK,CAAd,EAAiB;AACf,eAAOC,OAAP;AACD;;AAED,YAAM1B,OAAO,GAAG,MAAMjC,OAAO,CAACkC,GAAR,CAAYvB,KAAZ,CAAtB;AACA,YAAMkD,UAAU,GAAG;AACjBrB,QAAAA,aAAa,EAAEP,OAAO,CAACM,UAAR,CAAmBC,aADjB;AAEjBsB,QAAAA,WAAW,EAAE7B,OAAO,CAAC8B,MAAR,CAAeD;AAFX,OAAnB,CANyD,CAUzD;;AACA,UAAIF,WAAW,CAACI,QAAZ,CAAqBH,UAArB,CAAJ,EAAsC;AACpC,eAAOJ,IAAI,CAACC,KAAK,GAAG,CAAT,EAAYC,OAAZ,EAAqBC,WAArB,CAAX;AACD;;AAED,aAAOH,IAAI,CAACC,KAAK,GAAG,CAAT,EAAYC,OAAO,CAACM,MAAR,CAAehC,OAAf,CAAZ,EAAqC2B,WAAW,CAACK,MAAZ,CAAmBJ,UAAnB,CAArC,CAAX;AACD;AACF;;AAED,WAAShC,aAAT,CAAuBc,IAAvB,EAA6B;AAC3B,WAAOA,IAAI,CAACJ,UAAL,CAAgBX,OAAvB;AACD;AACF","sourcesContent":["\n/**\n*\n* @licstart The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020-2021 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program. If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport amqplib from 'amqplib';\nimport {MarcRecord} from '@natlibfi/marc-record';\nimport {Error as ApiError} from '@natlibfi/melinda-commons';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {CHUNK_SIZE} from './constants';\nimport {logError} from './utils';\nimport {inspect} from 'util';\nimport httpStatus from 'http-status';\n\nexport default async function (AMQP_URL) {\n const connection = await amqplib.connect(AMQP_URL);\n const channel = await connection.createChannel();\n const logger = createLogger();\n\n return {checkQueue, consumeChunk, consumeOne, ackMessages, nackMessages, sendToQueue, removeQueue, messagesToRecords};\n\n async function checkQueue({queue, style = 'basic', toRecord = true, purge = false}) {\n logger.silly(`checkQueue: ${queue}, Style: ${style}: toRecord: ${toRecord}, Purge: ${purge}, `);\n try {\n const channelInfo = await channel.assertQueue(queue, {durable: true});\n if (purge) {\n await purgeQueue(purge);\n logger.verbose(`Queue ${queue} has purged ${channelInfo.messageCount} messages`);\n return checkQueue({queue, style});\n }\n\n if (channelInfo.messageCount < 1) {\n logger.silly(`checkQueue: ${channelInfo.messageCount} - ${queue} is empty`);\n return false;\n }\n logger.silly(`Queue ${queue} has ${channelInfo.messageCount} messages`);\n\n if (style === 'messages') {\n return channelInfo.messageCount;\n }\n\n // Note: returns one message (+ record, of toRecord: true)\n // note: if toRecord is false returns just plain message / false\n // note: if toRecord is true returns {headers, records, messages} -object / false\n // should this be more consistent?\n if (style === 'one') {\n return consumeOne(queue, toRecord);\n }\n\n // Note: returns a chunk of (100) messages (+ records, if toRecord: true)\n // returns {headers, records, messages} object or {headers, messages} object depending on toRecord\n if (style === 'basic') {\n return consumeChunk(queue, toRecord);\n }\n\n // Defaults:\n throw new ApiError(httpStatus.UNPROCESSABLE_ENTITY);\n } catch (error) {\n logError(error);\n }\n\n function purgeQueue(purge) {\n if (purge) {\n return channel.purgeQueue(queue);\n }\n }\n }\n\n async function consumeChunk(queue, toRecord) {\n logger.silly(`Prepared to consumeChunk from queue: ${queue}`);\n try {\n await channel.assertQueue(queue, {durable: true});\n\n // getData: next chunk (100) messages\n const messages = await getData(queue);\n const headers = getHeaderInfo(messages[0]);\n logger.debug(`consumeChunk (${messages ? messages.length : '0'} from queue ${queue}) ${toRecord ? 'to records' : 'just messages'}`);\n\n if (toRecord) {\n const records = await messagesToRecords(messages);\n return {headers, records, messages};\n }\n\n return {headers, messages};\n } catch (error) {\n logError(error);\n }\n }\n\n async function consumeOne(queue, toRecord) {\n logger.silly(`Prepared to consumeOne from queue: ${queue}`);\n try {\n await channel.assertQueue(queue, {durable: true});\n\n // Returns false if 0 items in queue\n const message = await channel.get(queue);\n\n logger.silly(`Message: ${inspect(message, {colors: true, maxArrayLength: 3, depth: 3})}`);\n // Do not spam the logs\n logger.debug(`consumeOne from queue: ${queue} ${toRecord ? 'to records' : 'just the message'}`);\n\n if (message) {\n if (toRecord) {\n const headers = getHeaderInfo(message);\n const records = messagesToRecords([message]);\n return {headers, records, messages: [message]};\n }\n\n return message;\n }\n\n return false;\n } catch (error) {\n logError(error);\n }\n }\n\n function ackMessages(messages) {\n messages.forEach(message => {\n logger.silly(`Ack message ${message.properties.correlationId}`);\n channel.ack(message);\n });\n }\n\n function nackMessages(messages) {\n messages.forEach(message => {\n logger.silly(`Nack message ${message.properties.correlationId}`);\n channel.nack(message);\n });\n }\n\n async function sendToQueue({queue, correlationId, headers, data}) {\n try {\n logger.silly(`Queue ${queue}`);\n logger.silly(`CorrelationId ${correlationId}`);\n logger.silly(`Data ${JSON.stringify(data)}`);\n logger.silly(`Headers ${JSON.stringify(headers)}`);\n\n await channel.assertQueue(queue, {durable: true});\n\n channel.sendToQueue(\n queue,\n Buffer.from(JSON.stringify({data})),\n {\n correlationId,\n persistent: true,\n headers\n }\n );\n logger.silly(`Send message for ${correlationId} to queue: ${queue}`);\n } catch (error) {\n logError(error);\n }\n }\n\n async function removeQueue(queue) {\n logger.verbose(`Removing queue ${queue}.`);\n await channel.deleteQueue(queue);\n }\n\n // ----------------\n // Helper functions\n // ----------------\n\n function messagesToRecords(messages) {\n logger.debug(`Parsing messages (${messages.length}) to records`);\n\n return messages.map(message => {\n const content = JSON.parse(message.content.toString());\n return new MarcRecord(content.data);\n });\n }\n\n async function getData(queue) {\n logger.debug(`Getting queue data from ${queue}`);\n try {\n const {messageCount} = await channel.checkQueue(queue);\n logger.silly(`There is ${messageCount} messages in queue ${queue}`);\n const messagesToGet = messageCount >= CHUNK_SIZE ? CHUNK_SIZE : messageCount;\n logger.silly(`Getting ${messagesToGet} messages from queue ${queue}`);\n\n const messages = await pump(messagesToGet);\n\n logger.debug(`Returning ${messages.length} unique messages`);\n\n return messages;\n } catch (error) {\n logError(error);\n }\n\n async function pump(count, results = [], identifiers = []) {\n if (count === 0) {\n return results;\n }\n\n const message = await channel.get(queue);\n const identifier = {\n correlationId: message.properties.correlationId,\n deliveryTag: message.fields.deliveryTag\n };\n // Filter not unique messages\n if (identifiers.includes(identifier)) {\n return pump(count - 1, results, identifiers);\n }\n\n return pump(count - 1, results.concat(message), identifiers.concat(identifier));\n }\n }\n\n function getHeaderInfo(data) {\n return data.properties.headers;\n }\n}\n"],"file":"amqp.js"}
|
|
1
|
+
{"version":3,"sources":["../src/amqp.js"],"names":["AMQP_URL","connection","amqplib","connect","channel","createChannel","logger","checkQueue","consumeChunk","consumeOne","ackMessages","nackMessages","sendToQueue","removeQueue","messagesToRecords","queue","style","toRecord","purge","silly","channelInfo","assertQueue","durable","purgeQueue","verbose","messageCount","ApiError","httpStatus","UNPROCESSABLE_ENTITY","error","messages","getData","headers","getHeaderInfo","debug","length","records","message","get","colors","maxArrayLength","depth","forEach","properties","correlationId","ack","nack","data","JSON","stringify","Buffer","from","persistent","tempChannel","deleteQueue","close","map","content","parse","toString","MarcRecord","messagesToGet","CHUNK_SIZE","pump","count","results","identifiers","identifier","deliveryTag","fields","includes","concat"],"mappings":";;;;;;;AA6BA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;AACA;;;;AAnCA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAWe,wBAAgBA,QAAhB,EAA0B;AACvC,QAAMC,UAAU,GAAG,MAAMC,iBAAQC,OAAR,CAAgBH,QAAhB,CAAzB;AACA,QAAMI,OAAO,GAAG,MAAMH,UAAU,CAACI,aAAX,EAAtB;AACA,QAAMC,MAAM,GAAG,0CAAf;AAEA,SAAO;AAACC,IAAAA,UAAD;AAAaC,IAAAA,YAAb;AAA2BC,IAAAA,UAA3B;AAAuCC,IAAAA,WAAvC;AAAoDC,IAAAA,YAApD;AAAkEC,IAAAA,WAAlE;AAA+EC,IAAAA,WAA/E;AAA4FC,IAAAA;AAA5F,GAAP;;AAEA,iBAAeP,UAAf,CAA0B;AAACQ,IAAAA,KAAD;AAAQC,IAAAA,KAAK,GAAG,OAAhB;AAAyBC,IAAAA,QAAQ,GAAG,IAApC;AAA0CC,IAAAA,KAAK,GAAG;AAAlD,GAA1B,EAAoF;AAClFZ,IAAAA,MAAM,CAACa,KAAP,CAAc,eAAcJ,KAAM,YAAWC,KAAM,eAAcC,QAAS,YAAWC,KAAM,IAA3F;;AACA,QAAI;AACF,YAAME,WAAW,GAAG,MAAMhB,OAAO,CAACiB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAA1B;;AACA,UAAIJ,KAAJ,EAAW;AACT,cAAMK,UAAU,CAACL,KAAD,CAAhB;AACAZ,QAAAA,MAAM,CAACkB,OAAP,CAAgB,SAAQT,KAAM,eAAcK,WAAW,CAACK,YAAa,WAArE;AACA,eAAOlB,UAAU,CAAC;AAACQ,UAAAA,KAAD;AAAQC,UAAAA;AAAR,SAAD,CAAjB;AACD;;AAED,UAAII,WAAW,CAACK,YAAZ,GAA2B,CAA/B,EAAkC;AAChCnB,QAAAA,MAAM,CAACa,KAAP,CAAc,eAAcC,WAAW,CAACK,YAAa,MAAKV,KAAM,WAAhE;AACA,eAAO,KAAP;AACD;;AACDT,MAAAA,MAAM,CAACa,KAAP,CAAc,SAAQJ,KAAM,QAAOK,WAAW,CAACK,YAAa,WAA5D;;AAEA,UAAIT,KAAK,KAAK,UAAd,EAA0B;AACxB,eAAOI,WAAW,CAACK,YAAnB;AACD,OAhBC,CAkBF;AACA;AACA;AACA;;;AACA,UAAIT,KAAK,KAAK,KAAd,EAAqB;AACnB,eAAOP,UAAU,CAACM,KAAD,EAAQE,QAAR,CAAjB;AACD,OAxBC,CA0BF;AACA;;;AACA,UAAID,KAAK,KAAK,OAAd,EAAuB;AACrB,eAAOR,YAAY,CAACO,KAAD,EAAQE,QAAR,CAAnB;AACD,OA9BC,CAgCF;;;AACA,YAAM,IAAIS,qBAAJ,CAAaC,oBAAWC,oBAAxB,CAAN;AACD,KAlCD,CAkCE,OAAOC,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;;AAED,aAASN,UAAT,CAAoBL,KAApB,EAA2B;AACzB,UAAIA,KAAJ,EAAW;AACT,eAAOd,OAAO,CAACmB,UAAR,CAAmBR,KAAnB,CAAP;AACD;AACF;AACF;;AAED,iBAAeP,YAAf,CAA4BO,KAA5B,EAAmCE,QAAnC,EAA6C;AAC3CX,IAAAA,MAAM,CAACa,KAAP,CAAc,wCAAuCJ,KAAM,EAA3D;;AACA,QAAI;AACF,YAAMX,OAAO,CAACiB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAGF;;AACA,YAAMQ,QAAQ,GAAG,MAAMC,OAAO,CAAChB,KAAD,CAA9B;AACA,YAAMiB,OAAO,GAAGC,aAAa,CAACH,QAAQ,CAAC,CAAD,CAAT,CAA7B;AACAxB,MAAAA,MAAM,CAAC4B,KAAP,CAAc,iBAAgBJ,QAAQ,GAAGA,QAAQ,CAACK,MAAZ,GAAqB,GAAI,eAAcpB,KAAM,KAAIE,QAAQ,GAAG,YAAH,GAAkB,eAAgB,EAAjI;;AAEA,UAAIA,QAAJ,EAAc;AACZ,cAAMmB,OAAO,GAAG,MAAMtB,iBAAiB,CAACgB,QAAD,CAAvC;AACA,eAAO;AAACE,UAAAA,OAAD;AAAUI,UAAAA,OAAV;AAAmBN,UAAAA;AAAnB,SAAP;AACD;;AAED,aAAO;AAACE,QAAAA,OAAD;AAAUF,QAAAA;AAAV,OAAP;AACD,KAdD,CAcE,OAAOD,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAepB,UAAf,CAA0BM,KAA1B,EAAiCE,QAAjC,EAA2C;AACzCX,IAAAA,MAAM,CAACa,KAAP,CAAc,sCAAqCJ,KAAM,EAAzD;;AACA,QAAI;AACF,YAAMX,OAAO,CAACiB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN,CADE,CAGF;;AACA,YAAMe,OAAO,GAAG,MAAMjC,OAAO,CAACkC,GAAR,CAAYvB,KAAZ,CAAtB;AAEAT,MAAAA,MAAM,CAACa,KAAP,CAAc,YAAW,mBAAQkB,OAAR,EAAiB;AAACE,QAAAA,MAAM,EAAE,IAAT;AAAeC,QAAAA,cAAc,EAAE,CAA/B;AAAkCC,QAAAA,KAAK,EAAE;AAAzC,OAAjB,CAA8D,EAAvF,EANE,CAOF;;AACAnC,MAAAA,MAAM,CAAC4B,KAAP,CAAc,0BAAyBnB,KAAM,IAAGE,QAAQ,GAAG,YAAH,GAAkB,kBAAmB,EAA7F;;AAEA,UAAIoB,OAAJ,EAAa;AACX,YAAIpB,QAAJ,EAAc;AACZ,gBAAMe,OAAO,GAAGC,aAAa,CAACI,OAAD,CAA7B;AACA,gBAAMD,OAAO,GAAGtB,iBAAiB,CAAC,CAACuB,OAAD,CAAD,CAAjC;AACA,iBAAO;AAACL,YAAAA,OAAD;AAAUI,YAAAA,OAAV;AAAmBN,YAAAA,QAAQ,EAAE,CAACO,OAAD;AAA7B,WAAP;AACD;;AAED,eAAOA,OAAP;AACD;;AAED,aAAO,KAAP;AACD,KArBD,CAqBE,OAAOR,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,WAASnB,WAAT,CAAqBoB,QAArB,EAA+B;AAC7BA,IAAAA,QAAQ,CAACY,OAAT,CAAiBL,OAAO,IAAI;AAC1B/B,MAAAA,MAAM,CAACa,KAAP,CAAc,eAAckB,OAAO,CAACM,UAAR,CAAmBC,aAAc,EAA7D;AACAxC,MAAAA,OAAO,CAACyC,GAAR,CAAYR,OAAZ;AACD,KAHD;AAID;;AAED,WAAS1B,YAAT,CAAsBmB,QAAtB,EAAgC;AAC9BA,IAAAA,QAAQ,CAACY,OAAT,CAAiBL,OAAO,IAAI;AAC1B/B,MAAAA,MAAM,CAACa,KAAP,CAAc,gBAAekB,OAAO,CAACM,UAAR,CAAmBC,aAAc,EAA9D;AACAxC,MAAAA,OAAO,CAAC0C,IAAR,CAAaT,OAAb;AACD,KAHD;AAID;;AAED,iBAAezB,WAAf,CAA2B;AAACG,IAAAA,KAAD;AAAQ6B,IAAAA,aAAR;AAAuBZ,IAAAA,OAAvB;AAAgCe,IAAAA;AAAhC,GAA3B,EAAkE;AAChE,QAAI;AACFzC,MAAAA,MAAM,CAACa,KAAP,CAAc,SAAQJ,KAAM,EAA5B;AACAT,MAAAA,MAAM,CAACa,KAAP,CAAc,iBAAgByB,aAAc,EAA5C;AACAtC,MAAAA,MAAM,CAACa,KAAP,CAAc,QAAO6B,IAAI,CAACC,SAAL,CAAeF,IAAf,CAAqB,EAA1C;AACAzC,MAAAA,MAAM,CAACa,KAAP,CAAc,WAAU6B,IAAI,CAACC,SAAL,CAAejB,OAAf,CAAwB,EAAhD;AAEA,YAAM5B,OAAO,CAACiB,WAAR,CAAoBN,KAApB,EAA2B;AAACO,QAAAA,OAAO,EAAE;AAAV,OAA3B,CAAN;AAEAlB,MAAAA,OAAO,CAACQ,WAAR,CACEG,KADF,EAEEmC,MAAM,CAACC,IAAP,CAAYH,IAAI,CAACC,SAAL,CAAe;AAACF,QAAAA;AAAD,OAAf,CAAZ,CAFF,EAGE;AACEH,QAAAA,aADF;AAEEQ,QAAAA,UAAU,EAAE,IAFd;AAGEpB,QAAAA;AAHF,OAHF;AASA1B,MAAAA,MAAM,CAACa,KAAP,CAAc,oBAAmByB,aAAc,cAAa7B,KAAM,EAAlE;AACD,KAlBD,CAkBE,OAAOc,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;AACF;;AAED,iBAAehB,WAAf,CAA2BE,KAA3B,EAAkC;AAChC;AACA;AACA;AACA,UAAMsC,WAAW,GAAG,MAAMpD,UAAU,CAACI,aAAX,EAA1B;AACAC,IAAAA,MAAM,CAACkB,OAAP,CAAgB,kBAAiBT,KAAM,GAAvC;AACA,UAAMsC,WAAW,CAACC,WAAZ,CAAwBvC,KAAxB,CAAN;;AACA,QAAIsC,WAAJ,EAAiB;AAAE;AACjB,YAAMA,WAAW,CAACE,KAAZ,EAAN;AACD;AACF,GAvJsC,CAyJvC;AACA;AACA;;;AAEA,WAASzC,iBAAT,CAA2BgB,QAA3B,EAAqC;AACnCxB,IAAAA,MAAM,CAAC4B,KAAP,CAAc,qBAAoBJ,QAAQ,CAACK,MAAO,cAAlD;AAEA,WAAOL,QAAQ,CAAC0B,GAAT,CAAanB,OAAO,IAAI;AAC7B,YAAMoB,OAAO,GAAGT,IAAI,CAACU,KAAL,CAAWrB,OAAO,CAACoB,OAAR,CAAgBE,QAAhB,EAAX,CAAhB;AACA,aAAO,IAAIC,sBAAJ,CAAeH,OAAO,CAACV,IAAvB,CAAP;AACD,KAHM,CAAP;AAID;;AAED,iBAAehB,OAAf,CAAuBhB,KAAvB,EAA8B;AAC5BT,IAAAA,MAAM,CAAC4B,KAAP,CAAc,2BAA0BnB,KAAM,EAA9C;;AACA,QAAI;AACF,YAAM;AAACU,QAAAA;AAAD,UAAiB,MAAMrB,OAAO,CAACG,UAAR,CAAmBQ,KAAnB,CAA7B;AACAT,MAAAA,MAAM,CAACa,KAAP,CAAc,YAAWM,YAAa,sBAAqBV,KAAM,EAAjE;AACA,YAAM8C,aAAa,GAAGpC,YAAY,IAAIqC,qBAAhB,GAA6BA,qBAA7B,GAA0CrC,YAAhE;AACAnB,MAAAA,MAAM,CAACa,KAAP,CAAc,WAAU0C,aAAc,wBAAuB9C,KAAM,EAAnE;AAEA,YAAMe,QAAQ,GAAG,MAAMiC,IAAI,CAACF,aAAD,CAA3B;AAEAvD,MAAAA,MAAM,CAAC4B,KAAP,CAAc,aAAYJ,QAAQ,CAACK,MAAO,kBAA1C;AAEA,aAAOL,QAAP;AACD,KAXD,CAWE,OAAOD,KAAP,EAAc;AACd,2BAASA,KAAT;AACD;;AAED,mBAAekC,IAAf,CAAoBC,KAApB,EAA2BC,OAAO,GAAG,EAArC,EAAyCC,WAAW,GAAG,EAAvD,EAA2D;AACzD,UAAIF,KAAK,KAAK,CAAd,EAAiB;AACf,eAAOC,OAAP;AACD;;AAED,YAAM5B,OAAO,GAAG,MAAMjC,OAAO,CAACkC,GAAR,CAAYvB,KAAZ,CAAtB;AACA,YAAMoD,UAAU,GAAG;AACjBvB,QAAAA,aAAa,EAAEP,OAAO,CAACM,UAAR,CAAmBC,aADjB;AAEjBwB,QAAAA,WAAW,EAAE/B,OAAO,CAACgC,MAAR,CAAeD;AAFX,OAAnB,CANyD,CAUzD;;AACA,UAAIF,WAAW,CAACI,QAAZ,CAAqBH,UAArB,CAAJ,EAAsC;AACpC,eAAOJ,IAAI,CAACC,KAAK,GAAG,CAAT,EAAYC,OAAZ,EAAqBC,WAArB,CAAX;AACD;;AAED,aAAOH,IAAI,CAACC,KAAK,GAAG,CAAT,EAAYC,OAAO,CAACM,MAAR,CAAelC,OAAf,CAAZ,EAAqC6B,WAAW,CAACK,MAAZ,CAAmBJ,UAAnB,CAArC,CAAX;AACD;AACF;;AAED,WAASlC,aAAT,CAAuBc,IAAvB,EAA6B;AAC3B,WAAOA,IAAI,CAACJ,UAAL,CAAgBX,OAAvB;AACD;AACF","sourcesContent":["\n/**\n*\n* @licstart The following is the entire license notice for the JavaScript code in this file.\n*\n* Shared modules for microservices of Melinda rest api batch import system\n*\n* Copyright (C) 2020-2021 University Of Helsinki (The National Library Of Finland)\n*\n* This file is part of melinda-rest-api-commons\n*\n* melinda-rest-api-commons program is free software: you can redistribute it and/or modify\n* it under the terms of the GNU Affero General Public License as\n* published by the Free Software Foundation, either version 3 of the\n* License, or (at your option) any later version.\n*\n* melinda-rest-api-commons is distributed in the hope that it will be useful,\n* but WITHOUT ANY WARRANTY; without even the implied warranty of\n* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n* GNU Affero General Public License for more details.\n*\n* You should have received a copy of the GNU Affero General Public License\n* along with this program. If not, see <http://www.gnu.org/licenses/>.\n*\n* @licend The above is the entire license notice\n* for the JavaScript code in this file.\n*\n*/\n\nimport amqplib from 'amqplib';\nimport {MarcRecord} from '@natlibfi/marc-record';\nimport {Error as ApiError} from '@natlibfi/melinda-commons';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {CHUNK_SIZE} from './constants';\nimport {logError} from './utils';\nimport {inspect} from 'util';\nimport httpStatus from 'http-status';\n\nexport default async function (AMQP_URL) {\n const connection = await amqplib.connect(AMQP_URL);\n const channel = await connection.createChannel();\n const logger = createLogger();\n\n return {checkQueue, consumeChunk, consumeOne, ackMessages, nackMessages, sendToQueue, removeQueue, messagesToRecords};\n\n async function checkQueue({queue, style = 'basic', toRecord = true, purge = false}) {\n logger.silly(`checkQueue: ${queue}, Style: ${style}: toRecord: ${toRecord}, Purge: ${purge}, `);\n try {\n const channelInfo = await channel.assertQueue(queue, {durable: true});\n if (purge) {\n await purgeQueue(purge);\n logger.verbose(`Queue ${queue} has purged ${channelInfo.messageCount} messages`);\n return checkQueue({queue, style});\n }\n\n if (channelInfo.messageCount < 1) {\n logger.silly(`checkQueue: ${channelInfo.messageCount} - ${queue} is empty`);\n return false;\n }\n logger.silly(`Queue ${queue} has ${channelInfo.messageCount} messages`);\n\n if (style === 'messages') {\n return channelInfo.messageCount;\n }\n\n // Note: returns one message (+ record, of toRecord: true)\n // note: if toRecord is false returns just plain message / false\n // note: if toRecord is true returns {headers, records, messages} -object / false\n // should this be more consistent?\n if (style === 'one') {\n return consumeOne(queue, toRecord);\n }\n\n // Note: returns a chunk of (100) messages (+ records, if toRecord: true)\n // returns {headers, records, messages} object or {headers, messages} object depending on toRecord\n if (style === 'basic') {\n return consumeChunk(queue, toRecord);\n }\n\n // Defaults:\n throw new ApiError(httpStatus.UNPROCESSABLE_ENTITY);\n } catch (error) {\n logError(error);\n }\n\n function purgeQueue(purge) {\n if (purge) {\n return channel.purgeQueue(queue);\n }\n }\n }\n\n async function consumeChunk(queue, toRecord) {\n logger.silly(`Prepared to consumeChunk from queue: ${queue}`);\n try {\n await channel.assertQueue(queue, {durable: true});\n\n // getData: next chunk (100) messages\n const messages = await getData(queue);\n const headers = getHeaderInfo(messages[0]);\n logger.debug(`consumeChunk (${messages ? messages.length : '0'} from queue ${queue}) ${toRecord ? 'to records' : 'just messages'}`);\n\n if (toRecord) {\n const records = await messagesToRecords(messages);\n return {headers, records, messages};\n }\n\n return {headers, messages};\n } catch (error) {\n logError(error);\n }\n }\n\n async function consumeOne(queue, toRecord) {\n logger.silly(`Prepared to consumeOne from queue: ${queue}`);\n try {\n await channel.assertQueue(queue, {durable: true});\n\n // Returns false if 0 items in queue\n const message = await channel.get(queue);\n\n logger.silly(`Message: ${inspect(message, {colors: true, maxArrayLength: 3, depth: 3})}`);\n // Do not spam the logs\n logger.debug(`consumeOne from queue: ${queue} ${toRecord ? 'to records' : 'just the message'}`);\n\n if (message) {\n if (toRecord) {\n const headers = getHeaderInfo(message);\n const records = messagesToRecords([message]);\n return {headers, records, messages: [message]};\n }\n\n return message;\n }\n\n return false;\n } catch (error) {\n logError(error);\n }\n }\n\n function ackMessages(messages) {\n messages.forEach(message => {\n logger.silly(`Ack message ${message.properties.correlationId}`);\n channel.ack(message);\n });\n }\n\n function nackMessages(messages) {\n messages.forEach(message => {\n logger.silly(`Nack message ${message.properties.correlationId}`);\n channel.nack(message);\n });\n }\n\n async function sendToQueue({queue, correlationId, headers, data}) {\n try {\n logger.silly(`Queue ${queue}`);\n logger.silly(`CorrelationId ${correlationId}`);\n logger.silly(`Data ${JSON.stringify(data)}`);\n logger.silly(`Headers ${JSON.stringify(headers)}`);\n\n await channel.assertQueue(queue, {durable: true});\n\n channel.sendToQueue(\n queue,\n Buffer.from(JSON.stringify({data})),\n {\n correlationId,\n persistent: true,\n headers\n }\n );\n logger.silly(`Send message for ${correlationId} to queue: ${queue}`);\n } catch (error) {\n logError(error);\n }\n }\n\n async function removeQueue(queue) {\n // deleteQueue borks the channel if the queue does not exist\n // -> use throwaway tempChannel to avoid killing actual channel in use\n // this might be doable also with assertQueue before deleteQueue\n const tempChannel = await connection.createChannel();\n logger.verbose(`Removing queue ${queue}.`);\n await tempChannel.deleteQueue(queue);\n if (tempChannel) { // eslint-disable-line functional/no-conditional-statement\n await tempChannel.close();\n }\n }\n\n // ----------------\n // Helper functions\n // ----------------\n\n function messagesToRecords(messages) {\n logger.debug(`Parsing messages (${messages.length}) to records`);\n\n return messages.map(message => {\n const content = JSON.parse(message.content.toString());\n return new MarcRecord(content.data);\n });\n }\n\n async function getData(queue) {\n logger.debug(`Getting queue data from ${queue}`);\n try {\n const {messageCount} = await channel.checkQueue(queue);\n logger.silly(`There is ${messageCount} messages in queue ${queue}`);\n const messagesToGet = messageCount >= CHUNK_SIZE ? CHUNK_SIZE : messageCount;\n logger.silly(`Getting ${messagesToGet} messages from queue ${queue}`);\n\n const messages = await pump(messagesToGet);\n\n logger.debug(`Returning ${messages.length} unique messages`);\n\n return messages;\n } catch (error) {\n logError(error);\n }\n\n async function pump(count, results = [], identifiers = []) {\n if (count === 0) {\n return results;\n }\n\n const message = await channel.get(queue);\n const identifier = {\n correlationId: message.properties.correlationId,\n deliveryTag: message.fields.deliveryTag\n };\n // Filter not unique messages\n if (identifiers.includes(identifier)) {\n return pump(count - 1, results, identifiers);\n }\n\n return pump(count - 1, results.concat(message), identifiers.concat(identifier));\n }\n }\n\n function getHeaderInfo(data) {\n return data.properties.headers;\n }\n}\n"],"file":"amqp.js"}
|