@natlibfi/melinda-rest-api-commons 6.0.1-alpha.1 → 6.1.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/README.md +34 -0
  2. package/dist/amqp.js +37 -15
  3. package/dist/amqp.js.map +2 -2
  4. package/dist/helpers/conversion.js +1 -2
  5. package/dist/helpers/conversion.js.map +2 -2
  6. package/dist/helpers/fix-strip-f884s.js +18 -4
  7. package/dist/helpers/fix-strip-f884s.js.map +2 -2
  8. package/dist/helpers/fix.js +12 -8
  9. package/dist/helpers/fix.js.map +2 -2
  10. package/dist/helpers/validation.js +1 -1
  11. package/dist/helpers/validation.js.map +1 -1
  12. package/dist/mongo.js +200 -109
  13. package/dist/mongo.js.map +3 -3
  14. package/dist/mongoLog.js +113 -35
  15. package/dist/mongoLog.js.map +3 -3
  16. package/dist/utils.js +33 -4
  17. package/dist/utils.js.map +2 -2
  18. package/package.json +11 -17
  19. package/tests/amqp-error-branches.test.js +535 -0
  20. package/tests/amqp.healthtest.js +12 -4
  21. package/tests/amqp.tests.localtest.js +13 -18
  22. package/tests/helpers/conversion.test.js +16 -0
  23. package/tests/helpers/fix-edge.test.js +197 -0
  24. package/tests/helpers/fix.test.js +20 -2
  25. package/tests/helpers/validation.test.js +6 -2
  26. package/tests/mongo-check-and-check-state.test.js +3 -1
  27. package/tests/mongo-check-and-set-import-job-state.test.js +3 -1
  28. package/tests/mongo-error-branches.test.js +787 -0
  29. package/tests/mongo-get-stream.test.js +4 -1
  30. package/tests/mongo-read-content.test.js +10 -0
  31. package/tests/mongoLog-protect.test.js +181 -0
  32. package/tests/mongoLog.test.js +113 -6
  33. package/tests/utils.test.js +360 -0
package/dist/mongo.js.map CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "version": 3,
3
3
  "sources": ["../src/mongo.js"],
4
- "sourcesContent": ["/* eslint-disable max-lines */\n\nimport {MongoClient, GridFSBucket, MongoDriverError} from 'mongodb';\nimport moment from 'moment';\nimport httpStatus from 'http-status';\nimport sanitize from 'mongo-sanitize';\nimport createDebugLogger from 'debug';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {Error as ApiError, parseBoolean} from '@natlibfi/melinda-commons';\nimport {QUEUE_ITEM_STATE, IMPORT_JOB_STATE, OPERATIONS} from './constants.js';\nimport {logError} from './utils.js';\n\n//import isDeepStrictEqual from 'util';\n\n/* QueueItem:\n{\n \"correlationId\":\"FOO\",\n \"cataloger\":\"xxx0000\",\n \"oCatalogerIn\":\"xxx0000\"\n \"operation\":\"UPDATE\",\n \"operations\":[\"UPDATE\", \"CREATE\"]\n \"contentType\":\"application/json\",\n \"recordLoadParams\": {\n \"pActiveLibrary\": \"XXX00\",\n \"pInputFile\": \"filename.seq\",\n \"pRejectFile\": \"filename.rej\",\n \"pLogFile\": \"filename.syslog\",\n \"pOldNew\": \"NEW\"\n },\n \"queueItemState\":\"PENDING_QUEUING\",\n \"importJobState\": {\n \"CREATE\": \"EMPTY\",\n \"UPDATE\": \"EMPTY\"\n },\n blobSize: 1,\n records: [],\n errorMessage: '',\n errorStatus: '',\n \"creationTime\":\"2020-01-01T00:00:00.000Z\",\n \"modificationTime\":\"2020-01-01T00:00:01.000Z\",\n}\n*/\n\nexport default async function (MONGO_URI, collection, db = 'rest-api') {\n const logger = createLogger();\n const debug = createDebugLogger('@natlibfi/melinda-rest-api-commons/mongo');\n const debugDev = debug.extend(':dev');\n\n // Connect to mongo (MONGO)\n const client = await MongoClient.connect(MONGO_URI);\n const dbConnection = client.db(db);\n const gridFSBucket = new GridFSBucket(dbConnection, {bucketName: collection});\n const operator = dbConnection.collection(collection);\n debugDev(`mongo: ${MONGO_URI}, db: ${db}, collection: ${collection}`);\n\n return {createPrio, createBulk, checkAndSetState, checkAndSetImportJobState, query, createProjection, queryById, checkTimeOut, remove, readContent, removeContent, getOne, getStream, setState, setImportJobState, pushIds, pushMessages, setOperation, setOperations, addBlobSize, setBlobSize};\n\n async function createPrio({correlationId, cataloger, oCatalogerIn, operation, operationSettings}) {\n debugDev('createPrio');\n const time = moment().toDate();\n const newQueueItem = {\n correlationId,\n cataloger,\n operation,\n operations: [operation],\n oCatalogerIn,\n operationSettings: {\n prio: true,\n ...operationSettings\n },\n queueItemState: QUEUE_ITEM_STATE.VALIDATOR.PENDING_VALIDATION,\n blobSize: 1,\n importJobState: {\n CREATE: IMPORT_JOB_STATE.EMPTY,\n UPDATE: IMPORT_JOB_STATE.EMPTY,\n FIX: IMPORT_JOB_STATE.EMPTY\n },\n records: [],\n errorMessage: '',\n errorStatus: '',\n creationTime: time,\n modificationTime: time\n };\n try {\n debugDev(`Inserting: ${JSON.stringify(newQueueItem)} to ${collection}}`);\n const result = await operator.insertOne(newQueueItem);\n debugDev(JSON.stringify(result));\n if (result.acknowledged) {\n logger.info(`New PRIO queue item for ${operation} ${correlationId} has been made in ${collection}`);\n return;\n }\n // TEST-COVERAGE: this error is not (currently) tested\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n } catch (error) {\n // TEST-COVERAGE: this error is not (currently) tested\n const errorMessage = error.payload || error.message || '';\n logError(error);\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);\n }\n }\n\n async function createBulk({correlationId, cataloger, oCatalogerIn, operation, contentType, recordLoadParams, stream, operationSettings}) {\n const time = moment().toDate();\n const newQueueItem = {\n correlationId,\n cataloger,\n oCatalogerIn,\n operation,\n operations: [operation],\n operationSettings: {\n prio: false,\n ...operationSettings\n },\n contentType,\n recordLoadParams,\n queueItemState: stream ? QUEUE_ITEM_STATE.VALIDATOR.UPLOADING : QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS,\n blobSize: 0,\n importJobState: {\n CREATE: IMPORT_JOB_STATE.EMPTY,\n UPDATE: IMPORT_JOB_STATE.EMPTY,\n FIX: IMPORT_JOB_STATE.EMPTY\n },\n records: [],\n errorMessage: '',\n errorStatus: '',\n creationTime: time,\n modificationTime: time\n };\n\n if (stream) {\n debug(`Creating streamBulk`);\n try {\n await new Promise((resolve, reject) => {\n const outputStream = gridFSBucket.openUploadStream(correlationId);\n\n stream\n .on('error', reject)\n .on('data', chunk => outputStream.write(chunk))\n .on('end', () => outputStream.end(undefined, undefined, () => {\n resolve(correlationId);\n }));\n });\n logger.info(`New BULK queue item for ${operation} ${correlationId} has been made in ${collection}!`);\n debug(`New BULK queue item for ${operation} ${correlationId} has been made in ${collection}!`);\n return operator.insertOne(newQueueItem);\n } catch (error) {\n const errorMessage = error.payload || error.message || '';\n logError(error);\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);\n }\n }\n\n logger.debug(`No stream`);\n debug(`Creating noStreamBulk`);\n if (!stream) {\n try {\n const result = await operator.insertOne(newQueueItem);\n if (result.acknowledged) {\n logger.info(`New noStream BULK queue item for ${operation} ${correlationId} has been made in ${collection}`);\n debug(`New noStream BULK queue item for ${operation} ${correlationId} has been made in ${collection}`);\n return {correlationId, queueItemState: QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS};\n }\n // TEST-COVERAGE: this error is not (currently) tested\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n } catch (error) {\n // TEST-COVERAGE: this error is not (currently) tested\n const errorMessage = error.payload || error.message || '';\n logError(error);\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);\n }\n }\n }\n\n // Check state that the queueItem has not waited too long and set state\n // Optionally set also queueItems errorStatus & errorMessage\n async function checkAndSetState({correlationId, state, errorMessage = undefined, errorStatus = undefined}) {\n // checkTimeOut returns true, if queueItem is fresher than 1 minute and it's state is not ABORT/ERROR/DONE\n // otherwise it sets queueItem and importJobStates to state ABORT and add errorStatus and errorMessage (408, 'Timeout')\n const timeOut = await checkTimeOut({correlationId});\n if (timeOut) {\n return setState({correlationId, state, errorMessage, errorStatus});\n }\n return false;\n }\n\n // Check state that the queueItem has not waited too long and set importJobState\n // Note: checkAndSetImportJobState cannot be used to set queueItems errorStatus & errorMessage (expect in case of a TimeOut)\n async function checkAndSetImportJobState({correlationId, operation, importJobState}) {\n // checkTimeOut returns true, if queueItem is fresher than 1 minute and it's state is not ABORT/ERROR/DONE\n // otherwise it sets queueItem and importJobStates to state ABORT and errorStatue and errorMessage (408, 'Timeout')\n logger.debug(`${correlationId}, ${importJobState}, ${operation}`);\n const timeOut = await checkTimeOut({correlationId, operation, importJobState});\n if (timeOut) {\n return setImportJobState({correlationId, operation, importJobState});\n }\n return false;\n }\n\n async function query(params, showParams = {}) {\n logger.debug(`Querying: ${JSON.stringify(params)}, ${JSON.stringify(showParams)}`);\n debug(`Querying: ${JSON.stringify(params)}, ${JSON.stringify(showParams)}`);\n const {limit = 1000, skip = 0, ...rest} = params;\n\n const result = await operator.find(rest, {projection: createProjection(showParams)})\n .limit(parseInt(limit, 10))\n .skip(parseInt(skip, 10))\n .toArray();\n logger.debug(`Query result: ${result.length > 0 ? 'Found!' : 'Not found!'}`);\n logger.silly(`${JSON.stringify(result)}`);\n debug(`${JSON.stringify(result)}`);\n return result;\n }\n\n // We default to NOT showing operations, operationSettings, recordLoadParams and importJobState when a queueItem is queried\n // These can all be shown with showAll\n // Single fields can be shown by showOperation, showOperationSettings, showRecordLoadParams and showImportJobState\n\n function createProjection(showParams = {}) {\n logger.silly(`Creating projection for query: ${JSON.stringify(showParams)}`);\n debug(`Creating projection for query: ${JSON.stringify(showParams)}`);\n const {showAll = 0, showOperations = 0, showOperationSettings = 0, showRecordLoadParams = 0, showImportJobState = 0} = showParams;\n const combinedShowParams = {\n showAll,\n showOperations,\n showOperationSettings,\n showRecordLoadParams,\n showImportJobState\n };\n logger.debug(`showAll: ${showAll}, showOperations: ${showOperations}, showOperationSettings: ${showOperationSettings}, showRecordLoadParams: ${showRecordLoadParams}, showImportJobState: ${showImportJobState}`);\n debug(`showAll: ${showAll}, showOperations: ${showOperations}, showOperationSettings: ${showOperationSettings}, showRecordLoadParams: ${showRecordLoadParams}, showImportJobState: ${showImportJobState}`);\n debug(`${JSON.stringify(combinedShowParams)}`);\n\n // NOTE: parseBoolean parses any non-empty, non-\"false\" string as true!\n if (parseBoolean(showAll)) {\n //if (showAll === true || showAll === '1' || showAll === 1) {\n debug(`ShowAll: ${showAll}`);\n return {\n _id: 0\n };\n }\n\n const showParamToField = {\n 'showOperations': 'operations',\n 'showOperationSettings': 'operationSettings',\n 'showRecordLoadParams': 'recordLoadParams',\n 'showImportJobState': 'importJobState'\n };\n\n const result = Object.keys(combinedShowParams)\n //.filter(param => param !== 'showAll' && combinedShowParams[param] !== true && combinedShowParams[param] !== 1 && combinedShowParams[param] !== '1')\n .filter(param => param !== 'showAll' && !parseBoolean(combinedShowParams[param]))\n .filter(param => showParamToField[param])\n .map((param) => showParamToField[param]);\n logger.silly(`We want to exclude from projection: ${JSON.stringify(result)}`);\n debug(`We want to exclude from projection: ${JSON.stringify(result)}`);\n\n\n const excludeObject = Object.fromEntries(result.map(param => [param, 0]));\n logger.silly(`We want to exclude from projection: ${JSON.stringify(excludeObject)}`);\n debug(`We want to exclude from projection: ${JSON.stringify(excludeObject)}`);\n\n\n return {\n _id: 0,\n ...excludeObject\n };\n }\n\n // Note: edited in v4.2.4 to return queueItem *after* possible timout has aborted it\n // Note: previous versions returned queueItem *before* its state/queueItemState is timeOuted\n async function queryById({correlationId, checkModTime = false}) {\n if (checkModTime) {\n debug(`queryById checkModTime: ${checkModTime}`);\n await checkTimeOut({correlationId});\n return queryById({correlationId});\n }\n const result = await operator.findOne({correlationId}, {projection: {_id: 0}});\n return result;\n }\n\n // Check that if the item has waited too long\n // If the last modification time for the queueItem is older than 1 minute\n // set state to ABORT and return false, otherwise return true\n // If the state is already ABORT or ERROR or DONE return false\n\n async function checkTimeOut({correlationId}) {\n const {modificationTime, queueItemState: oldState, importJobState} = await operator.findOne({correlationId});\n //debug(`${modificationTime} - oldState: ${oldState}`);\n // should we check for DONE too?\n if ([QUEUE_ITEM_STATE.ABORT, QUEUE_ITEM_STATE.ERROR, QUEUE_ITEM_STATE.DONE].includes(oldState)) {\n logger.silly(`${correlationId} has already state: ${oldState}`);\n return false;\n }\n\n const timeoutTime = moment(modificationTime).add(1, 'm');\n logger.silly(`${correlationId} timeOut @ ${timeoutTime}`);\n\n if (timeoutTime.isBefore()) {\n const finalImportJobStates = [IMPORT_JOB_STATE.ABORT, IMPORT_JOB_STATE.DONE, IMPORT_JOB_STATE.EMPTY, IMPORT_JOB_STATE.ERROR];\n if (!finalImportJobStates.includes(importJobState.CREATE)) {\n await setImportJobState({correlationId, operation: 'CREATE', importJobState: IMPORT_JOB_STATE.ABORT});\n }\n\n if (!finalImportJobStates.includes(importJobState.UPDATE)) {\n await setImportJobState({correlationId, operation: 'UPDATE', importJobState: IMPORT_JOB_STATE.ABORT});\n }\n\n if (!finalImportJobStates.includes(importJobState.FIX)) {\n await setImportJobState({correlationId, operation: 'FIX', importJobState: IMPORT_JOB_STATE.ABORT});\n }\n\n await setState({correlationId, state: QUEUE_ITEM_STATE.ABORT, errorStatus: httpStatus.REQUEST_TIMEOUT, errorMessage: `Timeout in ${oldState}`});\n\n return false;\n }\n\n return true;\n }\n\n async function remove(params) {\n logger.silly(`${JSON.stringify(params)}`);\n logger.info(`Removing from Mongo (${collection}) id: ${params.correlationId}`);\n debug(`${JSON.stringify(params)}`);\n debug(`Removing from Mongo (${collection}) id: ${params.correlationId}`);\n const clean = sanitize(params.correlationId);\n logger.silly(`mongo/remove: clean: ${JSON.stringify(clean)}`);\n debug(`mongo/remove: clean: ${JSON.stringify(clean)}`);\n\n\n try {\n //const metadataResult = await getFileMetadata({filename: clean});\n //logger.debug(`mongo/remove: metadataResult: ${JSON.stringify(metadataResult)}`);\n const noContent = await removeContent({correlationId: params.correlationId});\n debug(`noContent (removeContent result): ${JSON.stringify(noContent)}`);\n if (noContent) {\n const deleteResult = await operator.deleteOne({correlationId: clean});\n debug(`deleteResult: ${JSON.stringify(deleteResult)}`);\n return true;\n }\n } catch (err) {\n // TEST-COVERAGE: this error is not (currently) tested\n if (err instanceof MongoDriverError) {\n debug(err);\n if (err.message.indexOf('File not found for id') !== -1) {\n logger.silly(`mongo/remove: File not found, removing queueItem ${JSON.stringify(clean)} from ${collection}`);\n debug(`mongo/remove: File not found, removing queueItem ${JSON.stringify(clean)} from ${collection}`);\n await operator.deleteOne({correlationId: clean});\n return true;\n }\n logger.error(err.message);\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n }\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n }\n }\n\n async function readContent(correlationId) {\n logger.info(`Reading content from mongo for id: ${correlationId} in ${collection}`);\n debug(`Reading content from mongo for id: ${correlationId} in ${collection}`);\n\n const clean = sanitize(correlationId);\n debug(clean);\n\n const result = await operator.findOne({correlationId: clean}); // njsscan-ignore: node_nosqli_injection\n debug(result);\n if (result) {\n const {operationSettings} = result;\n if (operationSettings.prio === false && operationSettings.noStream === false) {\n return {\n contentType: result.contentType,\n readStream: gridFSBucket.openDownloadStreamByName(clean)\n };\n }\n logger.debug(`OperationSettings for ${correlationId}: ${JSON.stringify(operationSettings)}`);\n debug(`OperationSettings for ${correlationId}: ${JSON.stringify(operationSettings)}`);\n debug(`Content is only available for streamBulk jobs. ${correlationId} is not a streamBulk job.`);\n throw new ApiError(httpStatus.BAD_REQUEST, {message: `Content is only available for streamBulk jobs. ${correlationId} is not a streamBulk job.`});\n }\n\n throw new ApiError(httpStatus.NOT_FOUND, {message: `Job ${correlationId} not found.`});\n }\n\n async function removeContent(params) {\n logger.info(`Removing content from mongo for id: ${params.correlationId} in ${collection}`);\n debug(`Removing content from mongo for id: ${params.correlationId} in ${collection}`);\n const clean = sanitize(params.correlationId);\n\n const result = await gridFSBucket.find({filename: clean}).toArray();\n debug(`mongo/removeContent: find bucket result ${JSON.stringify(result)}`);\n\n\n if (result.length > 0) {\n // Note: we assume and handle just one file for filename!\n const [firstBucketResult] = result;\n const fileId = firstBucketResult._id;\n\n await gridFSBucket.delete(fileId);\n return true;\n }\n\n logger.debug(`No files for ${params.correlationId} to delete`);\n return true;\n }\n\n function getOne({queueItemState, importJobState = undefined}) {\n //params example: \"queueItemState\": \"IMPORTER.IN_QUEUE\", \"importJobState\":{\"importJobState.UPDATE\": \"IN_QUEUE\"}\n logger.silly(`queueItemState: ${queueItemState}, importJobState: ${JSON.stringify(importJobState)}`);\n //debug(`queueItemState: ${queueItemState}, importJobState: ${JSON.stringify(importJobState)}`);\n const cleanQueueItemState = queueItemState ? {queueItemState: sanitize(queueItemState)} : undefined;\n const options = {projection: {_id: 0}};\n\n try {\n\n // Just queueItemState\n if (queueItemState && importJobState === undefined) {\n logger.silly(`Checking DB ${collection} for just ${JSON.stringify(cleanQueueItemState.queueItemState)}`);\n //debug(`Checking DB ${collection} for just ${JSON.stringify(cleanQueueItemState.queueItemState)}`);\n return operator.findOne({...cleanQueueItemState}, options);\n }\n\n // importJobState\n if (importJobState && queueItemState === undefined) {\n logger.silly(`Checking DB ${collection} for ${JSON.stringify(importJobState)}`);\n //debug(`Checking DB ${collection} for ${JSON.stringify(importJobState)}`);\n return operator.findOne({...importJobState}, options);\n }\n\n // importJobState and queueItemState\n if (importJobState && queueItemState) {\n logger.silly(`Checking DB ${collection} for ${queueItemState} and ${JSON.stringify(importJobState)}`);\n //debug(`Checking DB ${collection} for ${queueItemState} and ${JSON.stringify(importJobState)}`);\n return operator.findOne({...cleanQueueItemState, ...importJobState}, options);\n }\n\n logger.debug(`getOne not working!`);\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, 'Missing parameters, cannot get queueItem');\n\n } catch (error) {\n const errorMessage = error.payload || error.message || '';\n logError(error);\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);\n }\n }\n\n function getStream(correlationId) {\n logger.info(`Forming stream from mongo for ${correlationId} in ${collection}`);\n const clean = sanitize(correlationId);\n try {\n // Return content stream\n return gridFSBucket.openDownloadStreamByName(clean);\n } catch (error) {\n // TEST-COVERAGE: this error is not (currently) tested\n const errorMessage = error.payload || error.message || '';\n logError(error);\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);\n }\n }\n\n async function pushIds({correlationId, handledIds, rejectedIds}) {\n logger.verbose(`Push ids (${handledIds.length}) and rejectedIds (${rejectedIds.length}) ${correlationId} to ${collection}`);\n logger.debug(`ids (${handledIds.length}): ${JSON.stringify(handledIds)}, rejectedIds ${rejectedIds.length}: ${JSON.stringify(rejectedIds)}`);\n const clean = sanitize(correlationId);\n await operator.updateOne({\n correlationId: clean\n }, {\n $set: {\n modificationTime: moment().toDate()\n },\n $push: {\n handledIds: {$each: handledIds},\n rejectedIds: {$each: rejectedIds}\n }\n });\n }\n\n async function pushMessages({correlationId, messages, messageField = 'messages'}) {\n logger.verbose(`Push messages (${messages.length}) to ${messageField} ${correlationId} to ${collection}`);\n logger.silly(`Messages (${messages.length}): ${JSON.stringify(messages)}}`);\n const clean = sanitize(correlationId);\n const cleanMessageField = sanitize(messageField);\n debug(`${messageField} -> ${cleanMessageField}`);\n await operator.updateOne({\n correlationId: clean\n }, {\n $set: {\n modificationTime: moment().toDate()\n },\n $push: {\n [cleanMessageField]: {$each: messages}\n }\n });\n }\n\n function setState({correlationId, state, errorMessage = undefined, errorStatus = undefined}) {\n const errorString = errorMessage || errorStatus ? `, Error message: '${JSON.stringify(errorMessage) || ''}', Error status: '${errorStatus || ''}'` : '';\n logger.info(`Setting queue-item state ${state} for ${correlationId}${errorString} to ${collection}`);\n\n const stateInQueueItemStates = Object.values(QUEUE_ITEM_STATE).indexOf(state) > -1;\n const stateInQueueItemStatesValidator = Object.values(QUEUE_ITEM_STATE.VALIDATOR).indexOf(state) > -1;\n const stateInQueueItemStatesImporter = Object.values(QUEUE_ITEM_STATE.IMPORTER).indexOf(state) > -1;\n if (!stateInQueueItemStates && !stateInQueueItemStatesValidator && !stateInQueueItemStatesImporter) {\n debug(`Trying to set invalid state ${state}`);\n throw new ApiError(httpStatus.BAD_REQUEST, 'Trying to set invalid state');\n }\n\n const clean = sanitize(correlationId);\n const updateValues = {\n queueItemState: state,\n modificationTime: moment().toDate(),\n errorMessage,\n errorStatus\n };\n\n // Do not update value that are undefined\n Object.keys(updateValues).forEach(key => updateValues[key] === undefined && delete updateValues[key]);\n\n return operator\n .findOneAndUpdate(\n {correlationId: clean},\n {$set: updateValues},\n {projection: {_id: 0}, returnDocument: 'after'}\n );\n }\n\n function setImportJobState({correlationId, operation, importJobState}) {\n\n // should this also get importJobState as previously created object? {}\n logger.info(`Setting queue-item importJobState: {${operation}: ${importJobState}} for ${correlationId}`);\n debug(`Setting queue-item importJobState: {${operation}: ${importJobState}} for ${correlationId}`);\n const cleanCorrelationId = sanitize(correlationId);\n const cleanImportJobState = sanitize(importJobState);\n\n if (!(operation in OPERATIONS)) {\n debug(`Error: ${operation} is not in ${JSON.stringify(OPERATIONS)}`);\n throw new ApiError(httpStatus.BAD_REQUEST, 'Invalid operation for import job state');\n }\n\n if (!(cleanImportJobState in IMPORT_JOB_STATE)) {\n debug(`Error: ${cleanImportJobState} is not in ${JSON.stringify(IMPORT_JOB_STATE)}`);\n throw new ApiError(httpStatus.BAD_REQUEST, 'Invalid import job state');\n }\n\n const newJobState = getNewJobState(operation, cleanImportJobState);\n\n\n function getNewJobState(operation, cleanImportJobState) {\n if (operation === OPERATIONS.CREATE) {\n return {'importJobState.CREATE': cleanImportJobState};\n }\n if (operation === OPERATIONS.UPDATE) {\n return {'importJobState.UPDATE': cleanImportJobState};\n }\n if (operation === OPERATIONS.FIX) {\n return {'importJobState.FIX': cleanImportJobState};\n }\n // istanbul ignore next\n // Ignoring for coverage - we error this already\n throw new ApiError('400', 'Invalid operation for import job state');\n }\n\n return operator.findOneAndUpdate({\n correlationId: cleanCorrelationId\n }, {\n $set: {\n ...newJobState,\n modificationTime: moment().toDate()\n }\n }, {projection: {_id: 0}, returnDocument: 'after'});\n }\n\n async function setOperation({correlationId, operation}) {\n const newOperation = operation;\n const {operation: oldOperation} = await operator.findOne({correlationId});\n logger.info(`Setting queue-item operation from ${oldOperation} to ${newOperation} for ${correlationId} to ${collection}`);\n const cleanCorrelationId = sanitize(correlationId);\n const cleanNewOperation = sanitize(newOperation);\n\n const result = await operator.findOneAndUpdate({\n correlationId: cleanCorrelationId\n }, {\n $set: {\n operation: cleanNewOperation,\n modificationTime: moment().toDate()\n }\n }, {projection: {_id: 0}, returnDocument: 'after'});\n debugDev(`${JSON.stringify(result)}`);\n return result.ok || result;\n // logger.debug(JSON.stringify(result));\n }\n\n async function setOperations({correlationId, addOperation, removeOperation = undefined}) {\n const cleanCorrelationId = sanitize(correlationId);\n\n const queueItem = await operator.findOne({correlationId: cleanCorrelationId});\n logger.silly(`We found a queueItem: ${JSON.stringify(queueItem)}`);\n\n const oldOperations = queueItem.operations;\n\n logger.info(`Setting queue-item operations from ${JSON.stringify(oldOperations)} by adding ${addOperation} and removing ${removeOperation} for ${correlationId} to ${collection}`);\n\n const operationsAfterRemove = oldOperations.filter(operation => operation !== removeOperation);\n const operationsAfterRemoveAndAdd = operationsAfterRemove.includes(addOperation) ? operationsAfterRemove : [...operationsAfterRemove, addOperation];\n\n logger.silly(`operationsAfterRemove: ${operationsAfterRemove}`);\n logger.silly(`operationAfterRemoveAndAdd: ${operationsAfterRemoveAndAdd}`);\n\n const result = await operator.findOneAndUpdate({\n correlationId: cleanCorrelationId\n }, {\n $set: {\n operations: operationsAfterRemoveAndAdd,\n modificationTime: moment().toDate()\n }\n }, {projection: {_id: 0}, returnDocument: 'after'});\n debugDev(`${JSON.stringify(result)}`);\n return result.ok || result;\n // logger.debug(JSON.stringify(result));\n }\n\n async function addBlobSize({correlationId}) {\n const cleanCorrelationId = sanitize(correlationId);\n\n const result = await operator.findOneAndUpdate({\n correlationId: cleanCorrelationId,\n queueItemState: QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS\n }, {\n $inc: {\n blobSize: 1\n },\n $set: {\n modificationTime: moment().toDate()\n }\n }, {projection: {_id: 0}, returnDocument: 'after'});\n\n logger.verbose(`AddBlobSizeResult in mongo: ${JSON.stringify(result)}`);\n debugDev(`${JSON.stringify(result)}`);\n return result;\n }\n\n async function setBlobSize({correlationId, blobSize}) {\n const cleanCorrelationId = sanitize(correlationId);\n const cleanBlobSize = sanitize(blobSize);\n\n const result = await operator.findOneAndUpdate({\n correlationId: cleanCorrelationId\n }, {\n $set: {\n blobSize: cleanBlobSize,\n modificationTime: moment().toDate()\n }\n }, {projection: {_id: 0}, returnDocument: 'after'});\n\n logger.verbose(`SetBlobSizeResult in mongo: ${JSON.stringify(result)}`);\n debugDev(`${JSON.stringify(result)}`);\n return result;\n }\n\n}\n"],
5
- "mappings": "AAEA,SAAQ,aAAa,cAAc,wBAAuB;AAC1D,OAAO,YAAY;AACnB,OAAO,gBAAgB;AACvB,OAAO,cAAc;AACrB,OAAO,uBAAuB;AAC9B,SAAQ,oBAAmB;AAC3B,SAAQ,SAAS,UAAU,oBAAmB;AAC9C,SAAQ,kBAAkB,kBAAkB,kBAAiB;AAC7D,SAAQ,gBAAe;AAiCvB,8BAA+B,WAAW,YAAY,KAAK,YAAY;AACrE,QAAM,SAAS,aAAa;AAC5B,QAAM,QAAQ,kBAAkB,0CAA0C;AAC1E,QAAM,WAAW,MAAM,OAAO,MAAM;AAGpC,QAAM,SAAS,MAAM,YAAY,QAAQ,SAAS;AAClD,QAAM,eAAe,OAAO,GAAG,EAAE;AACjC,QAAM,eAAe,IAAI,aAAa,cAAc,EAAC,YAAY,WAAU,CAAC;AAC5E,QAAM,WAAW,aAAa,WAAW,UAAU;AACnD,WAAS,UAAU,SAAS,SAAS,EAAE,iBAAiB,UAAU,EAAE;AAEpE,SAAO,EAAC,YAAY,YAAY,kBAAkB,2BAA2B,OAAO,kBAAkB,WAAW,cAAc,QAAQ,aAAa,eAAe,QAAQ,WAAW,UAAU,mBAAmB,SAAS,cAAc,cAAc,eAAe,aAAa,YAAW;AAE/R,iBAAe,WAAW,EAAC,eAAe,WAAW,cAAc,WAAW,kBAAiB,GAAG;AAChG,aAAS,YAAY;AACrB,UAAM,OAAO,OAAO,EAAE,OAAO;AAC7B,UAAM,eAAe;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY,CAAC,SAAS;AAAA,MACtB;AAAA,MACA,mBAAmB;AAAA,QACjB,MAAM;AAAA,QACN,GAAG;AAAA,MACL;AAAA,MACA,gBAAgB,iBAAiB,UAAU;AAAA,MAC3C,UAAU;AAAA,MACV,gBAAgB;AAAA,QACd,QAAQ,iBAAiB;AAAA,QACzB,QAAQ,iBAAiB;AAAA,QACzB,KAAK,iBAAiB;AAAA,MACxB;AAAA,MACA,SAAS,CAAC;AAAA,MACV,cAAc;AAAA,MACd,aAAa;AAAA,MACb,cAAc;AAAA,MACd,kBAAkB;AAAA,IACpB;AACA,QAAI;AACF,eAAS,cAAc,KAAK,UAAU,YAAY,CAAC,OAAO,UAAU,GAAG;AACvE,YAAM,SAAS,MAAM,SAAS,UAAU,YAAY;AACpD,eAAS,KAAK,UAAU,MAAM,CAAC;AAC/B,UAAI,OAAO,cAAc;AACvB,eAAO,KAAK,2BAA2B,SAAS,IAAI,aAAa,qBAAqB,UAAU,EAAE;AAClG;AAAA,MACF;AAEA,YAAM,IAAI,SAAS,WAAW,qBAAqB;AAAA,IACrD,SAAS,OAAO;AAEd,YAAM,eAAe,MAAM,WAAW,MAAM,WAAW;AACvD,eAAS,KAAK;AACd,YAAM,IAAI,SAAS,WAAW,uBAAuB,kBAAkB,YAAY,EAAE;AAAA,IACvF;AAAA,EACF;AAEA,iBAAe,WAAW,EAAC,eAAe,WAAW,cAAc,WAAW,aAAa,kBAAkB,QAAQ,kBAAiB,GAAG;AACvI,UAAM,OAAO,OAAO,EAAE,OAAO;AAC7B,UAAM,eAAe;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY,CAAC,SAAS;AAAA,MACtB,mBAAmB;AAAA,QACjB,MAAM;AAAA,QACN,GAAG;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB,SAAS,iBAAiB,UAAU,YAAY,iBAAiB,UAAU;AAAA,MAC3F,UAAU;AAAA,MACV,gBAAgB;AAAA,QACd,QAAQ,iBAAiB;AAAA,QACzB,QAAQ,iBAAiB;AAAA,QACzB,KAAK,iBAAiB;AAAA,MACxB;AAAA,MACA,SAAS,CAAC;AAAA,MACV,cAAc;AAAA,MACd,aAAa;AAAA,MACb,cAAc;AAAA,MACd,kBAAkB;AAAA,IACpB;AAEA,QAAI,QAAQ;AACV,YAAM,qBAAqB;AAC3B,UAAI;AACF,cAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,gBAAM,eAAe,aAAa,iBAAiB,aAAa;AAEhE,iBACG,GAAG,SAAS,MAAM,EAClB,GAAG,QAAQ,WAAS,aAAa,MAAM,KAAK,CAAC,EAC7C,GAAG,OAAO,MAAM,aAAa,IAAI,QAAW,QAAW,MAAM;AAC5D,oBAAQ,aAAa;AAAA,UACvB,CAAC,CAAC;AAAA,QACN,CAAC;AACD,eAAO,KAAK,2BAA2B,SAAS,IAAI,aAAa,qBAAqB,UAAU,GAAG;AACnG,cAAM,2BAA2B,SAAS,IAAI,aAAa,qBAAqB,UAAU,GAAG;AAC7F,eAAO,SAAS,UAAU,YAAY;AAAA,MACxC,SAAS,OAAO;AACd,cAAM,eAAe,MAAM,WAAW,MAAM,WAAW;AACvD,iBAAS,KAAK;AACd,cAAM,IAAI,SAAS,WAAW,uBAAuB,kBAAkB,YAAY,EAAE;AAAA,MACvF;AAAA,IACF;AAEA,WAAO,MAAM,WAAW;AACxB,UAAM,uBAAuB;AAC7B,QAAI,CAAC,QAAQ;AACX,UAAI;AACF,cAAM,SAAS,MAAM,SAAS,UAAU,YAAY;AACpD,YAAI,OAAO,cAAc;AACvB,iBAAO,KAAK,oCAAoC,SAAS,IAAI,aAAa,qBAAqB,UAAU,EAAE;AAC3G,gBAAM,oCAAoC,SAAS,IAAI,aAAa,qBAAqB,UAAU,EAAE;AACrG,iBAAO,EAAC,eAAe,gBAAgB,iBAAiB,UAAU,oBAAmB;AAAA,QACvF;AAEA,cAAM,IAAI,SAAS,WAAW,qBAAqB;AAAA,MACrD,SAAS,OAAO;AAEd,cAAM,eAAe,MAAM,WAAW,MAAM,WAAW;AACvD,iBAAS,KAAK;AACd,cAAM,IAAI,SAAS,WAAW,uBAAuB,kBAAkB,YAAY,EAAE;AAAA,MACvF;AAAA,IACF;AAAA,EACF;AAIA,iBAAe,iBAAiB,EAAC,eAAe,OAAO,eAAe,QAAW,cAAc,OAAS,GAAG;AAGzG,UAAM,UAAU,MAAM,aAAa,EAAC,cAAa,CAAC;AAClD,QAAI,SAAS;AACX,aAAO,SAAS,EAAC,eAAe,OAAO,cAAc,YAAW,CAAC;AAAA,IACnE;AACA,WAAO;AAAA,EACT;AAIA,iBAAe,0BAA0B,EAAC,eAAe,WAAW,eAAc,GAAG;AAGnF,WAAO,MAAM,GAAG,aAAa,KAAK,cAAc,KAAK,SAAS,EAAE;AAChE,UAAM,UAAU,MAAM,aAAa,EAAC,eAAe,WAAW,eAAc,CAAC;AAC7E,QAAI,SAAS;AACX,aAAO,kBAAkB,EAAC,eAAe,WAAW,eAAc,CAAC;AAAA,IACrE;AACA,WAAO;AAAA,EACT;AAEA,iBAAe,MAAM,QAAQ,aAAa,CAAC,GAAG;AAC5C,WAAO,MAAM,aAAa,KAAK,UAAU,MAAM,CAAC,KAAK,KAAK,UAAU,UAAU,CAAC,EAAE;AACjF,UAAM,aAAa,KAAK,UAAU,MAAM,CAAC,KAAK,KAAK,UAAU,UAAU,CAAC,EAAE;AAC1E,UAAM,EAAC,QAAQ,KAAM,OAAO,GAAG,GAAG,KAAI,IAAI;AAE1C,UAAM,SAAS,MAAM,SAAS,KAAK,MAAM,EAAC,YAAY,iBAAiB,UAAU,EAAC,CAAC,EAChF,MAAM,SAAS,OAAO,EAAE,CAAC,EACzB,KAAK,SAAS,MAAM,EAAE,CAAC,EACvB,QAAQ;AACX,WAAO,MAAM,iBAAiB,OAAO,SAAS,IAAI,WAAW,YAAY,EAAE;AAC3E,WAAO,MAAM,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACxC,UAAM,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACjC,WAAO;AAAA,EACT;AAMA,WAAS,iBAAiB,aAAa,CAAC,GAAG;AACzC,WAAO,MAAM,kCAAkC,KAAK,UAAU,UAAU,CAAC,EAAE;AAC3E,UAAM,kCAAkC,KAAK,UAAU,UAAU,CAAC,EAAE;AACpE,UAAM,EAAC,UAAU,GAAG,iBAAiB,GAAG,wBAAwB,GAAG,uBAAuB,GAAG,qBAAqB,EAAC,IAAI;AACvH,UAAM,qBAAqB;AAAA,MACzB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AACA,WAAO,MAAM,YAAY,OAAO,qBAAqB,cAAc,4BAA4B,qBAAqB,2BAA2B,oBAAoB,yBAAyB,kBAAkB,EAAE;AAChN,UAAM,YAAY,OAAO,qBAAqB,cAAc,4BAA4B,qBAAqB,2BAA2B,oBAAoB,yBAAyB,kBAAkB,EAAE;AACzM,UAAM,GAAG,KAAK,UAAU,kBAAkB,CAAC,EAAE;AAG7C,QAAI,aAAa,OAAO,GAAG;AAEzB,YAAM,YAAY,OAAO,EAAE;AAC3B,aAAO;AAAA,QACL,KAAK;AAAA,MACP;AAAA,IACF;AAEA,UAAM,mBAAmB;AAAA,MACvB,kBAAkB;AAAA,MAClB,yBAAyB;AAAA,MACzB,wBAAwB;AAAA,MACxB,sBAAsB;AAAA,IACxB;AAEA,UAAM,SAAS,OAAO,KAAK,kBAAkB,EAE1C,OAAO,WAAS,UAAU,aAAa,CAAC,aAAa,mBAAmB,KAAK,CAAC,CAAC,EAC/E,OAAO,WAAS,iBAAiB,KAAK,CAAC,EACvC,IAAI,CAAC,UAAU,iBAAiB,KAAK,CAAC;AACzC,WAAO,MAAM,uCAAuC,KAAK,UAAU,MAAM,CAAC,EAAE;AAC5E,UAAM,uCAAuC,KAAK,UAAU,MAAM,CAAC,EAAE;AAGrE,UAAM,gBAAgB,OAAO,YAAY,OAAO,IAAI,WAAS,CAAC,OAAO,CAAC,CAAC,CAAC;AACxE,WAAO,MAAM,uCAAuC,KAAK,UAAU,aAAa,CAAC,EAAE;AACnF,UAAM,uCAAuC,KAAK,UAAU,aAAa,CAAC,EAAE;AAG5E,WAAO;AAAA,MACL,KAAK;AAAA,MACL,GAAG;AAAA,IACL;AAAA,EACF;AAIA,iBAAe,UAAU,EAAC,eAAe,eAAe,MAAK,GAAG;AAC9D,QAAI,cAAc;AAChB,YAAM,2BAA2B,YAAY,EAAE;AAC/C,YAAM,aAAa,EAAC,cAAa,CAAC;AAClC,aAAO,UAAU,EAAC,cAAa,CAAC;AAAA,IAClC;AACA,UAAM,SAAS,MAAM,SAAS,QAAQ,EAAC,cAAa,GAAG,EAAC,YAAY,EAAC,KAAK,EAAC,EAAC,CAAC;AAC7E,WAAO;AAAA,EACT;AAOA,iBAAe,aAAa,EAAC,cAAa,GAAG;AAC3C,UAAM,EAAC,kBAAkB,gBAAgB,UAAU,eAAc,IAAI,MAAM,SAAS,QAAQ,EAAC,cAAa,CAAC;AAG3G,QAAI,CAAC,iBAAiB,OAAO,iBAAiB,OAAO,iBAAiB,IAAI,EAAE,SAAS,QAAQ,GAAG;AAC9F,aAAO,MAAM,GAAG,aAAa,uBAAuB,QAAQ,EAAE;AAC9D,aAAO;AAAA,IACT;AAEA,UAAM,cAAc,OAAO,gBAAgB,EAAE,IAAI,GAAG,GAAG;AACvD,WAAO,MAAM,GAAG,aAAa,cAAc,WAAW,EAAE;AAExD,QAAI,YAAY,SAAS,GAAG;AAC1B,YAAM,uBAAuB,CAAC,iBAAiB,OAAO,iBAAiB,MAAM,iBAAiB,OAAO,iBAAiB,KAAK;AAC3H,UAAI,CAAC,qBAAqB,SAAS,eAAe,MAAM,GAAG;AACzD,cAAM,kBAAkB,EAAC,eAAe,WAAW,UAAU,gBAAgB,iBAAiB,MAAK,CAAC;AAAA,MACtG;AAEA,UAAI,CAAC,qBAAqB,SAAS,eAAe,MAAM,GAAG;AACzD,cAAM,kBAAkB,EAAC,eAAe,WAAW,UAAU,gBAAgB,iBAAiB,MAAK,CAAC;AAAA,MACtG;AAEA,UAAI,CAAC,qBAAqB,SAAS,eAAe,GAAG,GAAG;AACtD,cAAM,kBAAkB,EAAC,eAAe,WAAW,OAAO,gBAAgB,iBAAiB,MAAK,CAAC;AAAA,MACnG;AAEA,YAAM,SAAS,EAAC,eAAe,OAAO,iBAAiB,OAAO,aAAa,WAAW,iBAAiB,cAAc,cAAc,QAAQ,GAAE,CAAC;AAE9I,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,EACT;AAEA,iBAAe,OAAO,QAAQ;AAC5B,WAAO,MAAM,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACxC,WAAO,KAAK,wBAAwB,UAAU,SAAS,OAAO,aAAa,EAAE;AAC7E,UAAM,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACjC,UAAM,wBAAwB,UAAU,SAAS,OAAO,aAAa,EAAE;AACvE,UAAM,QAAQ,SAAS,OAAO,aAAa;AAC3C,WAAO,MAAM,wBAAwB,KAAK,UAAU,KAAK,CAAC,EAAE;AAC5D,UAAM,wBAAwB,KAAK,UAAU,KAAK,CAAC,EAAE;AAGrD,QAAI;AAGF,YAAM,YAAY,MAAM,cAAc,EAAC,eAAe,OAAO,cAAa,CAAC;AAC3E,YAAM,qCAAqC,KAAK,UAAU,SAAS,CAAC,EAAE;AACtE,UAAI,WAAW;AACb,cAAM,eAAe,MAAM,SAAS,UAAU,EAAC,eAAe,MAAK,CAAC;AACpE,cAAM,iBAAiB,KAAK,UAAU,YAAY,CAAC,EAAE;AACrD,eAAO;AAAA,MACT;AAAA,IACF,SAAS,KAAK;AAEZ,UAAI,eAAe,kBAAkB;AACnC,cAAM,GAAG;AACT,YAAI,IAAI,QAAQ,QAAQ,uBAAuB,MAAM,IAAI;AACvD,iBAAO,MAAM,oDAAoD,KAAK,UAAU,KAAK,CAAC,SAAS,UAAU,EAAE;AAC3G,gBAAM,oDAAoD,KAAK,UAAU,KAAK,CAAC,SAAS,UAAU,EAAE;AACpG,gBAAM,SAAS,UAAU,EAAC,eAAe,MAAK,CAAC;AAC/C,iBAAO;AAAA,QACT;AACA,eAAO,MAAM,IAAI,OAAO;AACxB,cAAM,IAAI,SAAS,WAAW,uBAAuB,IAAI,OAAO;AAAA,MAClE;AACA,YAAM,IAAI,SAAS,WAAW,uBAAuB,IAAI,OAAO;AAAA,IAClE;AAAA,EACF;AAEA,iBAAe,YAAY,eAAe;AACxC,WAAO,KAAK,sCAAsC,aAAa,OAAO,UAAU,EAAE;AAClF,UAAM,sCAAsC,aAAa,OAAO,UAAU,EAAE;AAE5E,UAAM,QAAQ,SAAS,aAAa;AACpC,UAAM,KAAK;AAEX,UAAM,SAAS,MAAM,SAAS,QAAQ,EAAC,eAAe,MAAK,CAAC;AAC5D,UAAM,MAAM;AACZ,QAAI,QAAQ;AACV,YAAM,EAAC,kBAAiB,IAAI;AAC5B,UAAI,kBAAkB,SAAS,SAAS,kBAAkB,aAAa,OAAO;AAC5E,eAAO;AAAA,UACL,aAAa,OAAO;AAAA,UACpB,YAAY,aAAa,yBAAyB,KAAK;AAAA,QACzD;AAAA,MACF;AACA,aAAO,MAAM,yBAAyB,aAAa,KAAK,KAAK,UAAU,iBAAiB,CAAC,EAAE;AAC3F,YAAM,yBAAyB,aAAa,KAAK,KAAK,UAAU,iBAAiB,CAAC,EAAE;AACpF,YAAM,kDAAkD,aAAa,2BAA2B;AAChG,YAAM,IAAI,SAAS,WAAW,aAAa,EAAC,SAAS,kDAAkD,aAAa,4BAA2B,CAAC;AAAA,IAClJ;AAEA,UAAM,IAAI,SAAS,WAAW,WAAW,EAAC,SAAS,OAAO,aAAa,cAAa,CAAC;AAAA,EACvF;AAEA,iBAAe,cAAc,QAAQ;AACnC,WAAO,KAAK,uCAAuC,OAAO,aAAa,OAAO,UAAU,EAAE;AAC1F,UAAM,uCAAuC,OAAO,aAAa,OAAO,UAAU,EAAE;AACpF,UAAM,QAAQ,SAAS,OAAO,aAAa;AAE3C,UAAM,SAAS,MAAM,aAAa,KAAK,EAAC,UAAU,MAAK,CAAC,EAAE,QAAQ;AAClE,UAAM,2CAA2C,KAAK,UAAU,MAAM,CAAC,EAAE;AAGzE,QAAI,OAAO,SAAS,GAAG;AAErB,YAAM,CAAC,iBAAiB,IAAI;AAC5B,YAAM,SAAS,kBAAkB;AAEjC,YAAM,aAAa,OAAO,MAAM;AAChC,aAAO;AAAA,IACT;AAEA,WAAO,MAAM,gBAAgB,OAAO,aAAa,YAAY;AAC7D,WAAO;AAAA,EACT;AAEA,WAAS,OAAO,EAAC,gBAAgB,iBAAiB,OAAS,GAAG;AAE5D,WAAO,MAAM,mBAAmB,cAAc,qBAAqB,KAAK,UAAU,cAAc,CAAC,EAAE;AAEnG,UAAM,sBAAsB,iBAAiB,EAAC,gBAAgB,SAAS,cAAc,EAAC,IAAI;AAC1F,UAAM,UAAU,EAAC,YAAY,EAAC,KAAK,EAAC,EAAC;AAErC,QAAI;AAGF,UAAI,kBAAkB,mBAAmB,QAAW;AAClD,eAAO,MAAM,eAAe,UAAU,aAAa,KAAK,UAAU,oBAAoB,cAAc,CAAC,EAAE;AAEvG,eAAO,SAAS,QAAQ,EAAC,GAAG,oBAAmB,GAAG,OAAO;AAAA,MAC3D;AAGA,UAAI,kBAAkB,mBAAmB,QAAW;AAClD,eAAO,MAAM,eAAe,UAAU,QAAQ,KAAK,UAAU,cAAc,CAAC,EAAE;AAE9E,eAAO,SAAS,QAAQ,EAAC,GAAG,eAAc,GAAG,OAAO;AAAA,MACtD;AAGA,UAAI,kBAAkB,gBAAgB;AACpC,eAAO,MAAM,eAAe,UAAU,QAAQ,cAAc,QAAQ,KAAK,UAAU,cAAc,CAAC,EAAE;AAEpG,eAAO,SAAS,QAAQ,EAAC,GAAG,qBAAqB,GAAG,eAAc,GAAG,OAAO;AAAA,MAC9E;AAEA,aAAO,MAAM,qBAAqB;AAClC,YAAM,IAAI,SAAS,WAAW,uBAAuB,0CAA0C;AAAA,IAEjG,SAAS,OAAO;AACd,YAAM,eAAe,MAAM,WAAW,MAAM,WAAW;AACvD,eAAS,KAAK;AACd,YAAM,IAAI,SAAS,WAAW,uBAAuB,kBAAkB,YAAY,EAAE;AAAA,IACvF;AAAA,EACF;AAEA,WAAS,UAAU,eAAe;AAChC,WAAO,KAAK,iCAAiC,aAAa,OAAO,UAAU,EAAE;AAC7E,UAAM,QAAQ,SAAS,aAAa;AACpC,QAAI;AAEF,aAAO,aAAa,yBAAyB,KAAK;AAAA,IACpD,SAAS,OAAO;AAEd,YAAM,eAAe,MAAM,WAAW,MAAM,WAAW;AACvD,eAAS,KAAK;AACd,YAAM,IAAI,SAAS,WAAW,uBAAuB,kBAAkB,YAAY,EAAE;AAAA,IACvF;AAAA,EACF;AAEA,iBAAe,QAAQ,EAAC,eAAe,YAAY,YAAW,GAAG;AAC/D,WAAO,QAAQ,aAAa,WAAW,MAAM,sBAAsB,YAAY,MAAM,KAAK,aAAa,OAAO,UAAU,EAAE;AAC1H,WAAO,MAAM,QAAQ,WAAW,MAAM,MAAM,KAAK,UAAU,UAAU,CAAC,iBAAiB,YAAY,MAAM,KAAK,KAAK,UAAU,WAAW,CAAC,EAAE;AAC3I,UAAM,QAAQ,SAAS,aAAa;AACpC,UAAM,SAAS,UAAU;AAAA,MACvB,eAAe;AAAA,IACjB,GAAG;AAAA,MACD,MAAM;AAAA,QACJ,kBAAkB,OAAO,EAAE,OAAO;AAAA,MACpC;AAAA,MACA,OAAO;AAAA,QACL,YAAY,EAAC,OAAO,WAAU;AAAA,QAC9B,aAAa,EAAC,OAAO,YAAW;AAAA,MAClC;AAAA,IACF,CAAC;AAAA,EACH;AAEA,iBAAe,aAAa,EAAC,eAAe,UAAU,eAAe,WAAU,GAAG;AAChF,WAAO,QAAQ,kBAAkB,SAAS,MAAM,QAAQ,YAAY,IAAI,aAAa,OAAO,UAAU,EAAE;AACxG,WAAO,MAAM,aAAa,SAAS,MAAM,MAAM,KAAK,UAAU,QAAQ,CAAC,GAAG;AAC1E,UAAM,QAAQ,SAAS,aAAa;AACpC,UAAM,oBAAoB,SAAS,YAAY;AAC/C,UAAM,GAAG,YAAY,OAAO,iBAAiB,EAAE;AAC/C,UAAM,SAAS,UAAU;AAAA,MACvB,eAAe;AAAA,IACjB,GAAG;AAAA,MACD,MAAM;AAAA,QACJ,kBAAkB,OAAO,EAAE,OAAO;AAAA,MACpC;AAAA,MACA,OAAO;AAAA,QACL,CAAC,iBAAiB,GAAG,EAAC,OAAO,SAAQ;AAAA,MACvC;AAAA,IACF,CAAC;AAAA,EACH;AAEA,WAAS,SAAS,EAAC,eAAe,OAAO,eAAe,QAAW,cAAc,OAAS,GAAG;AAC3F,UAAM,cAAc,gBAAgB,cAAc,qBAAqB,KAAK,UAAU,YAAY,KAAK,EAAE,qBAAqB,eAAe,EAAE,MAAM;AACrJ,WAAO,KAAK,4BAA4B,KAAK,QAAQ,aAAa,GAAG,WAAW,OAAO,UAAU,EAAE;AAEnG,UAAM,yBAAyB,OAAO,OAAO,gBAAgB,EAAE,QAAQ,KAAK,IAAI;AAChF,UAAM,kCAAkC,OAAO,OAAO,iBAAiB,SAAS,EAAE,QAAQ,KAAK,IAAI;AACnG,UAAM,iCAAiC,OAAO,OAAO,iBAAiB,QAAQ,EAAE,QAAQ,KAAK,IAAI;AACjG,QAAI,CAAC,0BAA0B,CAAC,mCAAmC,CAAC,gCAAgC;AAClG,YAAM,+BAA+B,KAAK,EAAE;AAC5C,YAAM,IAAI,SAAS,WAAW,aAAa,6BAA6B;AAAA,IAC1E;AAEA,UAAM,QAAQ,SAAS,aAAa;AACpC,UAAM,eAAe;AAAA,MACnB,gBAAgB;AAAA,MAChB,kBAAkB,OAAO,EAAE,OAAO;AAAA,MAClC;AAAA,MACA;AAAA,IACF;AAGA,WAAO,KAAK,YAAY,EAAE,QAAQ,SAAO,aAAa,GAAG,MAAM,UAAa,OAAO,aAAa,GAAG,CAAC;AAEpG,WAAO,SACJ;AAAA,MACC,EAAC,eAAe,MAAK;AAAA,MACrB,EAAC,MAAM,aAAY;AAAA,MACnB,EAAC,YAAY,EAAC,KAAK,EAAC,GAAG,gBAAgB,QAAO;AAAA,IAChD;AAAA,EACJ;AAEA,WAAS,kBAAkB,EAAC,eAAe,WAAW,eAAc,GAAG;AAGrE,WAAO,KAAK,uCAAuC,SAAS,KAAK,cAAc,SAAS,aAAa,EAAE;AACvG,UAAM,uCAAuC,SAAS,KAAK,cAAc,SAAS,aAAa,EAAE;AACjG,UAAM,qBAAqB,SAAS,aAAa;AACjD,UAAM,sBAAsB,SAAS,cAAc;AAEnD,QAAI,EAAE,aAAa,aAAa;AAC9B,YAAM,UAAU,SAAS,cAAc,KAAK,UAAU,UAAU,CAAC,EAAE;AACnE,YAAM,IAAI,SAAS,WAAW,aAAa,wCAAwC;AAAA,IACrF;AAEA,QAAI,EAAE,uBAAuB,mBAAmB;AAC9C,YAAM,UAAU,mBAAmB,cAAc,KAAK,UAAU,gBAAgB,CAAC,EAAE;AACnF,YAAM,IAAI,SAAS,WAAW,aAAa,0BAA0B;AAAA,IACvE;AAEA,UAAM,cAAc,eAAe,WAAW,mBAAmB;AAGjE,aAAS,eAAeA,YAAWC,sBAAqB;AACtD,UAAID,eAAc,WAAW,QAAQ;AACnC,eAAO,EAAC,yBAAyBC,qBAAmB;AAAA,MACtD;AACA,UAAID,eAAc,WAAW,QAAQ;AACnC,eAAO,EAAC,yBAAyBC,qBAAmB;AAAA,MACtD;AACA,UAAID,eAAc,WAAW,KAAK;AAChC,eAAO,EAAC,sBAAsBC,qBAAmB;AAAA,MACnD;AAGA,YAAM,IAAI,SAAS,OAAO,wCAAwC;AAAA,IACpE;AAEA,WAAO,SAAS,iBAAiB;AAAA,MAC/B,eAAe;AAAA,IACjB,GAAG;AAAA,MACD,MAAM;AAAA,QACJ,GAAG;AAAA,QACH,kBAAkB,OAAO,EAAE,OAAO;AAAA,MACpC;AAAA,IACF,GAAG,EAAC,YAAY,EAAC,KAAK,EAAC,GAAG,gBAAgB,QAAO,CAAC;AAAA,EACpD;AAEA,iBAAe,aAAa,EAAC,eAAe,UAAS,GAAG;AACtD,UAAM,eAAe;AACrB,UAAM,EAAC,WAAW,aAAY,IAAI,MAAM,SAAS,QAAQ,EAAC,cAAa,CAAC;AACxE,WAAO,KAAK,qCAAqC,YAAY,OAAO,YAAY,QAAQ,aAAa,OAAO,UAAU,EAAE;AACxH,UAAM,qBAAqB,SAAS,aAAa;AACjD,UAAM,oBAAoB,SAAS,YAAY;AAE/C,UAAM,SAAS,MAAM,SAAS,iBAAiB;AAAA,MAC7C,eAAe;AAAA,IACjB,GAAG;AAAA,MACD,MAAM;AAAA,QACJ,WAAW;AAAA,QACX,kBAAkB,OAAO,EAAE,OAAO;AAAA,MACpC;AAAA,IACF,GAAG,EAAC,YAAY,EAAC,KAAK,EAAC,GAAG,gBAAgB,QAAO,CAAC;AAClD,aAAS,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACpC,WAAO,OAAO,MAAM;AAAA,EAEtB;AAEA,iBAAe,cAAc,EAAC,eAAe,cAAc,kBAAkB,OAAS,GAAG;AACvF,UAAM,qBAAqB,SAAS,aAAa;AAEjD,UAAM,YAAY,MAAM,SAAS,QAAQ,EAAC,eAAe,mBAAkB,CAAC;AAC5E,WAAO,MAAM,yBAAyB,KAAK,UAAU,SAAS,CAAC,EAAE;AAEjE,UAAM,gBAAgB,UAAU;AAEhC,WAAO,KAAK,sCAAsC,KAAK,UAAU,aAAa,CAAC,cAAc,YAAY,iBAAiB,eAAe,QAAQ,aAAa,OAAO,UAAU,EAAE;AAEjL,UAAM,wBAAwB,cAAc,OAAO,eAAa,cAAc,eAAe;AAC7F,UAAM,8BAA8B,sBAAsB,SAAS,YAAY,IAAI,wBAAwB,CAAC,GAAG,uBAAuB,YAAY;AAElJ,WAAO,MAAM,0BAA0B,qBAAqB,EAAE;AAC9D,WAAO,MAAM,+BAA+B,2BAA2B,EAAE;AAEzE,UAAM,SAAS,MAAM,SAAS,iBAAiB;AAAA,MAC7C,eAAe;AAAA,IACjB,GAAG;AAAA,MACD,MAAM;AAAA,QACJ,YAAY;AAAA,QACZ,kBAAkB,OAAO,EAAE,OAAO;AAAA,MACpC;AAAA,IACF,GAAG,EAAC,YAAY,EAAC,KAAK,EAAC,GAAG,gBAAgB,QAAO,CAAC;AAClD,aAAS,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACpC,WAAO,OAAO,MAAM;AAAA,EAEtB;AAEA,iBAAe,YAAY,EAAC,cAAa,GAAG;AAC1C,UAAM,qBAAqB,SAAS,aAAa;AAEjD,UAAM,SAAS,MAAM,SAAS,iBAAiB;AAAA,MAC7C,eAAe;AAAA,MACf,gBAAgB,iBAAiB,UAAU;AAAA,IAC7C,GAAG;AAAA,MACD,MAAM;AAAA,QACJ,UAAU;AAAA,MACZ;AAAA,MACA,MAAM;AAAA,QACJ,kBAAkB,OAAO,EAAE,OAAO;AAAA,MACpC;AAAA,IACF,GAAG,EAAC,YAAY,EAAC,KAAK,EAAC,GAAG,gBAAgB,QAAO,CAAC;AAElD,WAAO,QAAQ,+BAA+B,KAAK,UAAU,MAAM,CAAC,EAAE;AACtE,aAAS,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACpC,WAAO;AAAA,EACT;AAEA,iBAAe,YAAY,EAAC,eAAe,SAAQ,GAAG;AACpD,UAAM,qBAAqB,SAAS,aAAa;AACjD,UAAM,gBAAgB,SAAS,QAAQ;AAEvC,UAAM,SAAS,MAAM,SAAS,iBAAiB;AAAA,MAC7C,eAAe;AAAA,IACjB,GAAG;AAAA,MACD,MAAM;AAAA,QACJ,UAAU;AAAA,QACV,kBAAkB,OAAO,EAAE,OAAO;AAAA,MACpC;AAAA,IACF,GAAG,EAAC,YAAY,EAAC,KAAK,EAAC,GAAG,gBAAgB,QAAO,CAAC;AAElD,WAAO,QAAQ,+BAA+B,KAAK,UAAU,MAAM,CAAC,EAAE;AACtE,aAAS,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACpC,WAAO;AAAA,EACT;AAEF;",
6
- "names": ["operation", "cleanImportJobState"]
4
+ "sourcesContent": ["/* eslint-disable max-lines */\n\nimport {MongoClient, GridFSBucket, MongoDriverError} from 'mongodb';\nimport httpStatus from 'http-status';\nimport createDebugLogger from 'debug';\nimport {createLogger, Error as ApiError, parseBoolean} from '@natlibfi/melinda-commons';\nimport {QUEUE_ITEM_STATE, IMPORT_JOB_STATE, OPERATIONS} from './constants.js';\nimport {logError, assertString, assertNumber, assertImportJobStateFilter} from './utils.js';\n\n//import isDeepStrictEqual from 'util';\n\n/* QueueItem:\n{\n \"correlationId\":\"FOO\",\n \"cataloger\":\"xxx0000\",\n \"oCatalogerIn\":\"xxx0000\"\n \"operation\":\"UPDATE\",\n \"operations\":[\"UPDATE\", \"CREATE\"]\n \"contentType\":\"application/json\",\n \"recordLoadParams\": {\n \"pActiveLibrary\": \"XXX00\",\n \"pInputFile\": \"filename.seq\",\n \"pRejectFile\": \"filename.rej\",\n \"pLogFile\": \"filename.syslog\",\n \"pOldNew\": \"NEW\"\n },\n \"queueItemState\":\"PENDING_QUEUING\",\n \"importJobState\": {\n \"CREATE\": \"EMPTY\",\n \"UPDATE\": \"EMPTY\"\n },\n blobSize: 1,\n records: [],\n errorMessage: '',\n errorStatus: '',\n \"creationTime\":\"2020-01-01T00:00:00.000Z\",\n \"modificationTime\":\"2020-01-01T00:00:01.000Z\",\n}\n*/\n\nexport default async function (MONGO_URI, collection, db = 'rest-api', deps = {}) {\n const logger = createLogger();\n // the checkTimeOut \"no progress\" window used to be a hardcoded\n // 1 minute, which is too short for large bulk imports (README benchmark:\n // ~9248 records can take ~10 minutes). Now configurable via the\n // MONGO_TIMEOUT_MINUTES env var (default: 1 minute). Each importer chunk\n // that writes (pushMessages / pushIds / setImportJobState / addBlobSize /\n // setBlobSize / setState ...) refreshes modificationTime, so the timeout\n // only triggers when there is genuinely no progress.\n const timeoutMinutes = Number.parseInt(process.env.MONGO_TIMEOUT_MINUTES, 10) || 1;\n logger.debug(`checkTimeOut window: ${timeoutMinutes} minute(s) (override with MONGO_TIMEOUT_MINUTES)`);\n const debug = createDebugLogger('@natlibfi/melinda-rest-api-commons/mongo');\n const debugDev = debug.extend(':dev');\n\n // Connect to mongo (MONGO), unless operator and gridFSBucket are both\n // injected via deps (used by unit tests to avoid a real connection)\n const stubbed = Boolean(deps.operator) && Boolean(deps.gridFSBucket);\n let gridFSBucket = deps.gridFSBucket;\n let operator = deps.operator;\n if (!stubbed) {\n const client = await MongoClient.connect(MONGO_URI);\n const realDb = client.db(db);\n gridFSBucket = gridFSBucket ?? new GridFSBucket(realDb, {bucketName: collection});\n operator = operator ?? realDb.collection(collection);\n }\n debugDev(`mongo: ${MONGO_URI}, db: ${db}, collection: ${collection}`);\n\n return {createPrio, createBulk, checkAndSetState, checkAndSetImportJobState, query, createProjection, queryById, checkTimeOut, remove, readContent, removeContent, getOne, getStream, setState, setImportJobState, pushIds, pushMessages, setOperation, setOperations, addBlobSize, setBlobSize};\n\n async function createPrio({correlationId, cataloger, oCatalogerIn, operation, operationSettings}) {\n debugDev('createPrio');\n const time = new Date();\n const newQueueItem = {\n correlationId,\n cataloger,\n operation,\n operations: [operation],\n oCatalogerIn,\n operationSettings: {\n prio: true,\n ...operationSettings\n },\n queueItemState: QUEUE_ITEM_STATE.VALIDATOR.PENDING_VALIDATION,\n blobSize: 1,\n importJobState: {\n CREATE: IMPORT_JOB_STATE.EMPTY,\n UPDATE: IMPORT_JOB_STATE.EMPTY,\n FIX: IMPORT_JOB_STATE.EMPTY\n },\n records: [],\n errorMessage: '',\n errorStatus: '',\n creationTime: time,\n modificationTime: time\n };\n try {\n debugDev(`Inserting: ${JSON.stringify(newQueueItem)} to ${collection}}`);\n const result = await operator.insertOne(newQueueItem);\n debugDev(JSON.stringify(result));\n if (result.acknowledged) {\n logger.info(`New PRIO queue item for ${operation} ${correlationId} has been made in ${collection}`);\n return;\n }\n // TEST-COVERAGE: this error is not (currently) tested\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n } catch (error) {\n // TEST-COVERAGE: this error is not (currently) tested\n const errorMessage = error.payload || error.message || '';\n logError(error);\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);\n }\n }\n\n async function createBulk({correlationId, cataloger, oCatalogerIn, operation, contentType, recordLoadParams, stream, operationSettings}) {\n const time = new Date();\n const newQueueItem = {\n correlationId,\n cataloger,\n oCatalogerIn,\n operation,\n operations: [operation],\n operationSettings: {\n prio: false,\n ...operationSettings\n },\n contentType,\n recordLoadParams,\n queueItemState: stream ? QUEUE_ITEM_STATE.VALIDATOR.UPLOADING : QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS,\n blobSize: 0,\n importJobState: {\n CREATE: IMPORT_JOB_STATE.EMPTY,\n UPDATE: IMPORT_JOB_STATE.EMPTY,\n FIX: IMPORT_JOB_STATE.EMPTY\n },\n records: [],\n errorMessage: '',\n errorStatus: '',\n creationTime: time,\n modificationTime: time\n };\n\n if (stream) {\n debug(`Creating streamBulk`);\n try {\n // NOTE: on success this returns the SAME normalized shape as\n // the no-stream path below: {correlationId, acknowledged, queueItemState}\n await new Promise((resolve, reject) => {\n const outputStream = gridFSBucket.openUploadStream(correlationId);\n\n // an upload stream error used to hang the promise forever\n // (no error handler on outputStream) -> wire it to reject\n outputStream.on('error', (error) => {\n outputStream.destroy();\n reject(error);\n });\n\n stream\n .on('error', (error) => {\n // input failed mid-upload -> abort, do not leave the\n // partial GridFS upload stream running\n outputStream.destroy();\n reject(error);\n })\n .on('data', chunk => outputStream.write(chunk))\n .on('end', () => outputStream.end(undefined, undefined, () => {\n resolve(correlationId);\n }));\n });\n logger.info(`New BULK queue item for ${operation} ${correlationId} has been made in ${collection}!`);\n debug(`New BULK queue item for ${operation} ${correlationId} has been made in ${collection}!`);\n const result = await operator.insertOne(newQueueItem);\n if (!result.acknowledged) {\n // TEST-COVERAGE: this error is not (currently) tested\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n }\n // normalize the return shape to the same one the no-stream\n // path returns - do not leak the driver InsertOneResult\n return {correlationId, acknowledged: result.acknowledged, queueItemState: newQueueItem.queueItemState};\n } catch (error) {\n const errorMessage = error.payload || error.message || '';\n logError(error);\n // best-effort cleanup of a partial GridFS file\n await removePartialContent(correlationId);\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);\n }\n }\n\n async function removePartialContent(correlationId) {\n try {\n const files = await gridFSBucket.find({filename: correlationId}).toArray();\n for (const file of files) {\n await gridFSBucket.delete(file._id);\n logger.debug(`mongo/createBulk: removed partial GridFS file for ${correlationId}`);\n }\n } catch (cleanupError) {\n logger.debug(`mongo/createBulk: partial GridFS cleanup failed for ${correlationId}: ${cleanupError.message}`);\n }\n }\n\n logger.debug(`No stream`);\n debug(`Creating noStreamBulk`);\n if (!stream) {\n try {\n const result = await operator.insertOne(newQueueItem);\n if (result.acknowledged) {\n logger.info(`New noStream BULK queue item for ${operation} ${correlationId} has been made in ${collection}`);\n debug(`New noStream BULK queue item for ${operation} ${correlationId} has been made in ${collection}`);\n // same normalized shape as the stream path above\n return {correlationId, acknowledged: result.acknowledged, queueItemState: QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS};\n }\n // TEST-COVERAGE: this error is not (currently) tested\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n } catch (error) {\n // TEST-COVERAGE: this error is not (currently) tested\n const errorMessage = error.payload || error.message || '';\n logError(error);\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);\n }\n }\n }\n\n // Check state that the queueItem has not waited too long and set state\n // Optionally set also queueItems errorStatus & errorMessage\n //\n // returns a discriminated result instead of a bare boolean so\n // callers can tell the three outcomes apart:\n // {status: 'updated', queueItem} - state was set\n // {status: 'timeout', queueItem} - item was aborted by checkTimeOut\n // {status: 'unchanged', queueItem} - state was ABORT/ERROR/DONE already,\n // or the item no longer exists\n async function checkAndSetState({correlationId, state, errorMessage = undefined, errorStatus = undefined}) {\n assertString(correlationId, 'correlationId');\n // checkTimeOut returns true, if queueItem is fresher than the configured\n // timeout window and it's state is not ABORT/ERROR/DONE\n // otherwise it sets queueItem and importJobStates to state ABORT and add errorStatus and errorMessage (408, 'Timeout')\n const timeOut = await checkTimeOut({correlationId});\n if (timeOut) {\n const queueItem = await setState({correlationId, state, errorMessage, errorStatus});\n return {status: 'updated', queueItem};\n }\n // Read the current item so the caller can tell \"aborted by timeout\"\n // apart from \"unchanged\" and see the resulting document\n const queueItem = await operator.findOne({correlationId}, {projection: {_id: 0}});\n const status = queueItem && queueItem.queueItemState === QUEUE_ITEM_STATE.ABORT ? 'timeout' : 'unchanged';\n return {status, queueItem};\n }\n\n // Check state that the queueItem has not waited too long and set importJobState\n // Note: checkAndSetImportJobState cannot be used to set queueItems errorStatus & errorMessage (expect in case of a TimeOut)\n //\n // returns a discriminated result, see checkAndSetState\n async function checkAndSetImportJobState({correlationId, operation, importJobState}) {\n assertString(correlationId, 'correlationId');\n // checkTimeOut returns true, if queueItem is fresher than the configured\n // timeout window and it's state is not ABORT/ERROR/DONE\n // otherwise it sets queueItem and importJobStates to state ABORT and errorStatue and errorMessage (408, 'Timeout')\n logger.debug(`${correlationId}, ${importJobState}, ${operation}`);\n const timeOut = await checkTimeOut({correlationId, operation, importJobState});\n if (timeOut) {\n const queueItem = await setImportJobState({correlationId, operation, importJobState});\n return {status: 'updated', queueItem};\n }\n const queueItem = await operator.findOne({correlationId}, {projection: {_id: 0}});\n const status = queueItem && queueItem.queueItemState === QUEUE_ITEM_STATE.ABORT ? 'timeout' : 'unchanged';\n return {status, queueItem};\n }\n\n async function query(params, showParams = {}) {\n logger.debug(`Querying: ${JSON.stringify(params)}, ${JSON.stringify(showParams)}`);\n debug(`Querying: ${JSON.stringify(params)}, ${JSON.stringify(showParams)}`);\n const {limit = 1000, skip = 0, ...rest} = params;\n\n // rest is an internal, trusted query object built by the calling service:\n // callers must not splice raw user input into it. Its keys may legitimately\n // use dot notation and $-operators, which is why rest is not\n // field-sanitized here.\n const result = await operator.find(rest, {projection: createProjection(showParams)})\n .limit(parseInt(limit, 10))\n .skip(parseInt(skip, 10))\n .toArray();\n logger.debug(`Query result: ${result.length > 0 ? 'Found!' : 'Not found!'}`);\n logger.silly(`${JSON.stringify(result)}`);\n debug(`${JSON.stringify(result)}`);\n return result;\n }\n\n // We default to NOT showing operations, operationSettings, recordLoadParams and importJobState when a queueItem is queried\n // These can all be shown with showAll\n // Single fields can be shown by showOperation, showOperationSettings, showRecordLoadParams and showImportJobState\n\n function createProjection(showParams = {}) {\n logger.silly(`Creating projection for query: ${JSON.stringify(showParams)}`);\n debug(`Creating projection for query: ${JSON.stringify(showParams)}`);\n const {showAll = 0, showOperations = 0, showOperationSettings = 0, showRecordLoadParams = 0, showImportJobState = 0} = showParams;\n const combinedShowParams = {\n showAll,\n showOperations,\n showOperationSettings,\n showRecordLoadParams,\n showImportJobState\n };\n logger.debug(`showAll: ${showAll}, showOperations: ${showOperations}, showOperationSettings: ${showOperationSettings}, showRecordLoadParams: ${showRecordLoadParams}, showImportJobState: ${showImportJobState}`);\n debug(`showAll: ${showAll}, showOperations: ${showOperations}, showOperationSettings: ${showOperationSettings}, showRecordLoadParams: ${showRecordLoadParams}, showImportJobState: ${showImportJobState}`);\n debug(`${JSON.stringify(combinedShowParams)}`);\n\n // NOTE: the show-* params are parsed with the shared parseBoolean, whose\n // semantics are: undefined -> false, numeric strings via Boolean(Number(v))\n // ('0' -> false, '1' -> true), other non-empty non-'false' strings -> true.\n if (parseBoolean(showAll)) {\n //if (showAll === true || showAll === '1' || showAll === 1) {\n debug(`ShowAll: ${showAll}`);\n return {\n _id: 0\n };\n }\n\n const showParamToField = {\n 'showOperations': 'operations',\n 'showOperationSettings': 'operationSettings',\n 'showRecordLoadParams': 'recordLoadParams',\n 'showImportJobState': 'importJobState'\n };\n\n const result = Object.keys(combinedShowParams)\n //.filter(param => param !== 'showAll' && combinedShowParams[param] !== true && combinedShowParams[param] !== 1 && combinedShowParams[param] !== '1')\n .filter(param => param !== 'showAll' && !parseBoolean(combinedShowParams[param]))\n .filter(param => showParamToField[param])\n .map((param) => showParamToField[param]);\n logger.silly(`We want to exclude from projection: ${JSON.stringify(result)}`);\n debug(`We want to exclude from projection: ${JSON.stringify(result)}`);\n\n\n const excludeObject = Object.fromEntries(result.map(param => [param, 0]));\n logger.silly(`We want to exclude from projection: ${JSON.stringify(excludeObject)}`);\n debug(`We want to exclude from projection: ${JSON.stringify(excludeObject)}`);\n\n\n return {\n _id: 0,\n ...excludeObject\n };\n }\n\n // Note: edited in v4.2.4 to return queueItem *after* possible timout has aborted it\n // Note: previous versions returned queueItem *before* its state/queueItemState is timeOuted\n async function queryById({correlationId, checkModTime = false}) {\n assertString(correlationId, 'correlationId');\n if (checkModTime) {\n debug(`queryById checkModTime: ${checkModTime}`);\n await checkTimeOut({correlationId});\n return queryById({correlationId});\n }\n const result = await operator.findOne({correlationId}, {projection: {_id: 0}});\n return result;\n }\n\n // Check that if the item has waited too long\n // If the last modification time for the queueItem is older than the\n // configured window (MONGO_TIMEOUT_MINUTES, default 1 minute)\n // set state to ABORT and return false, otherwise return true\n // If the state is already ABORT or ERROR or DONE return false\n // if the item does not exist (deleted concurrently) return false\n // instead of crashing on destructuring of null\n\n async function checkTimeOut({correlationId}) {\n assertString(correlationId, 'correlationId');\n const queueItem = await operator.findOne({correlationId});\n if (!queueItem) {\n logger.silly(`${correlationId} not found in ${collection} - not a timeout`);\n return false;\n }\n // coalesce a missing importJobState (legacy rows written before the\n // importJobState shape existed) to all-EMPTY so .CREATE/.UPDATE/.FIX never\n // read off undefined and crash the pipeline.\n const {modificationTime, queueItemState: oldState, importJobState = {CREATE: IMPORT_JOB_STATE.EMPTY, UPDATE: IMPORT_JOB_STATE.EMPTY, FIX: IMPORT_JOB_STATE.EMPTY}} = queueItem;\n //debug(`${modificationTime} - oldState: ${oldState}`);\n // should we check for DONE too?\n if ([QUEUE_ITEM_STATE.ABORT, QUEUE_ITEM_STATE.ERROR, QUEUE_ITEM_STATE.DONE].includes(oldState)) {\n logger.silly(`${correlationId} has already state: ${oldState}`);\n return false;\n }\n\n // configurable \"no progress\" window (default 1 minute)\n const timeoutTime = new Date(new Date(modificationTime).getTime() + timeoutMinutes * 60000);\n logger.silly(`${correlationId} timeOut @ ${timeoutTime}`);\n\n if (timeoutTime < new Date()) {\n const finalImportJobStates = [IMPORT_JOB_STATE.ABORT, IMPORT_JOB_STATE.DONE, IMPORT_JOB_STATE.EMPTY, IMPORT_JOB_STATE.ERROR];\n if (!finalImportJobStates.includes(importJobState.CREATE)) {\n await setImportJobState({correlationId, operation: 'CREATE', importJobState: IMPORT_JOB_STATE.ABORT});\n }\n\n if (!finalImportJobStates.includes(importJobState.UPDATE)) {\n await setImportJobState({correlationId, operation: 'UPDATE', importJobState: IMPORT_JOB_STATE.ABORT});\n }\n\n if (!finalImportJobStates.includes(importJobState.FIX)) {\n await setImportJobState({correlationId, operation: 'FIX', importJobState: IMPORT_JOB_STATE.ABORT});\n }\n\n await setState({correlationId, state: QUEUE_ITEM_STATE.ABORT, errorStatus: httpStatus.REQUEST_TIMEOUT, errorMessage: `Timeout in ${oldState}`});\n\n return false;\n }\n\n return true;\n }\n\n async function remove(params) {\n logger.silly(`${JSON.stringify(params)}`);\n logger.info(`Removing from Mongo (${collection}) id: ${params.correlationId}`);\n debug(`${JSON.stringify(params)}`);\n debug(`Removing from Mongo (${collection}) id: ${params.correlationId}`);\n assertString(params.correlationId, 'correlationId');\n\n\n try {\n // true only when a GridFS file was actually deleted, false when there\n // was no content to delete\n const removedContent = await removeContent({correlationId: params.correlationId});\n debug(`removedContent (removeContent result): ${JSON.stringify(removedContent)}`);\n const deleteResult = await operator.deleteOne({correlationId: params.correlationId});\n debug(`deleteResult: ${JSON.stringify(deleteResult)}`);\n // callers need to tell \"nothing was there\" apart from \"a queueItem was\n // deleted\" (the old bare `true` said both)\n return {removed: deleteResult.deletedCount > 0, deletedContent: removedContent};\n } catch (err) {\n // TEST-COVERAGE: this error is not (currently) tested\n if (err instanceof MongoDriverError) {\n debug(err);\n if (err.message.indexOf('File not found for id') !== -1) {\n logger.silly(`mongo/remove: File not found, removing queueItem ${JSON.stringify(params.correlationId)} from ${collection}`);\n debug(`mongo/remove: File not found, removing queueItem ${JSON.stringify(params.correlationId)} from ${collection}`);\n const deleteResult = await operator.deleteOne({correlationId: params.correlationId});\n return {removed: deleteResult.deletedCount > 0, deletedContent: false};\n }\n logger.error(err.message);\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n }\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n }\n }\n\n async function readContent(correlationId) {\n logger.info(`Reading content from mongo for id: ${correlationId} in ${collection}`);\n debug(`Reading content from mongo for id: ${correlationId} in ${collection}`);\n assertString(correlationId, 'correlationId');\n\n const result = await operator.findOne({correlationId}); // njsscan-ignore: node_nosqli_injection\n debug(result);\n if (result) {\n const {operationSettings} = result;\n // createBulk never sets operationSettings.noStream (phantom flag,\n // always undefined). Derive the condition from the GridFS file instead:\n // bulk content is available only when a file was actually uploaded.\n let files = [];\n if (operationSettings.prio === false) {\n files = await gridFSBucket.find({filename: correlationId}).toArray();\n }\n const hasContent = files.length > 0;\n if (hasContent) {\n return {\n contentType: result.contentType,\n // the file existence was verified by the find() above; a\n // 'error' event on the returned stream (e.g. file deleted\n // concurrently) is an unhandled event for the caller -> the\n // contract is: the caller MUST attach an 'error' handler.\n readStream: gridFSBucket.openDownloadStreamByName(correlationId)\n };\n }\n logger.debug(`OperationSettings for ${correlationId}: ${JSON.stringify(operationSettings)}`);\n debug(`OperationSettings for ${correlationId}: ${JSON.stringify(operationSettings)}`);\n debug(`Content is only available for streamBulk jobs. ${correlationId} is not a streamBulk job.`);\n throw new ApiError(httpStatus.BAD_REQUEST, {message: `Content is only available for streamBulk jobs. ${correlationId} is not a streamBulk job.`});\n }\n\n throw new ApiError(httpStatus.NOT_FOUND, {message: `Job ${correlationId} not found.`});\n }\n\n async function removeContent(params) {\n logger.info(`Removing content from mongo for id: ${params.correlationId} in ${collection}`);\n debug(`Removing content from mongo for id: ${params.correlationId} in ${collection}`);\n assertString(params.correlationId, 'correlationId');\n\n const result = await gridFSBucket.find({filename: params.correlationId}).toArray();\n debug(`mongo/removeContent: find bucket result ${JSON.stringify(result)}`);\n\n\n if (result.length > 0) {\n // Note: we assume and handle just one file for filename!\n const [firstBucketResult] = result;\n const fileId = firstBucketResult._id;\n\n await gridFSBucket.delete(fileId);\n return true;\n }\n\n logger.debug(`No files for ${params.correlationId} to delete`);\n // false: there was no content to delete (the old `true` here made callers\n // unable to tell \"file deleted\" apart from \"no file existed\")\n return false;\n }\n\n // async so the entry-point guard below rejects (400) instead of throwing\n // synchronously before any promise is returned\n async function getOne({queueItemState, importJobState = undefined}) {\n //params example: \"queueItemState\": \"IMPORTER.IN_QUEUE\", \"importJobState\":{\"importJobState.UPDATE\": \"IN_QUEUE\"}\n logger.silly(`queueItemState: ${queueItemState}, importJobState: ${JSON.stringify(importJobState)}`);\n //debug(`queueItemState: ${queueItemState}, importJobState: ${JSON.stringify(importJobState)}`);\n // guard is outside the try so the 400 is not re-wrapped as a 500\n if (queueItemState) {\n assertString(queueItemState, 'queueItemState');\n }\n if (importJobState) {\n assertImportJobStateFilter(importJobState, 'importJobState');\n }\n const cleanQueueItemState = queueItemState ? {queueItemState} : undefined;\n const options = {projection: {_id: 0}};\n\n try {\n\n // Just queueItemState\n if (queueItemState && importJobState === undefined) {\n logger.silly(`Checking DB ${collection} for just ${JSON.stringify(cleanQueueItemState.queueItemState)}`);\n //debug(`Checking DB ${collection} for just ${JSON.stringify(cleanQueueItemState.queueItemState)}`);\n return operator.findOne({...cleanQueueItemState}, options);\n }\n\n // importJobState\n if (importJobState && queueItemState === undefined) {\n logger.silly(`Checking DB ${collection} for ${JSON.stringify(importJobState)}`);\n //debug(`Checking DB ${collection} for ${JSON.stringify(importJobState)}`);\n return operator.findOne({...importJobState}, options);\n }\n\n // importJobState and queueItemState\n if (importJobState && queueItemState) {\n logger.silly(`Checking DB ${collection} for ${queueItemState} and ${JSON.stringify(importJobState)}`);\n //debug(`Checking DB ${collection} for ${queueItemState} and ${JSON.stringify(importJobState)}`);\n return operator.findOne({...cleanQueueItemState, ...importJobState}, options);\n }\n\n logger.debug(`getOne not working!`);\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, 'Missing parameters, cannot get queueItem');\n\n } catch (error) {\n const errorMessage = error.payload || error.message || '';\n logError(error);\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);\n }\n }\n\n // a missing GridFS file is reported by openDownloadStreamByName as an\n // async 'error' event, which would be an unhandled error event (a process\n // crash) if the caller never attaches a handler. Pre-check the file\n // existence here and throw NOT_FOUND synchronously instead. Note: this\n // makes getStream async (callers must await it). The contract for the\n // returned stream is the same as readContent: the caller MUST attach an\n // 'error' handler.\n async function getStream(correlationId) {\n logger.info(`Forming stream from mongo for ${correlationId} in ${collection}`);\n assertString(correlationId, 'correlationId');\n try {\n const files = await gridFSBucket.find({filename: correlationId}).toArray();\n if (files.length === 0) {\n logger.info(`getStream: file ${correlationId} not found in ${collection}`);\n throw new ApiError(httpStatus.NOT_FOUND, {message: `File ${correlationId} not found in ${collection}`});\n }\n // Return content stream\n return gridFSBucket.openDownloadStreamByName(correlationId);\n } catch (error) {\n // TEST-COVERAGE: this error is not (currently) tested\n if (error instanceof ApiError) {\n throw error;\n }\n const errorMessage = error.payload || error.message || '';\n logError(error);\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);\n }\n }\n\n async function pushIds({correlationId, handledIds, rejectedIds}) {\n // validate array inputs before dereferencing .length (guard is\n // outside the try so the 400 is not re-wrapped as a 500)\n if (!Array.isArray(handledIds) || !Array.isArray(rejectedIds)) {\n throw new ApiError(httpStatus.BAD_REQUEST, 'pushIds: handledIds and rejectedIds must be arrays');\n }\n assertString(correlationId, 'correlationId');\n logger.verbose(`Push ids (${handledIds.length}) and rejectedIds (${rejectedIds.length}) ${correlationId} to ${collection}`);\n logger.debug(`ids (${handledIds.length}): ${JSON.stringify(handledIds)}, rejectedIds ${rejectedIds.length}: ${JSON.stringify(rejectedIds)}`);\n try {\n await operator.updateOne({\n correlationId\n }, {\n $set: {\n modificationTime: new Date()\n },\n $push: {\n handledIds: {$each: handledIds},\n rejectedIds: {$each: rejectedIds}\n }\n });\n } catch (error) {\n const errorMessage = error.payload || error.message || '';\n logError(error);\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);\n }\n }\n\n async function pushMessages({correlationId, messages, messageField = 'messages'}) {\n // validate array input before dereferencing .length (guard is\n // outside the try so the 400 is not re-wrapped as a 500)\n if (!Array.isArray(messages)) {\n throw new ApiError(httpStatus.BAD_REQUEST, 'pushMessages: messages must be an array');\n }\n assertString(correlationId, 'correlationId');\n assertString(messageField, 'messageField');\n logger.verbose(`Push messages (${messages.length}) to ${messageField} ${correlationId} to ${collection}`);\n logger.silly(`Messages (${messages.length}): ${JSON.stringify(messages)}`);\n try {\n await operator.updateOne({\n correlationId\n }, {\n $set: {\n modificationTime: new Date()\n },\n $push: {\n [messageField]: {$each: messages}\n }\n });\n } catch (error) {\n const errorMessage = error.payload || error.message || '';\n logError(error);\n throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);\n }\n }\n\n function setState({correlationId, state, errorMessage = undefined, errorStatus = undefined}) {\n assertString(correlationId, 'correlationId');\n const errorString = errorMessage || errorStatus ? `, Error message: '${JSON.stringify(errorMessage) || ''}', Error status: '${errorStatus || ''}'` : '';\n logger.info(`Setting queue-item state ${state} for ${correlationId}${errorString} to ${collection}`);\n\n const stateInQueueItemStates = Object.values(QUEUE_ITEM_STATE).indexOf(state) > -1;\n const stateInQueueItemStatesValidator = Object.values(QUEUE_ITEM_STATE.VALIDATOR).indexOf(state) > -1;\n const stateInQueueItemStatesImporter = Object.values(QUEUE_ITEM_STATE.IMPORTER).indexOf(state) > -1;\n if (!stateInQueueItemStates && !stateInQueueItemStatesValidator && !stateInQueueItemStatesImporter) {\n debug(`Trying to set invalid state ${state}`);\n throw new ApiError(httpStatus.BAD_REQUEST, 'Trying to set invalid state');\n }\n\n const updateValues = {\n queueItemState: state,\n modificationTime: new Date(),\n errorMessage,\n errorStatus\n };\n\n // errorMessage/errorStatus are sticky: transitions that do not set them\n // (undefined) drop the keys from the $set instead of clearing the fields,\n // so a later state change intentionally keeps the previous values and the\n // last error stays visible on the queueItem.\n // Do not update value that are undefined\n Object.keys(updateValues).forEach(key => updateValues[key] === undefined && delete updateValues[key]);\n\n return operator\n .findOneAndUpdate(\n {correlationId},\n {$set: updateValues},\n {projection: {_id: 0}, returnDocument: 'after'}\n );\n }\n\n function setImportJobState({correlationId, operation, importJobState}) {\n\n // should this also get importJobState as previously created object? {}\n assertString(correlationId, 'correlationId');\n logger.info(`Setting queue-item importJobState: {${operation}: ${importJobState}} for ${correlationId}`);\n debug(`Setting queue-item importJobState: {${operation}: ${importJobState}} for ${correlationId}`);\n\n if (!(operation in OPERATIONS)) {\n debug(`Error: ${operation} is not in ${JSON.stringify(OPERATIONS)}`);\n throw new ApiError(httpStatus.BAD_REQUEST, 'Invalid operation for import job state');\n }\n\n if (!(importJobState in IMPORT_JOB_STATE)) {\n debug(`Error: ${importJobState} is not in ${JSON.stringify(IMPORT_JOB_STATE)}`);\n throw new ApiError(httpStatus.BAD_REQUEST, 'Invalid import job state');\n }\n\n const newJobState = getNewJobState(operation, importJobState);\n\n\n function getNewJobState(operation, importJobState) {\n if (operation === OPERATIONS.CREATE) {\n return {'importJobState.CREATE': importJobState};\n }\n if (operation === OPERATIONS.UPDATE) {\n return {'importJobState.UPDATE': importJobState};\n }\n if (operation === OPERATIONS.FIX) {\n return {'importJobState.FIX': importJobState};\n }\n // Unreachable in practice: setImportJobState validates operation\n // against OPERATIONS before calling; kept defensively.\n throw new ApiError('400', 'Invalid operation for import job state');\n }\n\n return operator.findOneAndUpdate({\n correlationId\n }, {\n $set: {\n ...newJobState,\n modificationTime: new Date()\n }\n }, {projection: {_id: 0}, returnDocument: 'after'});\n }\n\n // `findOneAndUpdate` (mongodb v6) returns the resulting document\n // directly, or `null` when nothing matched. The old `result.ok || result`\n // was v3 debris: a non-matching correlationId came back as a silent `null`\n // and callers could not tell \"no match\" from success. Now a `null` result\n // is mapped to an ApiError 404, and the document itself is returned on\n // success (no more `{ok, value}` shape to worry about).\n function unwrapAndUpdateResult(result, correlationId) {\n if (result === null || result === undefined) {\n logger.info(`Queue item ${correlationId} not found in ${collection}`);\n throw new ApiError(httpStatus.NOT_FOUND, {message: `Job ${correlationId} not found.`});\n }\n return result;\n }\n\n async function setOperation({correlationId, operation}) {\n assertString(correlationId, 'correlationId');\n const newOperation = operation;\n const existing = await operator.findOne({correlationId});\n if (!existing) {\n logger.info(`Setting queue-item operation: queue item ${correlationId} not found in ${collection}`);\n throw new ApiError(httpStatus.NOT_FOUND, {message: `Job ${correlationId} not found.`});\n }\n const {operation: oldOperation} = existing;\n logger.info(`Setting queue-item operation from ${oldOperation} to ${newOperation} for ${correlationId} to ${collection}`);\n\n const result = await operator.findOneAndUpdate({\n correlationId\n }, {\n $set: {\n operation: newOperation,\n modificationTime: new Date()\n }\n }, {projection: {_id: 0}, returnDocument: 'after'});\n debugDev(`${JSON.stringify(result)}`);\n return unwrapAndUpdateResult(result, correlationId);\n // logger.debug(JSON.stringify(result));\n }\n\n async function setOperations({correlationId, addOperation, removeOperation = undefined}) {\n assertString(correlationId, 'correlationId');\n\n const queueItem = await operator.findOne({correlationId});\n logger.silly(`We found a queueItem: ${JSON.stringify(queueItem)}`);\n if (!queueItem) {\n logger.info(`Setting queue-item operations: queue item ${correlationId} not found in ${collection}`);\n throw new ApiError(httpStatus.NOT_FOUND, {message: `Job ${correlationId} not found.`});\n }\n\n const oldOperations = queueItem.operations;\n\n logger.info(`Setting queue-item operations from ${JSON.stringify(oldOperations)} by adding ${addOperation} and removing ${removeOperation} for ${correlationId} to ${collection}`);\n\n const operationsAfterRemove = oldOperations.filter(operation => operation !== removeOperation);\n const operationsAfterRemoveAndAdd = operationsAfterRemove.includes(addOperation) ? operationsAfterRemove : [...operationsAfterRemove, addOperation];\n\n logger.silly(`operationsAfterRemove: ${operationsAfterRemove}`);\n logger.silly(`operationAfterRemoveAndAdd: ${operationsAfterRemoveAndAdd}`);\n\n const result = await operator.findOneAndUpdate({\n correlationId\n }, {\n $set: {\n operations: operationsAfterRemoveAndAdd,\n modificationTime: new Date()\n }\n }, {projection: {_id: 0}, returnDocument: 'after'});\n debugDev(`${JSON.stringify(result)}`);\n return unwrapAndUpdateResult(result, correlationId);\n // logger.debug(JSON.stringify(result));\n }\n\n async function addBlobSize({correlationId}) {\n assertString(correlationId, 'correlationId');\n\n const result = await operator.findOneAndUpdate({\n correlationId,\n queueItemState: QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS\n }, {\n $inc: {\n blobSize: 1\n },\n $set: {\n modificationTime: new Date()\n }\n }, {projection: {_id: 0}, returnDocument: 'after'});\n\n logger.verbose(`AddBlobSizeResult in mongo: ${JSON.stringify(result)}`);\n debugDev(`${JSON.stringify(result)}`);\n return result;\n }\n\n async function setBlobSize({correlationId, blobSize}) {\n assertString(correlationId, 'correlationId');\n assertNumber(blobSize, 'blobSize');\n\n const result = await operator.findOneAndUpdate({\n correlationId\n }, {\n $set: {\n blobSize,\n modificationTime: new Date()\n }\n }, {projection: {_id: 0}, returnDocument: 'after'});\n\n logger.verbose(`SetBlobSizeResult in mongo: ${JSON.stringify(result)}`);\n debugDev(`${JSON.stringify(result)}`);\n return result;\n }\n\n}\n"],
5
+ "mappings": "AAEA,SAAQ,aAAa,cAAc,wBAAuB;AAC1D,OAAO,gBAAgB;AACvB,OAAO,uBAAuB;AAC9B,SAAQ,cAAc,SAAS,UAAU,oBAAmB;AAC5D,SAAQ,kBAAkB,kBAAkB,kBAAiB;AAC7D,SAAQ,UAAU,cAAc,cAAc,kCAAiC;AAiC/E,8BAA+B,WAAW,YAAY,KAAK,YAAY,OAAO,CAAC,GAAG;AAChF,QAAM,SAAS,aAAa;AAQ5B,QAAM,iBAAiB,OAAO,SAAS,QAAQ,IAAI,uBAAuB,EAAE,KAAK;AACjF,SAAO,MAAM,wBAAwB,cAAc,kDAAkD;AACrG,QAAM,QAAQ,kBAAkB,0CAA0C;AAC1E,QAAM,WAAW,MAAM,OAAO,MAAM;AAIpC,QAAM,UAAU,QAAQ,KAAK,QAAQ,KAAK,QAAQ,KAAK,YAAY;AACnE,MAAI,eAAe,KAAK;AACxB,MAAI,WAAW,KAAK;AACpB,MAAI,CAAC,SAAS;AACZ,UAAM,SAAS,MAAM,YAAY,QAAQ,SAAS;AAClD,UAAM,SAAS,OAAO,GAAG,EAAE;AAC3B,mBAAe,gBAAgB,IAAI,aAAa,QAAQ,EAAC,YAAY,WAAU,CAAC;AAChF,eAAW,YAAY,OAAO,WAAW,UAAU;AAAA,EACrD;AACA,WAAS,UAAU,SAAS,SAAS,EAAE,iBAAiB,UAAU,EAAE;AAEpE,SAAO,EAAC,YAAY,YAAY,kBAAkB,2BAA2B,OAAO,kBAAkB,WAAW,cAAc,QAAQ,aAAa,eAAe,QAAQ,WAAW,UAAU,mBAAmB,SAAS,cAAc,cAAc,eAAe,aAAa,YAAW;AAE/R,iBAAe,WAAW,EAAC,eAAe,WAAW,cAAc,WAAW,kBAAiB,GAAG;AAChG,aAAS,YAAY;AACrB,UAAM,OAAO,oBAAI,KAAK;AACtB,UAAM,eAAe;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY,CAAC,SAAS;AAAA,MACtB;AAAA,MACA,mBAAmB;AAAA,QACjB,MAAM;AAAA,QACN,GAAG;AAAA,MACL;AAAA,MACA,gBAAgB,iBAAiB,UAAU;AAAA,MAC3C,UAAU;AAAA,MACV,gBAAgB;AAAA,QACd,QAAQ,iBAAiB;AAAA,QACzB,QAAQ,iBAAiB;AAAA,QACzB,KAAK,iBAAiB;AAAA,MACxB;AAAA,MACA,SAAS,CAAC;AAAA,MACV,cAAc;AAAA,MACd,aAAa;AAAA,MACb,cAAc;AAAA,MACd,kBAAkB;AAAA,IACpB;AACA,QAAI;AACF,eAAS,cAAc,KAAK,UAAU,YAAY,CAAC,OAAO,UAAU,GAAG;AACvE,YAAM,SAAS,MAAM,SAAS,UAAU,YAAY;AACpD,eAAS,KAAK,UAAU,MAAM,CAAC;AAC/B,UAAI,OAAO,cAAc;AACvB,eAAO,KAAK,2BAA2B,SAAS,IAAI,aAAa,qBAAqB,UAAU,EAAE;AAClG;AAAA,MACF;AAEA,YAAM,IAAI,SAAS,WAAW,qBAAqB;AAAA,IACrD,SAAS,OAAO;AAEd,YAAM,eAAe,MAAM,WAAW,MAAM,WAAW;AACvD,eAAS,KAAK;AACd,YAAM,IAAI,SAAS,WAAW,uBAAuB,kBAAkB,YAAY,EAAE;AAAA,IACvF;AAAA,EACF;AAEA,iBAAe,WAAW,EAAC,eAAe,WAAW,cAAc,WAAW,aAAa,kBAAkB,QAAQ,kBAAiB,GAAG;AACvI,UAAM,OAAO,oBAAI,KAAK;AACtB,UAAM,eAAe;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY,CAAC,SAAS;AAAA,MACtB,mBAAmB;AAAA,QACjB,MAAM;AAAA,QACN,GAAG;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB,SAAS,iBAAiB,UAAU,YAAY,iBAAiB,UAAU;AAAA,MAC3F,UAAU;AAAA,MACV,gBAAgB;AAAA,QACd,QAAQ,iBAAiB;AAAA,QACzB,QAAQ,iBAAiB;AAAA,QACzB,KAAK,iBAAiB;AAAA,MACxB;AAAA,MACA,SAAS,CAAC;AAAA,MACV,cAAc;AAAA,MACd,aAAa;AAAA,MACb,cAAc;AAAA,MACd,kBAAkB;AAAA,IACpB;AAEA,QAAI,QAAQ;AACV,YAAM,qBAAqB;AAC3B,UAAI;AAGF,cAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,gBAAM,eAAe,aAAa,iBAAiB,aAAa;AAIhE,uBAAa,GAAG,SAAS,CAAC,UAAU;AAClC,yBAAa,QAAQ;AACrB,mBAAO,KAAK;AAAA,UACd,CAAC;AAED,iBACG,GAAG,SAAS,CAAC,UAAU;AAGtB,yBAAa,QAAQ;AACrB,mBAAO,KAAK;AAAA,UACd,CAAC,EACA,GAAG,QAAQ,WAAS,aAAa,MAAM,KAAK,CAAC,EAC7C,GAAG,OAAO,MAAM,aAAa,IAAI,QAAW,QAAW,MAAM;AAC5D,oBAAQ,aAAa;AAAA,UACvB,CAAC,CAAC;AAAA,QACN,CAAC;AACD,eAAO,KAAK,2BAA2B,SAAS,IAAI,aAAa,qBAAqB,UAAU,GAAG;AACnG,cAAM,2BAA2B,SAAS,IAAI,aAAa,qBAAqB,UAAU,GAAG;AAC7F,cAAM,SAAS,MAAM,SAAS,UAAU,YAAY;AACpD,YAAI,CAAC,OAAO,cAAc;AAExB,gBAAM,IAAI,SAAS,WAAW,qBAAqB;AAAA,QACrD;AAGA,eAAO,EAAC,eAAe,cAAc,OAAO,cAAc,gBAAgB,aAAa,eAAc;AAAA,MACvG,SAAS,OAAO;AACd,cAAM,eAAe,MAAM,WAAW,MAAM,WAAW;AACvD,iBAAS,KAAK;AAEd,cAAM,qBAAqB,aAAa;AACxC,cAAM,IAAI,SAAS,WAAW,uBAAuB,kBAAkB,YAAY,EAAE;AAAA,MACvF;AAAA,IACF;AAEA,mBAAe,qBAAqBA,gBAAe;AACjD,UAAI;AACF,cAAM,QAAQ,MAAM,aAAa,KAAK,EAAC,UAAUA,eAAa,CAAC,EAAE,QAAQ;AACzE,mBAAW,QAAQ,OAAO;AACxB,gBAAM,aAAa,OAAO,KAAK,GAAG;AAClC,iBAAO,MAAM,qDAAqDA,cAAa,EAAE;AAAA,QACnF;AAAA,MACF,SAAS,cAAc;AACrB,eAAO,MAAM,uDAAuDA,cAAa,KAAK,aAAa,OAAO,EAAE;AAAA,MAC9G;AAAA,IACF;AAEA,WAAO,MAAM,WAAW;AACxB,UAAM,uBAAuB;AAC7B,QAAI,CAAC,QAAQ;AACX,UAAI;AACF,cAAM,SAAS,MAAM,SAAS,UAAU,YAAY;AACpD,YAAI,OAAO,cAAc;AACvB,iBAAO,KAAK,oCAAoC,SAAS,IAAI,aAAa,qBAAqB,UAAU,EAAE;AAC3G,gBAAM,oCAAoC,SAAS,IAAI,aAAa,qBAAqB,UAAU,EAAE;AAErG,iBAAO,EAAC,eAAe,cAAc,OAAO,cAAc,gBAAgB,iBAAiB,UAAU,oBAAmB;AAAA,QAC1H;AAEA,cAAM,IAAI,SAAS,WAAW,qBAAqB;AAAA,MACrD,SAAS,OAAO;AAEd,cAAM,eAAe,MAAM,WAAW,MAAM,WAAW;AACvD,iBAAS,KAAK;AACd,cAAM,IAAI,SAAS,WAAW,uBAAuB,kBAAkB,YAAY,EAAE;AAAA,MACvF;AAAA,IACF;AAAA,EACF;AAWA,iBAAe,iBAAiB,EAAC,eAAe,OAAO,eAAe,QAAW,cAAc,OAAS,GAAG;AACzG,iBAAa,eAAe,eAAe;AAI3C,UAAM,UAAU,MAAM,aAAa,EAAC,cAAa,CAAC;AAClD,QAAI,SAAS;AACX,YAAMC,aAAY,MAAM,SAAS,EAAC,eAAe,OAAO,cAAc,YAAW,CAAC;AAClF,aAAO,EAAC,QAAQ,WAAW,WAAAA,WAAS;AAAA,IACtC;AAGA,UAAM,YAAY,MAAM,SAAS,QAAQ,EAAC,cAAa,GAAG,EAAC,YAAY,EAAC,KAAK,EAAC,EAAC,CAAC;AAChF,UAAM,SAAS,aAAa,UAAU,mBAAmB,iBAAiB,QAAQ,YAAY;AAC9F,WAAO,EAAC,QAAQ,UAAS;AAAA,EAC3B;AAMA,iBAAe,0BAA0B,EAAC,eAAe,WAAW,eAAc,GAAG;AACnF,iBAAa,eAAe,eAAe;AAI3C,WAAO,MAAM,GAAG,aAAa,KAAK,cAAc,KAAK,SAAS,EAAE;AAChE,UAAM,UAAU,MAAM,aAAa,EAAC,eAAe,WAAW,eAAc,CAAC;AAC7E,QAAI,SAAS;AACX,YAAMA,aAAY,MAAM,kBAAkB,EAAC,eAAe,WAAW,eAAc,CAAC;AACpF,aAAO,EAAC,QAAQ,WAAW,WAAAA,WAAS;AAAA,IACtC;AACA,UAAM,YAAY,MAAM,SAAS,QAAQ,EAAC,cAAa,GAAG,EAAC,YAAY,EAAC,KAAK,EAAC,EAAC,CAAC;AAChF,UAAM,SAAS,aAAa,UAAU,mBAAmB,iBAAiB,QAAQ,YAAY;AAC9F,WAAO,EAAC,QAAQ,UAAS;AAAA,EAC3B;AAEA,iBAAe,MAAM,QAAQ,aAAa,CAAC,GAAG;AAC5C,WAAO,MAAM,aAAa,KAAK,UAAU,MAAM,CAAC,KAAK,KAAK,UAAU,UAAU,CAAC,EAAE;AACjF,UAAM,aAAa,KAAK,UAAU,MAAM,CAAC,KAAK,KAAK,UAAU,UAAU,CAAC,EAAE;AAC1E,UAAM,EAAC,QAAQ,KAAM,OAAO,GAAG,GAAG,KAAI,IAAI;AAM1C,UAAM,SAAS,MAAM,SAAS,KAAK,MAAM,EAAC,YAAY,iBAAiB,UAAU,EAAC,CAAC,EAChF,MAAM,SAAS,OAAO,EAAE,CAAC,EACzB,KAAK,SAAS,MAAM,EAAE,CAAC,EACvB,QAAQ;AACX,WAAO,MAAM,iBAAiB,OAAO,SAAS,IAAI,WAAW,YAAY,EAAE;AAC3E,WAAO,MAAM,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACxC,UAAM,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACjC,WAAO;AAAA,EACT;AAMA,WAAS,iBAAiB,aAAa,CAAC,GAAG;AACzC,WAAO,MAAM,kCAAkC,KAAK,UAAU,UAAU,CAAC,EAAE;AAC3E,UAAM,kCAAkC,KAAK,UAAU,UAAU,CAAC,EAAE;AACpE,UAAM,EAAC,UAAU,GAAG,iBAAiB,GAAG,wBAAwB,GAAG,uBAAuB,GAAG,qBAAqB,EAAC,IAAI;AACvH,UAAM,qBAAqB;AAAA,MACzB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AACA,WAAO,MAAM,YAAY,OAAO,qBAAqB,cAAc,4BAA4B,qBAAqB,2BAA2B,oBAAoB,yBAAyB,kBAAkB,EAAE;AAChN,UAAM,YAAY,OAAO,qBAAqB,cAAc,4BAA4B,qBAAqB,2BAA2B,oBAAoB,yBAAyB,kBAAkB,EAAE;AACzM,UAAM,GAAG,KAAK,UAAU,kBAAkB,CAAC,EAAE;AAK7C,QAAI,aAAa,OAAO,GAAG;AAEzB,YAAM,YAAY,OAAO,EAAE;AAC3B,aAAO;AAAA,QACL,KAAK;AAAA,MACP;AAAA,IACF;AAEA,UAAM,mBAAmB;AAAA,MACvB,kBAAkB;AAAA,MAClB,yBAAyB;AAAA,MACzB,wBAAwB;AAAA,MACxB,sBAAsB;AAAA,IACxB;AAEA,UAAM,SAAS,OAAO,KAAK,kBAAkB,EAE1C,OAAO,WAAS,UAAU,aAAa,CAAC,aAAa,mBAAmB,KAAK,CAAC,CAAC,EAC/E,OAAO,WAAS,iBAAiB,KAAK,CAAC,EACvC,IAAI,CAAC,UAAU,iBAAiB,KAAK,CAAC;AACzC,WAAO,MAAM,uCAAuC,KAAK,UAAU,MAAM,CAAC,EAAE;AAC5E,UAAM,uCAAuC,KAAK,UAAU,MAAM,CAAC,EAAE;AAGrE,UAAM,gBAAgB,OAAO,YAAY,OAAO,IAAI,WAAS,CAAC,OAAO,CAAC,CAAC,CAAC;AACxE,WAAO,MAAM,uCAAuC,KAAK,UAAU,aAAa,CAAC,EAAE;AACnF,UAAM,uCAAuC,KAAK,UAAU,aAAa,CAAC,EAAE;AAG5E,WAAO;AAAA,MACL,KAAK;AAAA,MACL,GAAG;AAAA,IACL;AAAA,EACF;AAIA,iBAAe,UAAU,EAAC,eAAe,eAAe,MAAK,GAAG;AAC9D,iBAAa,eAAe,eAAe;AAC3C,QAAI,cAAc;AAChB,YAAM,2BAA2B,YAAY,EAAE;AAC/C,YAAM,aAAa,EAAC,cAAa,CAAC;AAClC,aAAO,UAAU,EAAC,cAAa,CAAC;AAAA,IAClC;AACA,UAAM,SAAS,MAAM,SAAS,QAAQ,EAAC,cAAa,GAAG,EAAC,YAAY,EAAC,KAAK,EAAC,EAAC,CAAC;AAC7E,WAAO;AAAA,EACT;AAUA,iBAAe,aAAa,EAAC,cAAa,GAAG;AAC3C,iBAAa,eAAe,eAAe;AAC3C,UAAM,YAAY,MAAM,SAAS,QAAQ,EAAC,cAAa,CAAC;AACxD,QAAI,CAAC,WAAW;AACd,aAAO,MAAM,GAAG,aAAa,iBAAiB,UAAU,kBAAkB;AAC1E,aAAO;AAAA,IACT;AAIA,UAAM,EAAC,kBAAkB,gBAAgB,UAAU,iBAAiB,EAAC,QAAQ,iBAAiB,OAAO,QAAQ,iBAAiB,OAAO,KAAK,iBAAiB,MAAK,EAAC,IAAI;AAGrK,QAAI,CAAC,iBAAiB,OAAO,iBAAiB,OAAO,iBAAiB,IAAI,EAAE,SAAS,QAAQ,GAAG;AAC9F,aAAO,MAAM,GAAG,aAAa,uBAAuB,QAAQ,EAAE;AAC9D,aAAO;AAAA,IACT;AAGA,UAAM,cAAc,IAAI,KAAK,IAAI,KAAK,gBAAgB,EAAE,QAAQ,IAAI,iBAAiB,GAAK;AAC1F,WAAO,MAAM,GAAG,aAAa,cAAc,WAAW,EAAE;AAExD,QAAI,cAAc,oBAAI,KAAK,GAAG;AAC5B,YAAM,uBAAuB,CAAC,iBAAiB,OAAO,iBAAiB,MAAM,iBAAiB,OAAO,iBAAiB,KAAK;AAC3H,UAAI,CAAC,qBAAqB,SAAS,eAAe,MAAM,GAAG;AACzD,cAAM,kBAAkB,EAAC,eAAe,WAAW,UAAU,gBAAgB,iBAAiB,MAAK,CAAC;AAAA,MACtG;AAEA,UAAI,CAAC,qBAAqB,SAAS,eAAe,MAAM,GAAG;AACzD,cAAM,kBAAkB,EAAC,eAAe,WAAW,UAAU,gBAAgB,iBAAiB,MAAK,CAAC;AAAA,MACtG;AAEA,UAAI,CAAC,qBAAqB,SAAS,eAAe,GAAG,GAAG;AACtD,cAAM,kBAAkB,EAAC,eAAe,WAAW,OAAO,gBAAgB,iBAAiB,MAAK,CAAC;AAAA,MACnG;AAEA,YAAM,SAAS,EAAC,eAAe,OAAO,iBAAiB,OAAO,aAAa,WAAW,iBAAiB,cAAc,cAAc,QAAQ,GAAE,CAAC;AAE9I,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,EACT;AAEA,iBAAe,OAAO,QAAQ;AAC5B,WAAO,MAAM,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACxC,WAAO,KAAK,wBAAwB,UAAU,SAAS,OAAO,aAAa,EAAE;AAC7E,UAAM,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACjC,UAAM,wBAAwB,UAAU,SAAS,OAAO,aAAa,EAAE;AACvE,iBAAa,OAAO,eAAe,eAAe;AAGlD,QAAI;AAGF,YAAM,iBAAiB,MAAM,cAAc,EAAC,eAAe,OAAO,cAAa,CAAC;AAChF,YAAM,0CAA0C,KAAK,UAAU,cAAc,CAAC,EAAE;AAChF,YAAM,eAAe,MAAM,SAAS,UAAU,EAAC,eAAe,OAAO,cAAa,CAAC;AACnF,YAAM,iBAAiB,KAAK,UAAU,YAAY,CAAC,EAAE;AAGrD,aAAO,EAAC,SAAS,aAAa,eAAe,GAAG,gBAAgB,eAAc;AAAA,IAChF,SAAS,KAAK;AAEZ,UAAI,eAAe,kBAAkB;AACnC,cAAM,GAAG;AACT,YAAI,IAAI,QAAQ,QAAQ,uBAAuB,MAAM,IAAI;AACvD,iBAAO,MAAM,oDAAoD,KAAK,UAAU,OAAO,aAAa,CAAC,SAAS,UAAU,EAAE;AAC1H,gBAAM,oDAAoD,KAAK,UAAU,OAAO,aAAa,CAAC,SAAS,UAAU,EAAE;AACnH,gBAAM,eAAe,MAAM,SAAS,UAAU,EAAC,eAAe,OAAO,cAAa,CAAC;AACnF,iBAAO,EAAC,SAAS,aAAa,eAAe,GAAG,gBAAgB,MAAK;AAAA,QACvE;AACA,eAAO,MAAM,IAAI,OAAO;AACxB,cAAM,IAAI,SAAS,WAAW,uBAAuB,IAAI,OAAO;AAAA,MAClE;AACA,YAAM,IAAI,SAAS,WAAW,uBAAuB,IAAI,OAAO;AAAA,IAClE;AAAA,EACF;AAEA,iBAAe,YAAY,eAAe;AACxC,WAAO,KAAK,sCAAsC,aAAa,OAAO,UAAU,EAAE;AAClF,UAAM,sCAAsC,aAAa,OAAO,UAAU,EAAE;AAC5E,iBAAa,eAAe,eAAe;AAE3C,UAAM,SAAS,MAAM,SAAS,QAAQ,EAAC,cAAa,CAAC;AACrD,UAAM,MAAM;AACZ,QAAI,QAAQ;AACV,YAAM,EAAC,kBAAiB,IAAI;AAI5B,UAAI,QAAQ,CAAC;AACb,UAAI,kBAAkB,SAAS,OAAO;AACpC,gBAAQ,MAAM,aAAa,KAAK,EAAC,UAAU,cAAa,CAAC,EAAE,QAAQ;AAAA,MACrE;AACA,YAAM,aAAa,MAAM,SAAS;AAClC,UAAI,YAAY;AACd,eAAO;AAAA,UACL,aAAa,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA,UAKpB,YAAY,aAAa,yBAAyB,aAAa;AAAA,QACjE;AAAA,MACF;AACA,aAAO,MAAM,yBAAyB,aAAa,KAAK,KAAK,UAAU,iBAAiB,CAAC,EAAE;AAC3F,YAAM,yBAAyB,aAAa,KAAK,KAAK,UAAU,iBAAiB,CAAC,EAAE;AACpF,YAAM,kDAAkD,aAAa,2BAA2B;AAChG,YAAM,IAAI,SAAS,WAAW,aAAa,EAAC,SAAS,kDAAkD,aAAa,4BAA2B,CAAC;AAAA,IAClJ;AAEA,UAAM,IAAI,SAAS,WAAW,WAAW,EAAC,SAAS,OAAO,aAAa,cAAa,CAAC;AAAA,EACvF;AAEA,iBAAe,cAAc,QAAQ;AACnC,WAAO,KAAK,uCAAuC,OAAO,aAAa,OAAO,UAAU,EAAE;AAC1F,UAAM,uCAAuC,OAAO,aAAa,OAAO,UAAU,EAAE;AACpF,iBAAa,OAAO,eAAe,eAAe;AAElD,UAAM,SAAS,MAAM,aAAa,KAAK,EAAC,UAAU,OAAO,cAAa,CAAC,EAAE,QAAQ;AACjF,UAAM,2CAA2C,KAAK,UAAU,MAAM,CAAC,EAAE;AAGzE,QAAI,OAAO,SAAS,GAAG;AAErB,YAAM,CAAC,iBAAiB,IAAI;AAC5B,YAAM,SAAS,kBAAkB;AAEjC,YAAM,aAAa,OAAO,MAAM;AAChC,aAAO;AAAA,IACT;AAEA,WAAO,MAAM,gBAAgB,OAAO,aAAa,YAAY;AAG7D,WAAO;AAAA,EACT;AAIA,iBAAe,OAAO,EAAC,gBAAgB,iBAAiB,OAAS,GAAG;AAElE,WAAO,MAAM,mBAAmB,cAAc,qBAAqB,KAAK,UAAU,cAAc,CAAC,EAAE;AAGnG,QAAI,gBAAgB;AAClB,mBAAa,gBAAgB,gBAAgB;AAAA,IAC/C;AACA,QAAI,gBAAgB;AAClB,iCAA2B,gBAAgB,gBAAgB;AAAA,IAC7D;AACA,UAAM,sBAAsB,iBAAiB,EAAC,eAAc,IAAI;AAChE,UAAM,UAAU,EAAC,YAAY,EAAC,KAAK,EAAC,EAAC;AAErC,QAAI;AAGF,UAAI,kBAAkB,mBAAmB,QAAW;AAClD,eAAO,MAAM,eAAe,UAAU,aAAa,KAAK,UAAU,oBAAoB,cAAc,CAAC,EAAE;AAEvG,eAAO,SAAS,QAAQ,EAAC,GAAG,oBAAmB,GAAG,OAAO;AAAA,MAC3D;AAGA,UAAI,kBAAkB,mBAAmB,QAAW;AAClD,eAAO,MAAM,eAAe,UAAU,QAAQ,KAAK,UAAU,cAAc,CAAC,EAAE;AAE9E,eAAO,SAAS,QAAQ,EAAC,GAAG,eAAc,GAAG,OAAO;AAAA,MACtD;AAGA,UAAI,kBAAkB,gBAAgB;AACpC,eAAO,MAAM,eAAe,UAAU,QAAQ,cAAc,QAAQ,KAAK,UAAU,cAAc,CAAC,EAAE;AAEpG,eAAO,SAAS,QAAQ,EAAC,GAAG,qBAAqB,GAAG,eAAc,GAAG,OAAO;AAAA,MAC9E;AAEA,aAAO,MAAM,qBAAqB;AAClC,YAAM,IAAI,SAAS,WAAW,uBAAuB,0CAA0C;AAAA,IAEjG,SAAS,OAAO;AACd,YAAM,eAAe,MAAM,WAAW,MAAM,WAAW;AACvD,eAAS,KAAK;AACd,YAAM,IAAI,SAAS,WAAW,uBAAuB,kBAAkB,YAAY,EAAE;AAAA,IACvF;AAAA,EACF;AASA,iBAAe,UAAU,eAAe;AACtC,WAAO,KAAK,iCAAiC,aAAa,OAAO,UAAU,EAAE;AAC7E,iBAAa,eAAe,eAAe;AAC3C,QAAI;AACF,YAAM,QAAQ,MAAM,aAAa,KAAK,EAAC,UAAU,cAAa,CAAC,EAAE,QAAQ;AACzE,UAAI,MAAM,WAAW,GAAG;AACtB,eAAO,KAAK,mBAAmB,aAAa,iBAAiB,UAAU,EAAE;AACzE,cAAM,IAAI,SAAS,WAAW,WAAW,EAAC,SAAS,QAAQ,aAAa,iBAAiB,UAAU,GAAE,CAAC;AAAA,MACxG;AAEA,aAAO,aAAa,yBAAyB,aAAa;AAAA,IAC5D,SAAS,OAAO;AAEd,UAAI,iBAAiB,UAAU;AAC7B,cAAM;AAAA,MACR;AACA,YAAM,eAAe,MAAM,WAAW,MAAM,WAAW;AACvD,eAAS,KAAK;AACd,YAAM,IAAI,SAAS,WAAW,uBAAuB,kBAAkB,YAAY,EAAE;AAAA,IACvF;AAAA,EACF;AAEA,iBAAe,QAAQ,EAAC,eAAe,YAAY,YAAW,GAAG;AAG/D,QAAI,CAAC,MAAM,QAAQ,UAAU,KAAK,CAAC,MAAM,QAAQ,WAAW,GAAG;AAC7D,YAAM,IAAI,SAAS,WAAW,aAAa,oDAAoD;AAAA,IACjG;AACA,iBAAa,eAAe,eAAe;AAC3C,WAAO,QAAQ,aAAa,WAAW,MAAM,sBAAsB,YAAY,MAAM,KAAK,aAAa,OAAO,UAAU,EAAE;AAC1H,WAAO,MAAM,QAAQ,WAAW,MAAM,MAAM,KAAK,UAAU,UAAU,CAAC,iBAAiB,YAAY,MAAM,KAAK,KAAK,UAAU,WAAW,CAAC,EAAE;AAC3I,QAAI;AACF,YAAM,SAAS,UAAU;AAAA,QACvB;AAAA,MACF,GAAG;AAAA,QACD,MAAM;AAAA,UACJ,kBAAkB,oBAAI,KAAK;AAAA,QAC7B;AAAA,QACA,OAAO;AAAA,UACL,YAAY,EAAC,OAAO,WAAU;AAAA,UAC9B,aAAa,EAAC,OAAO,YAAW;AAAA,QAClC;AAAA,MACF,CAAC;AAAA,IACH,SAAS,OAAO;AACd,YAAM,eAAe,MAAM,WAAW,MAAM,WAAW;AACvD,eAAS,KAAK;AACd,YAAM,IAAI,SAAS,WAAW,uBAAuB,kBAAkB,YAAY,EAAE;AAAA,IACvF;AAAA,EACF;AAEA,iBAAe,aAAa,EAAC,eAAe,UAAU,eAAe,WAAU,GAAG;AAGhF,QAAI,CAAC,MAAM,QAAQ,QAAQ,GAAG;AAC5B,YAAM,IAAI,SAAS,WAAW,aAAa,yCAAyC;AAAA,IACtF;AACA,iBAAa,eAAe,eAAe;AAC3C,iBAAa,cAAc,cAAc;AACzC,WAAO,QAAQ,kBAAkB,SAAS,MAAM,QAAQ,YAAY,IAAI,aAAa,OAAO,UAAU,EAAE;AACxG,WAAO,MAAM,aAAa,SAAS,MAAM,MAAM,KAAK,UAAU,QAAQ,CAAC,EAAE;AACzE,QAAI;AACF,YAAM,SAAS,UAAU;AAAA,QACvB;AAAA,MACF,GAAG;AAAA,QACD,MAAM;AAAA,UACJ,kBAAkB,oBAAI,KAAK;AAAA,QAC7B;AAAA,QACA,OAAO;AAAA,UACL,CAAC,YAAY,GAAG,EAAC,OAAO,SAAQ;AAAA,QAClC;AAAA,MACF,CAAC;AAAA,IACH,SAAS,OAAO;AACd,YAAM,eAAe,MAAM,WAAW,MAAM,WAAW;AACvD,eAAS,KAAK;AACd,YAAM,IAAI,SAAS,WAAW,uBAAuB,kBAAkB,YAAY,EAAE;AAAA,IACvF;AAAA,EACF;AAEA,WAAS,SAAS,EAAC,eAAe,OAAO,eAAe,QAAW,cAAc,OAAS,GAAG;AAC3F,iBAAa,eAAe,eAAe;AAC3C,UAAM,cAAc,gBAAgB,cAAc,qBAAqB,KAAK,UAAU,YAAY,KAAK,EAAE,qBAAqB,eAAe,EAAE,MAAM;AACrJ,WAAO,KAAK,4BAA4B,KAAK,QAAQ,aAAa,GAAG,WAAW,OAAO,UAAU,EAAE;AAEnG,UAAM,yBAAyB,OAAO,OAAO,gBAAgB,EAAE,QAAQ,KAAK,IAAI;AAChF,UAAM,kCAAkC,OAAO,OAAO,iBAAiB,SAAS,EAAE,QAAQ,KAAK,IAAI;AACnG,UAAM,iCAAiC,OAAO,OAAO,iBAAiB,QAAQ,EAAE,QAAQ,KAAK,IAAI;AACjG,QAAI,CAAC,0BAA0B,CAAC,mCAAmC,CAAC,gCAAgC;AAClG,YAAM,+BAA+B,KAAK,EAAE;AAC5C,YAAM,IAAI,SAAS,WAAW,aAAa,6BAA6B;AAAA,IAC1E;AAEA,UAAM,eAAe;AAAA,MACnB,gBAAgB;AAAA,MAChB,kBAAkB,oBAAI,KAAK;AAAA,MAC3B;AAAA,MACA;AAAA,IACF;AAOA,WAAO,KAAK,YAAY,EAAE,QAAQ,SAAO,aAAa,GAAG,MAAM,UAAa,OAAO,aAAa,GAAG,CAAC;AAEpG,WAAO,SACJ;AAAA,MACC,EAAC,cAAa;AAAA,MACd,EAAC,MAAM,aAAY;AAAA,MACnB,EAAC,YAAY,EAAC,KAAK,EAAC,GAAG,gBAAgB,QAAO;AAAA,IAChD;AAAA,EACJ;AAEA,WAAS,kBAAkB,EAAC,eAAe,WAAW,eAAc,GAAG;AAGrE,iBAAa,eAAe,eAAe;AAC3C,WAAO,KAAK,uCAAuC,SAAS,KAAK,cAAc,SAAS,aAAa,EAAE;AACvG,UAAM,uCAAuC,SAAS,KAAK,cAAc,SAAS,aAAa,EAAE;AAEjG,QAAI,EAAE,aAAa,aAAa;AAC9B,YAAM,UAAU,SAAS,cAAc,KAAK,UAAU,UAAU,CAAC,EAAE;AACnE,YAAM,IAAI,SAAS,WAAW,aAAa,wCAAwC;AAAA,IACrF;AAEA,QAAI,EAAE,kBAAkB,mBAAmB;AACzC,YAAM,UAAU,cAAc,cAAc,KAAK,UAAU,gBAAgB,CAAC,EAAE;AAC9E,YAAM,IAAI,SAAS,WAAW,aAAa,0BAA0B;AAAA,IACvE;AAEA,UAAM,cAAc,eAAe,WAAW,cAAc;AAG5D,aAAS,eAAeC,YAAWC,iBAAgB;AACjD,UAAID,eAAc,WAAW,QAAQ;AACnC,eAAO,EAAC,yBAAyBC,gBAAc;AAAA,MACjD;AACA,UAAID,eAAc,WAAW,QAAQ;AACnC,eAAO,EAAC,yBAAyBC,gBAAc;AAAA,MACjD;AACA,UAAID,eAAc,WAAW,KAAK;AAChC,eAAO,EAAC,sBAAsBC,gBAAc;AAAA,MAC9C;AAGA,YAAM,IAAI,SAAS,OAAO,wCAAwC;AAAA,IACpE;AAEA,WAAO,SAAS,iBAAiB;AAAA,MAC/B;AAAA,IACF,GAAG;AAAA,MACD,MAAM;AAAA,QACJ,GAAG;AAAA,QACH,kBAAkB,oBAAI,KAAK;AAAA,MAC7B;AAAA,IACF,GAAG,EAAC,YAAY,EAAC,KAAK,EAAC,GAAG,gBAAgB,QAAO,CAAC;AAAA,EACpD;AAQA,WAAS,sBAAsB,QAAQ,eAAe;AACpD,QAAI,WAAW,QAAQ,WAAW,QAAW;AAC3C,aAAO,KAAK,cAAc,aAAa,iBAAiB,UAAU,EAAE;AACpE,YAAM,IAAI,SAAS,WAAW,WAAW,EAAC,SAAS,OAAO,aAAa,cAAa,CAAC;AAAA,IACvF;AACA,WAAO;AAAA,EACT;AAEA,iBAAe,aAAa,EAAC,eAAe,UAAS,GAAG;AACtD,iBAAa,eAAe,eAAe;AAC3C,UAAM,eAAe;AACrB,UAAM,WAAW,MAAM,SAAS,QAAQ,EAAC,cAAa,CAAC;AACvD,QAAI,CAAC,UAAU;AACb,aAAO,KAAK,4CAA4C,aAAa,iBAAiB,UAAU,EAAE;AAClG,YAAM,IAAI,SAAS,WAAW,WAAW,EAAC,SAAS,OAAO,aAAa,cAAa,CAAC;AAAA,IACvF;AACA,UAAM,EAAC,WAAW,aAAY,IAAI;AAClC,WAAO,KAAK,qCAAqC,YAAY,OAAO,YAAY,QAAQ,aAAa,OAAO,UAAU,EAAE;AAExH,UAAM,SAAS,MAAM,SAAS,iBAAiB;AAAA,MAC7C;AAAA,IACF,GAAG;AAAA,MACD,MAAM;AAAA,QACJ,WAAW;AAAA,QACX,kBAAkB,oBAAI,KAAK;AAAA,MAC7B;AAAA,IACF,GAAG,EAAC,YAAY,EAAC,KAAK,EAAC,GAAG,gBAAgB,QAAO,CAAC;AAClD,aAAS,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACpC,WAAO,sBAAsB,QAAQ,aAAa;AAAA,EAEpD;AAEA,iBAAe,cAAc,EAAC,eAAe,cAAc,kBAAkB,OAAS,GAAG;AACvF,iBAAa,eAAe,eAAe;AAE3C,UAAM,YAAY,MAAM,SAAS,QAAQ,EAAC,cAAa,CAAC;AACxD,WAAO,MAAM,yBAAyB,KAAK,UAAU,SAAS,CAAC,EAAE;AACjE,QAAI,CAAC,WAAW;AACd,aAAO,KAAK,6CAA6C,aAAa,iBAAiB,UAAU,EAAE;AACnG,YAAM,IAAI,SAAS,WAAW,WAAW,EAAC,SAAS,OAAO,aAAa,cAAa,CAAC;AAAA,IACvF;AAEA,UAAM,gBAAgB,UAAU;AAEhC,WAAO,KAAK,sCAAsC,KAAK,UAAU,aAAa,CAAC,cAAc,YAAY,iBAAiB,eAAe,QAAQ,aAAa,OAAO,UAAU,EAAE;AAEjL,UAAM,wBAAwB,cAAc,OAAO,eAAa,cAAc,eAAe;AAC7F,UAAM,8BAA8B,sBAAsB,SAAS,YAAY,IAAI,wBAAwB,CAAC,GAAG,uBAAuB,YAAY;AAElJ,WAAO,MAAM,0BAA0B,qBAAqB,EAAE;AAC9D,WAAO,MAAM,+BAA+B,2BAA2B,EAAE;AAEzE,UAAM,SAAS,MAAM,SAAS,iBAAiB;AAAA,MAC7C;AAAA,IACF,GAAG;AAAA,MACD,MAAM;AAAA,QACJ,YAAY;AAAA,QACZ,kBAAkB,oBAAI,KAAK;AAAA,MAC7B;AAAA,IACF,GAAG,EAAC,YAAY,EAAC,KAAK,EAAC,GAAG,gBAAgB,QAAO,CAAC;AAClD,aAAS,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACpC,WAAO,sBAAsB,QAAQ,aAAa;AAAA,EAEpD;AAEA,iBAAe,YAAY,EAAC,cAAa,GAAG;AAC1C,iBAAa,eAAe,eAAe;AAE3C,UAAM,SAAS,MAAM,SAAS,iBAAiB;AAAA,MAC7C;AAAA,MACA,gBAAgB,iBAAiB,UAAU;AAAA,IAC7C,GAAG;AAAA,MACD,MAAM;AAAA,QACJ,UAAU;AAAA,MACZ;AAAA,MACA,MAAM;AAAA,QACJ,kBAAkB,oBAAI,KAAK;AAAA,MAC7B;AAAA,IACF,GAAG,EAAC,YAAY,EAAC,KAAK,EAAC,GAAG,gBAAgB,QAAO,CAAC;AAElD,WAAO,QAAQ,+BAA+B,KAAK,UAAU,MAAM,CAAC,EAAE;AACtE,aAAS,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACpC,WAAO;AAAA,EACT;AAEA,iBAAe,YAAY,EAAC,eAAe,SAAQ,GAAG;AACpD,iBAAa,eAAe,eAAe;AAC3C,iBAAa,UAAU,UAAU;AAEjC,UAAM,SAAS,MAAM,SAAS,iBAAiB;AAAA,MAC7C;AAAA,IACF,GAAG;AAAA,MACD,MAAM;AAAA,QACJ;AAAA,QACA,kBAAkB,oBAAI,KAAK;AAAA,MAC7B;AAAA,IACF,GAAG,EAAC,YAAY,EAAC,KAAK,EAAC,GAAG,gBAAgB,QAAO,CAAC;AAElD,WAAO,QAAQ,+BAA+B,KAAK,UAAU,MAAM,CAAC,EAAE;AACtE,aAAS,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACpC,WAAO;AAAA,EACT;AAEF;",
6
+ "names": ["correlationId", "queueItem", "operation", "importJobState"]
7
7
  }
package/dist/mongoLog.js CHANGED
@@ -1,19 +1,36 @@
1
1
  import { MongoClient } from "mongodb";
2
- import moment from "moment";
3
2
  import httpStatus from "http-status";
4
- import sanitize from "mongo-sanitize";
5
- import { createLogger } from "@natlibfi/melinda-backend-commons";
6
- import { Error as ApiError } from "@natlibfi/melinda-commons";
7
- import { logError } from "./utils.js";
3
+ import { createLogger, Error as ApiError } from "@natlibfi/melinda-commons";
4
+ import { logError, assertString } from "./utils.js";
8
5
  import { LOG_ITEM_TYPE } from "./constants.js";
9
- export default async function(MONGO_URI, dbName = "rest-api") {
6
+ export default async function(MONGO_URI, dbName = "rest-api", deps = {}) {
10
7
  const logger = createLogger();
11
- const client = await MongoClient.connect(MONGO_URI);
12
- const db = client.db(dbName);
8
+ function buildBlobSequenceFilter(correlationId, blobSequence) {
9
+ if (!blobSequence) {
10
+ return { correlationId };
11
+ }
12
+ if (typeof blobSequence === "number") {
13
+ return { correlationId, blobSequence: parseInt(blobSequence, 10) };
14
+ }
15
+ if (typeof blobSequence === "string") {
16
+ const parsed = parseInt(blobSequence, 10);
17
+ if (!Number.isNaN(parsed)) {
18
+ return { correlationId, blobSequence: parsed };
19
+ }
20
+ }
21
+ throw new ApiError(httpStatus.BAD_REQUEST, "blobSequence must be a number or a numeric string");
22
+ }
13
23
  const collection = "logs";
14
- return { addLogItem, query, queryById, getListOfLogs, getListOfCatalogers, getListOfCorrelationIds, getExpandedListOfLogs, protect, remove, removeBySequences };
24
+ let db;
25
+ if (deps.collection) {
26
+ db = { collection: () => deps.collection };
27
+ } else {
28
+ const client = await MongoClient.connect(MONGO_URI);
29
+ db = client.db(dbName);
30
+ }
31
+ return { addLogItem, query, queryById, getListOfLogs, getListOfCatalogers, getListOfCorrelationIds, getExpandedListOfLogs, protect, unprotect, remove, removeBySequences };
15
32
  async function addLogItem(logItem) {
16
- const time = moment().toDate();
33
+ const time = /* @__PURE__ */ new Date();
17
34
  const newLogItem = {
18
35
  ...logItem,
19
36
  creationTime: time,
@@ -38,20 +55,21 @@ export default async function(MONGO_URI, dbName = "rest-api") {
38
55
  async function query({ limit = 5, skip = 0, ...rest }) {
39
56
  logger.debug(`Query params: ${JSON.stringify(rest)}`);
40
57
  logger.debug(`Limit and skip params: ${limit} | ${skip}`);
41
- checkLogItemType(rest.logItemType, false, false);
58
+ checkLogItemType(rest.logItemType, false);
42
59
  const result = await db.collection(collection).find(rest).sort({ creationTime: 1 }).skip(parseInt(skip, 10)).limit(parseInt(limit, 10)).toArray();
43
60
  logger.debug(`Query result: ${result.length > 0 ? `Found ${result.length} log items! (skip: ${skip} limit: ${limit})` : "Not found!"}`);
44
61
  return result;
45
62
  }
46
63
  async function queryById(correlationId, logItemType = LOG_ITEM_TYPE.MERGE_LOG, skip = 0, limit = 1) {
64
+ assertString(correlationId, "correlationId");
47
65
  logger.debug(`QueryById: ${correlationId}, logItemType: ${logItemType}, skip: ${skip}, limit: ${limit}`);
48
66
  const result = await db.collection(collection).find({ correlationId, logItemType }).sort({ blobSequence: 1 }).skip(parseInt(skip, 10)).limit(parseInt(limit, 10)).toArray();
49
67
  logger.debug(`Query result: ${result.length > 0 ? `Found ${result.length} log items!` : "Not found!"}`);
50
68
  return result;
51
69
  }
52
70
  async function getListOfLogs(logItemType = LOG_ITEM_TYPE.MERGE_LOG) {
53
- checkLogItemType(logItemType, true, true);
54
- const result = await db.collection(collection).distinct("correlationId", { logItemType: `${logItemType}` });
71
+ checkLogItemType(logItemType, true);
72
+ const result = await db.collection(collection).distinct("correlationId", { logItemType });
55
73
  logger.debug(`Query result for getListOfLogs: ${result.length > 0 ? `Found ${result.length} correlationIds for ${logItemType}!` : "Not found!"}`);
56
74
  return result;
57
75
  }
@@ -65,12 +83,24 @@ export default async function(MONGO_URI, dbName = "rest-api") {
65
83
  const result = await db.collection(collection).distinct("correlationId");
66
84
  return result;
67
85
  }
68
- async function getExpandedListOfLogs({ logItemTypes = [LOG_ITEM_TYPE.MERGE_LOG, LOG_ITEM_TYPE.MATCH_LOG], catalogers = [], dateBefore = /* @__PURE__ */ new Date(), dateAfter = "2000-01-01", test = false }) {
69
- logger.debug(`commons/getExpandedListOfLogs: logItemTypes: ${JSON.stringify(logItemTypes)}, dateAfter: ${dateAfter}, dateBefore: ${dateBefore}}, catalogers: ${JSON.stringify(catalogers)}`);
70
- logger.silly(JSON.stringify(generateMatchObject({ logItemTypes, catalogers, dateBefore, dateAfter })));
86
+ async function getExpandedListOfLogs({ logItemTypes, catalogers = [], dateBefore = /* @__PURE__ */ new Date(), dateAfter = "1970-01-01", test = false }) {
87
+ const wantedLogItemTypes = logItemTypes === void 0 ? [LOG_ITEM_TYPE.MERGE_LOG, LOG_ITEM_TYPE.MATCH_LOG] : logItemTypes;
88
+ if (!Array.isArray(wantedLogItemTypes)) {
89
+ throw new ApiError(httpStatus.BAD_REQUEST, `logItemTypes must be an array: ${logItemTypes}`);
90
+ }
91
+ if (logItemTypes !== void 0) {
92
+ logItemTypes.forEach((logItemType) => {
93
+ if (Object.values(LOG_ITEM_TYPE).includes(logItemType)) {
94
+ return;
95
+ }
96
+ throw new ApiError(httpStatus.BAD_REQUEST, `Unknown logItemType: ${logItemType}`);
97
+ });
98
+ }
99
+ logger.debug(`commons/getExpandedListOfLogs: logItemTypes: ${JSON.stringify(wantedLogItemTypes)}, dateAfter: ${dateAfter}, dateBefore: ${dateBefore}}, catalogers: ${JSON.stringify(catalogers)}`);
100
+ logger.silly(JSON.stringify(generateMatchObject({ logItemTypes: wantedLogItemTypes, catalogers, dateBefore, dateAfter })));
71
101
  const pipeline = [
72
102
  // currently return only MERGE_LOG and MATCH_LOG
73
- generateMatchObject({ logItemTypes, catalogers, dateBefore, dateAfter, test }),
103
+ generateMatchObject({ logItemTypes: wantedLogItemTypes, catalogers, dateBefore, dateAfter }),
74
104
  {
75
105
  "$sort": { "correlationId": 1, "logItemType": 1, "creationTime": 1 }
76
106
  },
@@ -82,8 +112,10 @@ export default async function(MONGO_URI, dbName = "rest-api") {
82
112
  "logCount": { "$sum": 1 }
83
113
  }
84
114
  },
115
+ // after $group the output has no top-level correlationId /
116
+ // logItemType, so the sort must address them via _id
85
117
  {
86
- "$sort": { "correlationId": 1, "logItemType": 1, "creationTime": 1 }
118
+ "$sort": { "_id.correlationId": 1, "_id.logItemType": 1, "creationTime": 1 }
87
119
  }
88
120
  ];
89
121
  const result = await db.collection(collection).aggregate(pipeline).toArray();
@@ -96,33 +128,82 @@ export default async function(MONGO_URI, dbName = "rest-api") {
96
128
  logger.silly(`Query result: ${JSON.stringify(fixedResult)}`);
97
129
  logger.debug(`Query result: ${fixedResult.length > 0 ? `Found ${fixedResult.length} log items!` : "Not found!"}`);
98
130
  return fixedResult;
99
- function generateMatchObject({ logItemTypes: logItemTypes2, catalogers: catalogers2, dateBefore: dateBefore2, dateAfter: dateAfter2, test: test2 = false }) {
131
+ function generateMatchObject({ logItemTypes: logItemTypes2, catalogers: catalogers2, dateBefore: dateBefore2, dateAfter: dateAfter2 }) {
100
132
  const matchOptions = {
101
133
  "$match": {
102
134
  "logItemType": logItemTypes2.length > 0 ? { "$in": logItemTypes2 } : /.*/ui,
103
135
  "cataloger": catalogers2.length > 0 ? { "$in": catalogers2 } : /.*/ui,
136
+ // creationTime is stored as a Date, so the bounds must be
137
+ // Dates too (the old test=true branch compared against ISO strings
138
+ // and never matched; the old non-test branch relied on
139
+ // local-timezone parsing of new Date('YYYY-MM-DD')). Date strings
140
+ // are anchored at UTC midnight so the result is deterministic
141
+ // regardless of the server timezone.
104
142
  "creationTime": {
105
- "$gte": test2 ? new Date(dateAfter2).toISOString() : new Date(dateAfter2),
106
- "$lte": test2 ? new Date(dateBefore2).toISOString() : new Date(dateBefore2)
143
+ "$gte": dateToUtcStartOfDay(dateAfter2),
144
+ "$lt": dateToUtcExclusiveUpperBound(dateBefore2)
107
145
  }
108
146
  }
109
147
  };
110
148
  return matchOptions;
149
+ function dateToUtcStartOfDay(value) {
150
+ if (value instanceof Date) {
151
+ return value;
152
+ }
153
+ const parts = String(value).slice(0, 10).split("-");
154
+ if (parts.length === 3) {
155
+ return new Date(Date.UTC(parts[0], parts[1] - 1, parts[2]));
156
+ }
157
+ return new Date(value);
158
+ }
159
+ function dateToUtcExclusiveUpperBound(value) {
160
+ if (value instanceof Date) {
161
+ return value;
162
+ }
163
+ const parts = String(value).slice(0, 10).split("-");
164
+ if (parts.length === 3) {
165
+ return new Date(Date.UTC(parts[0], parts[1] - 1, parts[2]) + 864e5);
166
+ }
167
+ return new Date(value);
168
+ }
111
169
  }
112
170
  }
113
171
  async function protect(correlationId, blobSequence) {
114
172
  logger.info(`Protecting in Mongo (${collection}) correlationId: ${correlationId}, blobSequence: ${blobSequence}`);
115
- const cleanCorrelationId = sanitize(correlationId);
116
- const cleanBlobSequence = sanitize(blobSequence);
117
- const filter = blobSequence ? { correlationId: cleanCorrelationId, blobSequence: parseInt(cleanBlobSequence, 10) } : { correlationId: cleanCorrelationId };
173
+ assertString(correlationId, "correlationId");
174
+ const filter = buildBlobSequenceFilter(correlationId, blobSequence);
175
+ try {
176
+ const result = await db.collection(collection).updateMany(
177
+ filter,
178
+ [
179
+ {
180
+ $set: {
181
+ modificationTime: /* @__PURE__ */ new Date(),
182
+ // plain true, not a $not toggle: protect must be idempotent
183
+ protected: true
184
+ }
185
+ }
186
+ ]
187
+ );
188
+ return result;
189
+ } catch (error) {
190
+ const errorMessage = error.payload || error.message || "";
191
+ logError(error);
192
+ throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);
193
+ }
194
+ }
195
+ async function unprotect(correlationId, blobSequence) {
196
+ logger.info(`Unprotecting in Mongo (${collection}) correlationId: ${correlationId}, blobSequence: ${blobSequence}`);
197
+ assertString(correlationId, "correlationId");
198
+ const filter = buildBlobSequenceFilter(correlationId, blobSequence);
118
199
  try {
119
200
  const result = await db.collection(collection).updateMany(
120
201
  filter,
121
202
  [
122
203
  {
123
204
  $set: {
124
- modificationTime: moment().toDate(),
125
- protected: { $not: "$protected" }
205
+ modificationTime: /* @__PURE__ */ new Date(),
206
+ protected: false
126
207
  }
127
208
  }
128
209
  ]
@@ -136,8 +217,8 @@ export default async function(MONGO_URI, dbName = "rest-api") {
136
217
  }
137
218
  async function remove(correlationId, force = false) {
138
219
  logger.info(`Removing from Mongo (${collection}) correlationId: ${correlationId}`);
139
- const clean = sanitize(correlationId);
140
- const filter = force ? { correlationId: clean } : { correlationId: clean, protected: { $ne: true } };
220
+ assertString(correlationId, "correlationId");
221
+ const filter = force ? { correlationId } : { correlationId, protected: { $ne: true } };
141
222
  try {
142
223
  const result = await db.collection(collection).deleteMany(filter);
143
224
  return result;
@@ -149,12 +230,12 @@ export default async function(MONGO_URI, dbName = "rest-api") {
149
230
  }
150
231
  async function removeBySequences(correlationId, sequences = [], force = false) {
151
232
  logger.info(`Removing from Mongo (${collection}) correlationId: ${correlationId}, sequences: ${sequences.length}`);
152
- const clean = sanitize(correlationId);
153
- const cleanSequences = sequences.filter((sequence) => Number.isInteger(Number(sequence)) && sequence > 0);
233
+ assertString(correlationId, "correlationId");
234
+ const cleanSequences = sequences.map((sequence) => Number(sequence)).filter((sequence) => Number.isInteger(sequence) && sequence > 0);
154
235
  if (sequences.length !== cleanSequences.length) {
155
236
  throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Invalid sequences in removeBySequences. Sequences should be positive integers`);
156
237
  }
157
- const filter = force ? { correlationId: clean, blobSequence: { $in: cleanSequences } } : { correlationId: clean, blobSequence: { $in: sequences }, protected: { $ne: true } };
238
+ const filter = force ? { correlationId, blobSequence: { $in: cleanSequences } } : { correlationId, blobSequence: { $in: cleanSequences }, protected: { $ne: true } };
158
239
  try {
159
240
  const result = await db.collection(collection).deleteMany(filter);
160
241
  return result;
@@ -164,7 +245,7 @@ export default async function(MONGO_URI, dbName = "rest-api") {
164
245
  throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);
165
246
  }
166
247
  }
167
- function checkLogItemType(logItemType, errorUnknown = false, errorNotExisting = false) {
248
+ function checkLogItemType(logItemType, errorUnknown = false) {
168
249
  if (logItemType) {
169
250
  const typeInLogItemTypes = Object.values(LOG_ITEM_TYPE).indexOf(logItemType) > -1;
170
251
  if (typeInLogItemTypes) {
@@ -175,9 +256,6 @@ export default async function(MONGO_URI, dbName = "rest-api") {
175
256
  }
176
257
  return logger.debug(`WARN: We have unknown logType: ${logItemType}`);
177
258
  }
178
- if (errorNotExisting) {
179
- throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `No logItemType: ${logItemType}`);
180
- }
181
259
  return logger.debug(`No logItemType: ${logItemType}`);
182
260
  }
183
261
  }