@natlibfi/melinda-rest-api-commons 5.0.3 → 5.0.4

package/dist/mongo.js

@@ -0,0 +1,486 @@
+import { MongoClient, GridFSBucket, MongoDriverError } from "mongodb";
+import moment from "moment";
+import httpStatus from "http-status";
+import sanitize from "mongo-sanitize";
+import createDebugLogger from "debug";
+import { createLogger } from "@natlibfi/melinda-backend-commons";
+import { Error as ApiError, parseBoolean } from "@natlibfi/melinda-commons";
+import { QUEUE_ITEM_STATE, IMPORT_JOB_STATE, OPERATIONS } from "./constants.js";
+import { logError } from "./utils.js";
+export default async function(MONGO_URI, collection, db = "rest-api") {
+  const logger = createLogger();
+  const debug = createDebugLogger("@natlibfi/melinda-rest-api-commons/mongo");
+  const debugDev = debug.extend(":dev");
+  const client = await MongoClient.connect(MONGO_URI);
+  const dbConnection = client.db(db);
+  const gridFSBucket = new GridFSBucket(dbConnection, { bucketName: collection });
+  const operator = dbConnection.collection(collection);
+  debugDev(`mongo: ${MONGO_URI}, db: ${db}, collection: ${collection}`);
+  return { createPrio, createBulk, checkAndSetState, checkAndSetImportJobState, query, createProjection, queryById, checkTimeOut, remove, readContent, removeContent, getOne, getStream, setState, setImportJobState, pushIds, pushMessages, setOperation, setOperations, addBlobSize, setBlobSize };
+  async function createPrio({ correlationId, cataloger, oCatalogerIn, operation, operationSettings }) {
+    debugDev("createPrio");
+    const time = moment().toDate();
+    const newQueueItem = {
+      correlationId,
+      cataloger,
+      operation,
+      operations: [operation],
+      oCatalogerIn,
+      operationSettings: {
+        prio: true,
+        ...operationSettings
+      },
+      queueItemState: QUEUE_ITEM_STATE.VALIDATOR.PENDING_VALIDATION,
+      blobSize: 1,
+      importJobState: {
+        CREATE: IMPORT_JOB_STATE.EMPTY,
+        UPDATE: IMPORT_JOB_STATE.EMPTY,
+        FIX: IMPORT_JOB_STATE.EMPTY
+      },
+      records: [],
+      errorMessage: "",
+      errorStatus: "",
+      creationTime: time,
+      modificationTime: time
+    };
+    try {
+      debugDev(`Inserting: ${JSON.stringify(newQueueItem)} to ${collection}}`);
+      const result = await operator.insertOne(newQueueItem);
+      debugDev(JSON.stringify(result));
+      if (result.acknowledged) {
+        logger.info(`New PRIO queue item for ${operation} ${correlationId} has been made in ${collection}`);
+        return;
+      }
+      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);
+    } catch (error) {
+      const errorMessage = error.payload || error.message || "";
+      logError(error);
+      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);
+    }
+  }
+  async function createBulk({ correlationId, cataloger, oCatalogerIn, operation, contentType, recordLoadParams, stream, operationSettings }) {
+    const time = moment().toDate();
+    const newQueueItem = {
+      correlationId,
+      cataloger,
+      oCatalogerIn,
+      operation,
+      operations: [operation],
+      operationSettings: {
+        prio: false,
+        ...operationSettings
+      },
+      contentType,
+      recordLoadParams,
+      queueItemState: stream ? QUEUE_ITEM_STATE.VALIDATOR.UPLOADING : QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS,
+      blobSize: 0,
+      importJobState: {
+        CREATE: IMPORT_JOB_STATE.EMPTY,
+        UPDATE: IMPORT_JOB_STATE.EMPTY,
+        FIX: IMPORT_JOB_STATE.EMPTY
+      },
+      records: [],
+      errorMessage: "",
+      errorStatus: "",
+      creationTime: time,
+      modificationTime: time
+    };
+    if (stream) {
+      debug(`Creating streamBulk`);
+      try {
+        await new Promise((resolve, reject) => {
+          const outputStream = gridFSBucket.openUploadStream(correlationId);
+          stream.on("error", reject).on("data", (chunk) => outputStream.write(chunk)).on("end", () => outputStream.end(void 0, void 0, () => {
+            resolve(correlationId);
+          }));
+        });
+        logger.info(`New BULK queue item for ${operation} ${correlationId} has been made in ${collection}!`);
+        debug(`New BULK queue item for ${operation} ${correlationId} has been made in ${collection}!`);
+        return operator.insertOne(newQueueItem);
+      } catch (error) {
+        const errorMessage = error.payload || error.message || "";
+        logError(error);
+        throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);
+      }
+    }
+    logger.debug(`No stream`);
+    debug(`Creating noStreamBulk`);
+    if (!stream) {
+      try {
+        const result = await operator.insertOne(newQueueItem);
+        if (result.acknowledged) {
+          logger.info(`New noStream BULK queue item for ${operation} ${correlationId} has been made in ${collection}`);
+          debug(`New noStream BULK queue item for ${operation} ${correlationId} has been made in ${collection}`);
+          return { correlationId, queueItemState: QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS };
+        }
+        throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);
+      } catch (error) {
+        const errorMessage = error.payload || error.message || "";
+        logError(error);
+        throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);
+      }
+    }
+  }
+  async function checkAndSetState({ correlationId, state, errorMessage = void 0, errorStatus = void 0 }) {
+    const timeOut = await checkTimeOut({ correlationId });
+    if (timeOut) {
+      return setState({ correlationId, state, errorMessage, errorStatus });
+    }
+    return false;
+  }
+  async function checkAndSetImportJobState({ correlationId, operation, importJobState }) {
+    logger.debug(`${correlationId}, ${importJobState}, ${operation}`);
+    const timeOut = await checkTimeOut({ correlationId, operation, importJobState });
+    if (timeOut) {
+      return setImportJobState({ correlationId, operation, importJobState });
+    }
+    return false;
+  }
+  async function query(params, showParams = {}) {
+    logger.debug(`Querying: ${JSON.stringify(params)}, ${JSON.stringify(showParams)}`);
+    debug(`Querying: ${JSON.stringify(params)}, ${JSON.stringify(showParams)}`);
+    const { limit = 1e3, skip = 0, ...rest } = params;
+    const result = await operator.find(rest, { projection: createProjection(showParams) }).limit(parseInt(limit, 10)).skip(parseInt(skip, 10)).toArray();
+    logger.debug(`Query result: ${result.length > 0 ? "Found!" : "Not found!"}`);
+    logger.silly(`${JSON.stringify(result)}`);
+    debug(`${JSON.stringify(result)}`);
+    return result;
+  }
+  function createProjection(showParams = {}) {
+    logger.silly(`Creating projection for query: ${JSON.stringify(showParams)}`);
+    debug(`Creating projection for query: ${JSON.stringify(showParams)}`);
+    const { showAll = 0, showOperations = 0, showOperationSettings = 0, showRecordLoadParams = 0, showImportJobState = 0 } = showParams;
+    const combinedShowParams = {
+      showAll,
+      showOperations,
+      showOperationSettings,
+      showRecordLoadParams,
+      showImportJobState
+    };
+    logger.debug(`showAll: ${showAll}, showOperations: ${showOperations}, showOperationSettings: ${showOperationSettings}, showRecordLoadParams: ${showRecordLoadParams}, showImportJobState: ${showImportJobState}`);
+    debug(`showAll: ${showAll}, showOperations: ${showOperations}, showOperationSettings: ${showOperationSettings}, showRecordLoadParams: ${showRecordLoadParams}, showImportJobState: ${showImportJobState}`);
+    debug(`${JSON.stringify(combinedShowParams)}`);
+    if (parseBoolean(showAll)) {
+      debug(`ShowAll: ${showAll}`);
+      return {
+        _id: 0
+      };
+    }
+    const showParamToField = {
+      "showOperations": "operations",
+      "showOperationSettings": "operationSettings",
+      "showRecordLoadParams": "recordLoadParams",
+      "showImportJobState": "importJobState"
+    };
+    const result = Object.keys(combinedShowParams).filter((param) => param !== "showAll" && !parseBoolean(combinedShowParams[param])).filter((param) => showParamToField[param]).map((param) => showParamToField[param]);
+    logger.silly(`We want to exclude from projection: ${JSON.stringify(result)}`);
+    debug(`We want to exclude from projection: ${JSON.stringify(result)}`);
+    const excludeObject = Object.fromEntries(result.map((param) => [param, 0]));
+    logger.silly(`We want to exclude from projection: ${JSON.stringify(excludeObject)}`);
+    debug(`We want to exclude from projection: ${JSON.stringify(excludeObject)}`);
+    return {
+      _id: 0,
+      ...excludeObject
+    };
+  }
+  async function queryById({ correlationId, checkModTime = false }) {
+    if (checkModTime) {
+      debug(`queryById checkModTime: ${checkModTime}`);
+      await checkTimeOut({ correlationId });
+      return queryById({ correlationId });
+    }
+    const result = await operator.findOne({ correlationId }, { projection: { _id: 0 } });
+    return result;
+  }
+  async function checkTimeOut({ correlationId }) {
+    const { modificationTime, queueItemState: oldState, importJobState } = await operator.findOne({ correlationId });
+    if ([QUEUE_ITEM_STATE.ABORT, QUEUE_ITEM_STATE.ERROR, QUEUE_ITEM_STATE.DONE].includes(oldState)) {
+      logger.silly(`${correlationId} has already state: ${oldState}`);
+      return false;
+    }
+    const timeoutTime = moment(modificationTime).add(1, "m");
+    logger.silly(`${correlationId} timeOut @ ${timeoutTime}`);
+    if (timeoutTime.isBefore()) {
+      const finalImportJobStates = [IMPORT_JOB_STATE.ABORT, IMPORT_JOB_STATE.DONE, IMPORT_JOB_STATE.EMPTY, IMPORT_JOB_STATE.ERROR];
+      if (!finalImportJobStates.includes(importJobState.CREATE)) {
+        await setImportJobState({ correlationId, operation: "CREATE", importJobState: IMPORT_JOB_STATE.ABORT });
+      }
+      if (!finalImportJobStates.includes(importJobState.UPDATE)) {
+        await setImportJobState({ correlationId, operation: "UPDATE", importJobState: IMPORT_JOB_STATE.ABORT });
+      }
+      if (!finalImportJobStates.includes(importJobState.FIX)) {
+        await setImportJobState({ correlationId, operation: "FIX", importJobState: IMPORT_JOB_STATE.ABORT });
+      }
+      await setState({ correlationId, state: QUEUE_ITEM_STATE.ABORT, errorStatus: httpStatus.REQUEST_TIMEOUT, errorMessage: `Timeout in ${oldState}` });
+      return false;
+    }
+    return true;
+  }
+  async function remove(params) {
+    logger.silly(`${JSON.stringify(params)}`);
+    logger.info(`Removing from Mongo (${collection}) id: ${params.correlationId}`);
+    debug(`${JSON.stringify(params)}`);
+    debug(`Removing from Mongo (${collection}) id: ${params.correlationId}`);
+    const clean = sanitize(params.correlationId);
+    logger.silly(`mongo/remove: clean: ${JSON.stringify(clean)}`);
+    debug(`mongo/remove: clean: ${JSON.stringify(clean)}`);
+    try {
+      const noContent = await removeContent({ correlationId: params.correlationId });
+      debug(`noContent (removeContent result): ${JSON.stringify(noContent)}`);
+      if (noContent) {
+        const deleteResult = await operator.deleteOne({ correlationId: clean });
+        debug(`deleteResult: ${JSON.stringify(deleteResult)}`);
+        return true;
+      }
+    } catch (err) {
+      if (err instanceof MongoDriverError) {
+        debug(err);
+        if (err.message.indexOf("File not found for id") !== -1) {
+          logger.silly(`mongo/remove: File not found, removing queueItem ${JSON.stringify(clean)} from ${collection}`);
+          debug(`mongo/remove: File not found, removing queueItem ${JSON.stringify(clean)} from ${collection}`);
+          await operator.deleteOne({ correlationId: clean });
+          return true;
+        }
+        logger.error(err.message);
+        throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);
+      }
+      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);
+    }
+  }
+  async function readContent(correlationId) {
+    logger.info(`Reading content from mongo for id: ${correlationId} in ${collection}`);
+    debug(`Reading content from mongo for id: ${correlationId} in ${collection}`);
+    const clean = sanitize(correlationId);
+    debug(clean);
+    const result = await operator.findOne({ correlationId: clean });
+    debug(result);
+    if (result) {
+      const { operationSettings } = result;
+      if (operationSettings.prio === false && operationSettings.noStream === false) {
+        return {
+          contentType: result.contentType,
+          readStream: gridFSBucket.openDownloadStreamByName(clean)
+        };
+      }
+      logger.debug(`OperationSettings for ${correlationId}: ${JSON.stringify(operationSettings)}`);
+      debug(`OperationSettings for ${correlationId}: ${JSON.stringify(operationSettings)}`);
+      debug(`Content is only available for streamBulk jobs. ${correlationId} is not a streamBulk job.`);
+      throw new ApiError(httpStatus.BAD_REQUEST, { message: `Content is only available for streamBulk jobs. ${correlationId} is not a streamBulk job.` });
+    }
+    throw new ApiError(httpStatus.NOT_FOUND, { message: `Job ${correlationId} not found.` });
+  }
+  async function removeContent(params) {
+    logger.info(`Removing content from mongo for id: ${params.correlationId} in ${collection}`);
+    debug(`Removing content from mongo for id: ${params.correlationId} in ${collection}`);
+    const clean = sanitize(params.correlationId);
+    const result = await gridFSBucket.find({ filename: clean }).toArray();
+    debug(`mongo/removeContent: find bucket result ${JSON.stringify(result)}`);
+    if (result.length > 0) {
+      const [firstBucketResult] = result;
+      const fileId = firstBucketResult._id;
+      await gridFSBucket.delete(fileId);
+      return true;
+    }
+    logger.debug(`No files for ${params.correlationId} to delete`);
+    return true;
+  }
+  function getOne({ queueItemState, importJobState = void 0 }) {
+    logger.silly(`queueItemState: ${queueItemState}, importJobState: ${JSON.stringify(importJobState)}`);
+    const cleanQueueItemState = queueItemState ? { queueItemState: sanitize(queueItemState) } : void 0;
+    const options = { projection: { _id: 0 } };
+    try {
+      if (queueItemState && importJobState === void 0) {
+        logger.silly(`Checking DB ${collection} for just ${JSON.stringify(cleanQueueItemState.queueItemState)}`);
+        return operator.findOne({ ...cleanQueueItemState }, options);
+      }
+      if (importJobState && queueItemState === void 0) {
+        logger.silly(`Checking DB ${collection} for ${JSON.stringify(importJobState)}`);
+        return operator.findOne({ ...importJobState }, options);
+      }
+      if (importJobState && queueItemState) {
+        logger.silly(`Checking DB ${collection} for ${queueItemState} and ${JSON.stringify(importJobState)}`);
+        return operator.findOne({ ...cleanQueueItemState, ...importJobState }, options);
+      }
+      logger.debug(`getOne not working!`);
+      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, "Missing parameters, cannot get queueItem");
+    } catch (error) {
+      const errorMessage = error.payload || error.message || "";
+      logError(error);
+      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);
+    }
+  }
+  function getStream(correlationId) {
+    logger.info(`Forming stream from mongo for ${correlationId} in ${collection}`);
+    const clean = sanitize(correlationId);
+    try {
+      return gridFSBucket.openDownloadStreamByName(clean);
+    } catch (error) {
+      const errorMessage = error.payload || error.message || "";
+      logError(error);
+      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);
+    }
+  }
+  async function pushIds({ correlationId, handledIds, rejectedIds }) {
+    logger.verbose(`Push ids (${handledIds.length}) and rejectedIds (${rejectedIds.length}) ${correlationId} to ${collection}`);
+    logger.debug(`ids (${handledIds.length}): ${JSON.stringify(handledIds)}, rejectedIds ${rejectedIds.length}: ${JSON.stringify(rejectedIds)}`);
+    const clean = sanitize(correlationId);
+    await operator.updateOne({
+      correlationId: clean
+    }, {
+      $set: {
+        modificationTime: moment().toDate()
+      },
+      $push: {
+        handledIds: { $each: handledIds },
+        rejectedIds: { $each: rejectedIds }
+      }
+    });
+  }
+  async function pushMessages({ correlationId, messages, messageField = "messages" }) {
+    logger.verbose(`Push messages (${messages.length}) to ${messageField} ${correlationId} to ${collection}`);
+    logger.silly(`Messages (${messages.length}): ${JSON.stringify(messages)}}`);
+    const clean = sanitize(correlationId);
+    const cleanMessageField = sanitize(messageField);
+    debug(`${messageField} -> ${cleanMessageField}`);
+    await operator.updateOne({
+      correlationId: clean
+    }, {
+      $set: {
+        modificationTime: moment().toDate()
+      },
+      $push: {
+        [cleanMessageField]: { $each: messages }
+      }
+    });
+  }
+  function setState({ correlationId, state, errorMessage = void 0, errorStatus = void 0 }) {
+    const errorString = errorMessage || errorStatus ? `, Error message: '${JSON.stringify(errorMessage) || ""}', Error status: '${errorStatus || ""}'` : "";
+    logger.info(`Setting queue-item state ${state} for ${correlationId}${errorString} to ${collection}`);
+    const stateInQueueItemStates = Object.values(QUEUE_ITEM_STATE).indexOf(state) > -1;
+    const stateInQueueItemStatesValidator = Object.values(QUEUE_ITEM_STATE.VALIDATOR).indexOf(state) > -1;
+    const stateInQueueItemStatesImporter = Object.values(QUEUE_ITEM_STATE.IMPORTER).indexOf(state) > -1;
+    if (!stateInQueueItemStates && !stateInQueueItemStatesValidator && !stateInQueueItemStatesImporter) {
+      debug(`Trying to set invalid state ${state}`);
+      throw new ApiError(httpStatus.BAD_REQUEST, "Trying to set invalid state");
+    }
+    const clean = sanitize(correlationId);
+    const updateValues = {
+      queueItemState: state,
+      modificationTime: moment().toDate(),
+      errorMessage,
+      errorStatus
+    };
+    Object.keys(updateValues).forEach((key) => updateValues[key] === void 0 && delete updateValues[key]);
+    return operator.findOneAndUpdate(
+      { correlationId: clean },
+      { $set: updateValues },
+      { projection: { _id: 0 }, returnDocument: "after" }
+    );
+  }
+  function setImportJobState({ correlationId, operation, importJobState }) {
+    logger.info(`Setting queue-item importJobState: {${operation}: ${importJobState}} for ${correlationId}`);
+    debug(`Setting queue-item importJobState: {${operation}: ${importJobState}} for ${correlationId}`);
+    const cleanCorrelationId = sanitize(correlationId);
+    const cleanImportJobState = sanitize(importJobState);
+    if (!(operation in OPERATIONS)) {
+      debug(`Error: ${operation} is not in ${JSON.stringify(OPERATIONS)}`);
+      throw new ApiError(httpStatus.BAD_REQUEST, "Invalid operation for import job state");
+    }
+    if (!(cleanImportJobState in IMPORT_JOB_STATE)) {
+      debug(`Error: ${cleanImportJobState} is not in ${JSON.stringify(IMPORT_JOB_STATE)}`);
+      throw new ApiError(httpStatus.BAD_REQUEST, "Invalid import job state");
+    }
+    const newJobState = getNewJobState(operation, cleanImportJobState);
+    function getNewJobState(operation2, cleanImportJobState2) {
+      if (operation2 === OPERATIONS.CREATE) {
+        return { "importJobState.CREATE": cleanImportJobState2 };
+      }
+      if (operation2 === OPERATIONS.UPDATE) {
+        return { "importJobState.UPDATE": cleanImportJobState2 };
+      }
+      if (operation2 === OPERATIONS.FIX) {
+        return { "importJobState.FIX": cleanImportJobState2 };
+      }
+      throw new ApiError("400", "Invalid operation for import job state");
+    }
+    return operator.findOneAndUpdate({
+      correlationId: cleanCorrelationId
+    }, {
+      $set: {
+        ...newJobState,
+        modificationTime: moment().toDate()
+      }
+    }, { projection: { _id: 0 }, returnDocument: "after" });
+  }
+  async function setOperation({ correlationId, operation }) {
+    const newOperation = operation;
+    const { operation: oldOperation } = await operator.findOne({ correlationId });
+    logger.info(`Setting queue-item operation from ${oldOperation} to ${newOperation} for ${correlationId} to ${collection}`);
+    const cleanCorrelationId = sanitize(correlationId);
+    const cleanNewOperation = sanitize(newOperation);
+    const result = await operator.findOneAndUpdate({
+      correlationId: cleanCorrelationId
+    }, {
+      $set: {
+        operation: cleanNewOperation,
+        modificationTime: moment().toDate()
+      }
+    }, { projection: { _id: 0 }, returnDocument: "after" });
+    debugDev(`${JSON.stringify(result)}`);
+    return result.ok || result;
+  }
+  async function setOperations({ correlationId, addOperation, removeOperation = void 0 }) {
+    const cleanCorrelationId = sanitize(correlationId);
+    const queueItem = await operator.findOne({ correlationId: cleanCorrelationId });
+    logger.silly(`We found a queueItem: ${JSON.stringify(queueItem)}`);
+    const oldOperations = queueItem.operations;
+    logger.info(`Setting queue-item operations from ${JSON.stringify(oldOperations)} by adding ${addOperation} and removing ${removeOperation} for ${correlationId} to ${collection}`);
+    const operationsAfterRemove = oldOperations.filter((operation) => operation !== removeOperation);
+    const operationsAfterRemoveAndAdd = operationsAfterRemove.includes(addOperation) ? operationsAfterRemove : [...operationsAfterRemove, addOperation];
+    logger.silly(`operationsAfterRemove: ${operationsAfterRemove}`);
+    logger.silly(`operationAfterRemoveAndAdd: ${operationsAfterRemoveAndAdd}`);
+    const result = await operator.findOneAndUpdate({
+      correlationId: cleanCorrelationId
+    }, {
+      $set: {
+        operations: operationsAfterRemoveAndAdd,
+        modificationTime: moment().toDate()
+      }
+    }, { projection: { _id: 0 }, returnDocument: "after" });
+    debugDev(`${JSON.stringify(result)}`);
+    return result.ok || result;
+  }
+  async function addBlobSize({ correlationId }) {
+    const cleanCorrelationId = sanitize(correlationId);
+    const result = await operator.findOneAndUpdate({
+      correlationId: cleanCorrelationId,
+      queueItemState: QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS
+    }, {
+      $inc: {
+        blobSize: 1
+      },
+      $set: {
+        modificationTime: moment().toDate()
+      }
+    }, { projection: { _id: 0 }, returnDocument: "after" });
+    logger.verbose(`AddBlobSizeResult in mongo: ${JSON.stringify(result)}`);
+    debugDev(`${JSON.stringify(result)}`);
+    return result;
+  }
+  async function setBlobSize({ correlationId, blobSize }) {
+    const cleanCorrelationId = sanitize(correlationId);
+    const cleanBlobSize = sanitize(blobSize);
+    const result = await operator.findOneAndUpdate({
+      correlationId: cleanCorrelationId
+    }, {
+      $set: {
+        blobSize: cleanBlobSize,
+        modificationTime: moment().toDate()
+      }
+    }, { projection: { _id: 0 }, returnDocument: "after" });
+    logger.verbose(`SetBlobSizeResult in mongo: ${JSON.stringify(result)}`);
+    debugDev(`${JSON.stringify(result)}`);
+    return result;
+  }
+}
+//# sourceMappingURL=mongo.js.map

package/dist/mongo.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/mongo.js"],
+  "sourcesContent": ["/* eslint-disable max-lines */\n\nimport {MongoClient, GridFSBucket, MongoDriverError} from 'mongodb';\nimport moment from 'moment';\nimport httpStatus from 'http-status';\nimport sanitize from 'mongo-sanitize';\nimport createDebugLogger from 'debug';\nimport {createLogger} from '@natlibfi/melinda-backend-commons';\nimport {Error as ApiError, parseBoolean} from '@natlibfi/melinda-commons';\nimport {QUEUE_ITEM_STATE, IMPORT_JOB_STATE, OPERATIONS} from './constants.js';\nimport {logError} from './utils.js';\n\n//import isDeepStrictEqual from 'util';\n\n/* QueueItem:\n{\n  \"correlationId\":\"FOO\",\n  \"cataloger\":\"xxx0000\",\n  \"oCatalogerIn\":\"xxx0000\"\n  \"operation\":\"UPDATE\",\n  \"operations\":[\"UPDATE\", \"CREATE\"]\n  \"contentType\":\"application/json\",\n  \"recordLoadParams\": {\n    \"pActiveLibrary\": \"XXX00\",\n    \"pInputFile\": \"filename.seq\",\n    \"pRejectFile\": \"filename.rej\",\n    \"pLogFile\": \"filename.syslog\",\n    \"pOldNew\": \"NEW\"\n  },\n  \"queueItemState\":\"PENDING_QUEUING\",\n  \"importJobState\": {\n    \"CREATE\": \"EMPTY\",\n    \"UPDATE\": \"EMPTY\"\n  },\n  blobSize: 1,\n  records: [],\n  errorMessage: '',\n  errorStatus: '',\n  \"creationTime\":\"2020-01-01T00:00:00.000Z\",\n  \"modificationTime\":\"2020-01-01T00:00:01.000Z\",\n}\n*/\n\nexport default async function (MONGO_URI, collection, db = 'rest-api') {\n  const logger = createLogger();\n  const debug = createDebugLogger('@natlibfi/melinda-rest-api-commons/mongo');\n  const debugDev = debug.extend(':dev');\n\n  // Connect to mongo (MONGO)\n  const client = await MongoClient.connect(MONGO_URI);\n  const dbConnection = client.db(db);\n  const gridFSBucket = new GridFSBucket(dbConnection, {bucketName: collection});\n  const operator = dbConnection.collection(collection);\n  debugDev(`mongo: ${MONGO_URI}, db: ${db}, collection: ${collection}`);\n\n  return {createPrio, createBulk, checkAndSetState, checkAndSetImportJobState, query, createProjection, queryById, checkTimeOut, remove, readContent, removeContent, getOne, getStream, setState, setImportJobState, pushIds, pushMessages, setOperation, setOperations, addBlobSize, setBlobSize};\n\n  async function createPrio({correlationId, cataloger, oCatalogerIn, operation, operationSettings}) {\n    debugDev('createPrio');\n    const time = moment().toDate();\n    const newQueueItem = {\n      correlationId,\n      cataloger,\n      operation,\n      operations: [operation],\n      oCatalogerIn,\n      operationSettings: {\n        prio: true,\n        ...operationSettings\n      },\n      queueItemState: QUEUE_ITEM_STATE.VALIDATOR.PENDING_VALIDATION,\n      blobSize: 1,\n      importJobState: {\n        CREATE: IMPORT_JOB_STATE.EMPTY,\n        UPDATE: IMPORT_JOB_STATE.EMPTY,\n        FIX: IMPORT_JOB_STATE.EMPTY\n      },\n      records: [],\n      errorMessage: '',\n      errorStatus: '',\n      creationTime: time,\n      modificationTime: time\n    };\n    try {\n      debugDev(`Inserting: ${JSON.stringify(newQueueItem)} to ${collection}}`);\n      const result = await operator.insertOne(newQueueItem);\n      debugDev(JSON.stringify(result));\n      if (result.acknowledged) {\n        logger.info(`New PRIO queue item for ${operation} ${correlationId} has been made in ${collection}`);\n        return;\n      }\n      // TEST-COVERAGE: this error is not (currently) tested\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n    } catch (error) {\n      // TEST-COVERAGE: this error is not (currently) tested\n      const errorMessage = error.payload || error.message || '';\n      logError(error);\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);\n    }\n  }\n\n  async function createBulk({correlationId, cataloger, oCatalogerIn, operation, contentType, recordLoadParams, stream, operationSettings}) {\n    const time = moment().toDate();\n    const newQueueItem = {\n      correlationId,\n      cataloger,\n      oCatalogerIn,\n      operation,\n      operations: [operation],\n      operationSettings: {\n        prio: false,\n        ...operationSettings\n      },\n      contentType,\n      recordLoadParams,\n      queueItemState: stream ? QUEUE_ITEM_STATE.VALIDATOR.UPLOADING : QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS,\n      blobSize: 0,\n      importJobState: {\n        CREATE: IMPORT_JOB_STATE.EMPTY,\n        UPDATE: IMPORT_JOB_STATE.EMPTY,\n        FIX: IMPORT_JOB_STATE.EMPTY\n      },\n      records: [],\n      errorMessage: '',\n      errorStatus: '',\n      creationTime: time,\n      modificationTime: time\n    };\n\n    if (stream) {\n      debug(`Creating streamBulk`);\n      try {\n        await new Promise((resolve, reject) => {\n          const outputStream = gridFSBucket.openUploadStream(correlationId);\n\n          stream\n            .on('error', reject)\n            .on('data', chunk => outputStream.write(chunk))\n            .on('end', () => outputStream.end(undefined, undefined, () => {\n              resolve(correlationId);\n            }));\n        });\n        logger.info(`New BULK queue item for ${operation} ${correlationId} has been made in ${collection}!`);\n        debug(`New BULK queue item for ${operation} ${correlationId} has been made in ${collection}!`);\n        return operator.insertOne(newQueueItem);\n      } catch (error) {\n        const errorMessage = error.payload || error.message || '';\n        logError(error);\n        throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);\n      }\n    }\n\n    logger.debug(`No stream`);\n    debug(`Creating noStreamBulk`);\n    if (!stream) {\n      try {\n        const result = await operator.insertOne(newQueueItem);\n        if (result.acknowledged) {\n          logger.info(`New noStream BULK queue item for ${operation} ${correlationId} has been made in ${collection}`);\n          debug(`New noStream BULK queue item for ${operation} ${correlationId} has been made in ${collection}`);\n          return {correlationId, queueItemState: QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS};\n        }\n        // TEST-COVERAGE: this error is not (currently) tested\n        throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR);\n      } catch (error) {\n        // TEST-COVERAGE: this error is not (currently) tested\n        const errorMessage = error.payload || error.message || '';\n        logError(error);\n        throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);\n      }\n    }\n  }\n\n  // Check state that the queueItem has not waited too long and set state\n  // Optionally set also queueItems errorStatus & errorMessage\n  async function checkAndSetState({correlationId, state, errorMessage = undefined, errorStatus = undefined}) {\n    // checkTimeOut returns true, if queueItem is fresher than 1 minute and it's state is not ABORT/ERROR/DONE\n    // otherwise it sets queueItem and importJobStates to state ABORT and add errorStatus and errorMessage (408, 'Timeout')\n    const timeOut = await checkTimeOut({correlationId});\n    if (timeOut) {\n      return setState({correlationId, state, errorMessage, errorStatus});\n    }\n    return false;\n  }\n\n  // Check state that the queueItem has not waited too long and set importJobState\n  // Note: checkAndSetImportJobState cannot be used to set queueItems errorStatus & errorMessage (expect in case of a TimeOut)\n  async function checkAndSetImportJobState({correlationId, operation, importJobState}) {\n    // checkTimeOut returns true, if queueItem is fresher than 1 minute and it's state is not ABORT/ERROR/DONE\n    // otherwise it sets queueItem and importJobStates to state ABORT and errorStatue and errorMessage (408, 'Timeout')\n    logger.debug(`${correlationId}, ${importJobState}, ${operation}`);\n    const timeOut = await checkTimeOut({correlationId, operation, importJobState});\n    if (timeOut) {\n      return setImportJobState({correlationId, operation, importJobState});\n    }\n    return false;\n  }\n\n  async function query(params, showParams = {}) {\n    logger.debug(`Querying: ${JSON.stringify(params)}, ${JSON.stringify(showParams)}`);\n    debug(`Querying: ${JSON.stringify(params)}, ${JSON.stringify(showParams)}`);\n    const {limit = 1000, skip = 0, ...rest} = params;\n\n    const result = await operator.find(rest, {projection: createProjection(showParams)})\n      .limit(parseInt(limit, 10))\n      .skip(parseInt(skip, 10))\n      .toArray();\n    logger.debug(`Query result: ${result.length > 0 ? 'Found!' : 'Not found!'}`);\n    logger.silly(`${JSON.stringify(result)}`);\n    debug(`${JSON.stringify(result)}`);\n    return result;\n  }\n\n  // We default to NOT showing operations, operationSettings, recordLoadParams and importJobState when a queueItem is queried\n  // These can all be shown with showAll\n  // Single fields can be shown by showOperation, showOperationSettings, showRecordLoadParams and showImportJobState\n\n  function createProjection(showParams = {}) {\n    logger.silly(`Creating projection for query: ${JSON.stringify(showParams)}`);\n    debug(`Creating projection for query: ${JSON.stringify(showParams)}`);\n    const {showAll = 0, showOperations = 0, showOperationSettings = 0, showRecordLoadParams = 0, showImportJobState = 0} = showParams;\n    const combinedShowParams = {\n      showAll,\n      showOperations,\n      showOperationSettings,\n      showRecordLoadParams,\n      showImportJobState\n    };\n    logger.debug(`showAll: ${showAll}, showOperations: ${showOperations}, showOperationSettings: ${showOperationSettings}, showRecordLoadParams: ${showRecordLoadParams}, showImportJobState: ${showImportJobState}`);\n    debug(`showAll: ${showAll}, showOperations: ${showOperations}, showOperationSettings: ${showOperationSettings}, showRecordLoadParams: ${showRecordLoadParams}, showImportJobState: ${showImportJobState}`);\n    debug(`${JSON.stringify(combinedShowParams)}`);\n\n    // NOTE: parseBoolean parses any non-empty, non-\"false\" string as true!\n    if (parseBoolean(showAll)) {\n      //if (showAll === true || showAll === '1' || showAll === 1) {\n      debug(`ShowAll: ${showAll}`);\n      return {\n        _id: 0\n      };\n    }\n\n    const showParamToField = {\n      'showOperations': 'operations',\n      'showOperationSettings': 'operationSettings',\n      'showRecordLoadParams': 'recordLoadParams',\n      'showImportJobState': 'importJobState'\n    };\n\n    const result = Object.keys(combinedShowParams)\n      //.filter(param => param !== 'showAll' && combinedShowParams[param] !== true && combinedShowParams[param] !== 1 && combinedShowParams[param] !== '1')\n      .filter(param => param !== 'showAll' && !parseBoolean(combinedShowParams[param]))\n      .filter(param => showParamToField[param])\n      .map((param) => showParamToField[param]);\n    logger.silly(`We want to exclude from projection: ${JSON.stringify(result)}`);\n    debug(`We want to exclude from projection: ${JSON.stringify(result)}`);\n\n\n    const excludeObject = Object.fromEntries(result.map(param => [param, 0]));\n    logger.silly(`We want to exclude from projection: ${JSON.stringify(excludeObject)}`);\n    debug(`We want to exclude from projection: ${JSON.stringify(excludeObject)}`);\n\n\n    return {\n      _id: 0,\n      ...excludeObject\n    };\n  }\n\n  // Note: edited in v4.2.4 to return queueItem *after* possible timout has aborted it\n  // Note: previous versions returned queueItem *before* its state/queueItemState is timeOuted\n  async function queryById({correlationId, checkModTime = false}) {\n    if (checkModTime) {\n      debug(`queryById checkModTime: ${checkModTime}`);\n      await checkTimeOut({correlationId});\n      return queryById({correlationId});\n    }\n    const result = await operator.findOne({correlationId}, {projection: {_id: 0}});\n    return result;\n  }\n\n  // Check that if the item has waited too long\n  // If the last modification time for the queueItem is older than 1 minute\n  // set state to ABORT and return false, otherwise return true\n  // If the state is already ABORT or ERROR or DONE return false\n\n  async function checkTimeOut({correlationId}) {\n    const {modificationTime, queueItemState: oldState, importJobState} = await operator.findOne({correlationId});\n    //debug(`${modificationTime} - oldState: ${oldState}`);\n    // should we check for DONE too?\n    if ([QUEUE_ITEM_STATE.ABORT, QUEUE_ITEM_STATE.ERROR, QUEUE_ITEM_STATE.DONE].includes(oldState)) {\n      logger.silly(`${correlationId} has already state: ${oldState}`);\n      return false;\n    }\n\n    const timeoutTime = moment(modificationTime).add(1, 'm');\n    logger.silly(`${correlationId} timeOut @ ${timeoutTime}`);\n\n    if (timeoutTime.isBefore()) {\n      const finalImportJobStates = [IMPORT_JOB_STATE.ABORT, IMPORT_JOB_STATE.DONE, IMPORT_JOB_STATE.EMPTY, IMPORT_JOB_STATE.ERROR];\n      if (!finalImportJobStates.includes(importJobState.CREATE)) {\n        await setImportJobState({correlationId, operation: 'CREATE', importJobState: IMPORT_JOB_STATE.ABORT});\n      }\n\n      if (!finalImportJobStates.includes(importJobState.UPDATE)) {\n        await setImportJobState({correlationId, operation: 'UPDATE', importJobState: IMPORT_JOB_STATE.ABORT});\n      }\n\n      if (!finalImportJobStates.includes(importJobState.FIX)) {\n        await setImportJobState({correlationId, operation: 'FIX', importJobState: IMPORT_JOB_STATE.ABORT});\n      }\n\n      await setState({correlationId, state: QUEUE_ITEM_STATE.ABORT, errorStatus: httpStatus.REQUEST_TIMEOUT, errorMessage: `Timeout in ${oldState}`});\n\n      return false;\n    }\n\n    return true;\n  }\n\n  async function remove(params) {\n    logger.silly(`${JSON.stringify(params)}`);\n    logger.info(`Removing from Mongo (${collection}) id: ${params.correlationId}`);\n    debug(`${JSON.stringify(params)}`);\n    debug(`Removing from Mongo (${collection}) id: ${params.correlationId}`);\n    const clean = sanitize(params.correlationId);\n    logger.silly(`mongo/remove: clean: ${JSON.stringify(clean)}`);\n    debug(`mongo/remove: clean: ${JSON.stringify(clean)}`);\n\n\n    try {\n      //const metadataResult = await getFileMetadata({filename: clean});\n      //logger.debug(`mongo/remove: metadataResult: ${JSON.stringify(metadataResult)}`);\n      const noContent = await removeContent({correlationId: params.correlationId});\n      debug(`noContent (removeContent result): ${JSON.stringify(noContent)}`);\n      if (noContent) {\n        const deleteResult = await operator.deleteOne({correlationId: clean});\n        debug(`deleteResult: ${JSON.stringify(deleteResult)}`);\n        return true;\n      }\n    } catch (err) {\n      // TEST-COVERAGE: this error is not (currently) tested\n      if (err instanceof MongoDriverError) {\n        debug(err);\n        if (err.message.indexOf('File not found for id') !== -1) {\n          logger.silly(`mongo/remove: File not found, removing queueItem ${JSON.stringify(clean)} from ${collection}`);\n          debug(`mongo/remove: File not found, removing queueItem ${JSON.stringify(clean)} from ${collection}`);\n          await operator.deleteOne({correlationId: clean});\n          return true;\n        }\n        logger.error(err.message);\n        throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n      }\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, err.message);\n    }\n  }\n\n  async function readContent(correlationId) {\n    logger.info(`Reading content from mongo for id: ${correlationId} in ${collection}`);\n    debug(`Reading content from mongo for id: ${correlationId} in ${collection}`);\n\n    const clean = sanitize(correlationId);\n    debug(clean);\n\n    const result = await operator.findOne({correlationId: clean}); // njsscan-ignore: node_nosqli_injection\n    debug(result);\n    if (result) {\n      const {operationSettings} = result;\n      if (operationSettings.prio === false && operationSettings.noStream === false) {\n        return {\n          contentType: result.contentType,\n          readStream: gridFSBucket.openDownloadStreamByName(clean)\n        };\n      }\n      logger.debug(`OperationSettings for ${correlationId}: ${JSON.stringify(operationSettings)}`);\n      debug(`OperationSettings for ${correlationId}: ${JSON.stringify(operationSettings)}`);\n      debug(`Content is only available for streamBulk jobs. ${correlationId} is not a streamBulk job.`);\n      throw new ApiError(httpStatus.BAD_REQUEST, {message: `Content is only available for streamBulk jobs. ${correlationId} is not a streamBulk job.`});\n    }\n\n    throw new ApiError(httpStatus.NOT_FOUND, {message: `Job ${correlationId} not found.`});\n  }\n\n  async function removeContent(params) {\n    logger.info(`Removing content from mongo for id: ${params.correlationId} in ${collection}`);\n    debug(`Removing content from mongo for id: ${params.correlationId} in ${collection}`);\n    const clean = sanitize(params.correlationId);\n\n    const result = await gridFSBucket.find({filename: clean}).toArray();\n    debug(`mongo/removeContent: find bucket result ${JSON.stringify(result)}`);\n\n\n    if (result.length > 0) {\n      // Note: we assume and handle just one file for filename!\n      const [firstBucketResult] = result;\n      const fileId = firstBucketResult._id;\n\n      await gridFSBucket.delete(fileId);\n      return true;\n    }\n\n    logger.debug(`No files for ${params.correlationId} to delete`);\n    return true;\n  }\n\n  function getOne({queueItemState, importJobState = undefined}) {\n    //params example: \"queueItemState\": \"IMPORTER.IN_QUEUE\", \"importJobState\":{\"importJobState.UPDATE\": \"IN_QUEUE\"}\n    logger.silly(`queueItemState: ${queueItemState}, importJobState: ${JSON.stringify(importJobState)}`);\n    //debug(`queueItemState: ${queueItemState}, importJobState: ${JSON.stringify(importJobState)}`);\n    const cleanQueueItemState = queueItemState ? {queueItemState: sanitize(queueItemState)} : undefined;\n    const options = {projection: {_id: 0}};\n\n    try {\n\n      // Just queueItemState\n      if (queueItemState && importJobState === undefined) {\n        logger.silly(`Checking DB ${collection} for just ${JSON.stringify(cleanQueueItemState.queueItemState)}`);\n        //debug(`Checking DB ${collection} for just ${JSON.stringify(cleanQueueItemState.queueItemState)}`);\n        return operator.findOne({...cleanQueueItemState}, options);\n      }\n\n      // importJobState\n      if (importJobState && queueItemState === undefined) {\n        logger.silly(`Checking DB ${collection} for ${JSON.stringify(importJobState)}`);\n        //debug(`Checking DB ${collection} for ${JSON.stringify(importJobState)}`);\n        return operator.findOne({...importJobState}, options);\n      }\n\n      // importJobState and queueItemState\n      if (importJobState && queueItemState) {\n        logger.silly(`Checking DB ${collection} for ${queueItemState} and ${JSON.stringify(importJobState)}`);\n        //debug(`Checking DB ${collection} for ${queueItemState} and ${JSON.stringify(importJobState)}`);\n        return operator.findOne({...cleanQueueItemState, ...importJobState}, options);\n      }\n\n      logger.debug(`getOne not working!`);\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, 'Missing parameters, cannot get queueItem');\n\n    } catch (error) {\n      const errorMessage = error.payload || error.message || '';\n      logError(error);\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);\n    }\n  }\n\n  function getStream(correlationId) {\n    logger.info(`Forming stream from mongo for ${correlationId} in ${collection}`);\n    const clean = sanitize(correlationId);\n    try {\n      // Return content stream\n      return gridFSBucket.openDownloadStreamByName(clean);\n    } catch (error) {\n      // TEST-COVERAGE: this error is not (currently) tested\n      const errorMessage = error.payload || error.message || '';\n      logError(error);\n      throw new ApiError(httpStatus.INTERNAL_SERVER_ERROR, `Mongo errored: ${errorMessage}`);\n    }\n  }\n\n  async function pushIds({correlationId, handledIds, rejectedIds}) {\n    logger.verbose(`Push ids (${handledIds.length}) and rejectedIds (${rejectedIds.length}) ${correlationId} to ${collection}`);\n    logger.debug(`ids (${handledIds.length}): ${JSON.stringify(handledIds)}, rejectedIds ${rejectedIds.length}: ${JSON.stringify(rejectedIds)}`);\n    const clean = sanitize(correlationId);\n    await operator.updateOne({\n      correlationId: clean\n    }, {\n      $set: {\n        modificationTime: moment().toDate()\n      },\n      $push: {\n        handledIds: {$each: handledIds},\n        rejectedIds: {$each: rejectedIds}\n      }\n    });\n  }\n\n  async function pushMessages({correlationId, messages, messageField = 'messages'}) {\n    logger.verbose(`Push messages (${messages.length}) to ${messageField} ${correlationId} to ${collection}`);\n    logger.silly(`Messages (${messages.length}): ${JSON.stringify(messages)}}`);\n    const clean = sanitize(correlationId);\n    const cleanMessageField = sanitize(messageField);\n    debug(`${messageField} -> ${cleanMessageField}`);\n    await operator.updateOne({\n      correlationId: clean\n    }, {\n      $set: {\n        modificationTime: moment().toDate()\n      },\n      $push: {\n        [cleanMessageField]: {$each: messages}\n      }\n    });\n  }\n\n  function setState({correlationId, state, errorMessage = undefined, errorStatus = undefined}) {\n    const errorString = errorMessage || errorStatus ? `, Error message: '${JSON.stringify(errorMessage) || ''}', Error status: '${errorStatus || ''}'` : '';\n    logger.info(`Setting queue-item state ${state} for ${correlationId}${errorString} to ${collection}`);\n\n    const stateInQueueItemStates = Object.values(QUEUE_ITEM_STATE).indexOf(state) > -1;\n    const stateInQueueItemStatesValidator = Object.values(QUEUE_ITEM_STATE.VALIDATOR).indexOf(state) > -1;\n    const stateInQueueItemStatesImporter = Object.values(QUEUE_ITEM_STATE.IMPORTER).indexOf(state) > -1;\n    if (!stateInQueueItemStates && !stateInQueueItemStatesValidator && !stateInQueueItemStatesImporter) {\n      debug(`Trying to set invalid state ${state}`);\n      throw new ApiError(httpStatus.BAD_REQUEST, 'Trying to set invalid state');\n    }\n\n    const clean = sanitize(correlationId);\n    const updateValues = {\n      queueItemState: state,\n      modificationTime: moment().toDate(),\n      errorMessage,\n      errorStatus\n    };\n\n    // Do not update value that are undefined\n    Object.keys(updateValues).forEach(key => updateValues[key] === undefined && delete updateValues[key]);\n\n    return operator\n      .findOneAndUpdate(\n        {correlationId: clean},\n        {$set: updateValues},\n        {projection: {_id: 0}, returnDocument: 'after'}\n      );\n  }\n\n  function setImportJobState({correlationId, operation, importJobState}) {\n\n    // should this also get importJobState as previously created object? {}\n    logger.info(`Setting queue-item importJobState: {${operation}: ${importJobState}} for ${correlationId}`);\n    debug(`Setting queue-item importJobState: {${operation}: ${importJobState}} for ${correlationId}`);\n    const cleanCorrelationId = sanitize(correlationId);\n    const cleanImportJobState = sanitize(importJobState);\n\n    if (!(operation in OPERATIONS)) {\n      debug(`Error: ${operation} is not in ${JSON.stringify(OPERATIONS)}`);\n      throw new ApiError(httpStatus.BAD_REQUEST, 'Invalid operation for import job state');\n    }\n\n    if (!(cleanImportJobState in IMPORT_JOB_STATE)) {\n      debug(`Error: ${cleanImportJobState} is not in ${JSON.stringify(IMPORT_JOB_STATE)}`);\n      throw new ApiError(httpStatus.BAD_REQUEST, 'Invalid import job state');\n    }\n\n    const newJobState = getNewJobState(operation, cleanImportJobState);\n\n\n    function getNewJobState(operation, cleanImportJobState) {\n      if (operation === OPERATIONS.CREATE) {\n        return {'importJobState.CREATE': cleanImportJobState};\n      }\n      if (operation === OPERATIONS.UPDATE) {\n        return {'importJobState.UPDATE': cleanImportJobState};\n      }\n      if (operation === OPERATIONS.FIX) {\n        return {'importJobState.FIX': cleanImportJobState};\n      }\n      // istanbul ignore next\n      // Ignoring for coverage - we error this already\n      throw new ApiError('400', 'Invalid operation for import job state');\n    }\n\n    return operator.findOneAndUpdate({\n      correlationId: cleanCorrelationId\n    }, {\n      $set: {\n        ...newJobState,\n        modificationTime: moment().toDate()\n      }\n    }, {projection: {_id: 0}, returnDocument: 'after'});\n  }\n\n  async function setOperation({correlationId, operation}) {\n    const newOperation = operation;\n    const {operation: oldOperation} = await operator.findOne({correlationId});\n    logger.info(`Setting queue-item operation from ${oldOperation} to ${newOperation} for ${correlationId} to ${collection}`);\n    const cleanCorrelationId = sanitize(correlationId);\n    const cleanNewOperation = sanitize(newOperation);\n\n    const result = await operator.findOneAndUpdate({\n      correlationId: cleanCorrelationId\n    }, {\n      $set: {\n        operation: cleanNewOperation,\n        modificationTime: moment().toDate()\n      }\n    }, {projection: {_id: 0}, returnDocument: 'after'});\n    debugDev(`${JSON.stringify(result)}`);\n    return result.ok || result;\n    // logger.debug(JSON.stringify(result));\n  }\n\n  async function setOperations({correlationId, addOperation, removeOperation = undefined}) {\n    const cleanCorrelationId = sanitize(correlationId);\n\n    const queueItem = await operator.findOne({correlationId: cleanCorrelationId});\n    logger.silly(`We found a queueItem: ${JSON.stringify(queueItem)}`);\n\n    const oldOperations = queueItem.operations;\n\n    logger.info(`Setting queue-item operations from ${JSON.stringify(oldOperations)} by adding ${addOperation} and removing ${removeOperation} for ${correlationId} to ${collection}`);\n\n    const operationsAfterRemove = oldOperations.filter(operation => operation !== removeOperation);\n    const operationsAfterRemoveAndAdd = operationsAfterRemove.includes(addOperation) ? operationsAfterRemove : [...operationsAfterRemove, addOperation];\n\n    logger.silly(`operationsAfterRemove: ${operationsAfterRemove}`);\n    logger.silly(`operationAfterRemoveAndAdd: ${operationsAfterRemoveAndAdd}`);\n\n    const result = await operator.findOneAndUpdate({\n      correlationId: cleanCorrelationId\n    }, {\n      $set: {\n        operations: operationsAfterRemoveAndAdd,\n        modificationTime: moment().toDate()\n      }\n    }, {projection: {_id: 0}, returnDocument: 'after'});\n    debugDev(`${JSON.stringify(result)}`);\n    return result.ok || result;\n    // logger.debug(JSON.stringify(result));\n  }\n\n  async function addBlobSize({correlationId}) {\n    const cleanCorrelationId = sanitize(correlationId);\n\n    const result = await operator.findOneAndUpdate({\n      correlationId: cleanCorrelationId,\n      queueItemState: QUEUE_ITEM_STATE.VALIDATOR.WAITING_FOR_RECORDS\n    }, {\n      $inc: {\n        blobSize: 1\n      },\n      $set: {\n        modificationTime: moment().toDate()\n      }\n    }, {projection: {_id: 0}, returnDocument: 'after'});\n\n    logger.verbose(`AddBlobSizeResult in mongo: ${JSON.stringify(result)}`);\n    debugDev(`${JSON.stringify(result)}`);\n    return result;\n  }\n\n  async function setBlobSize({correlationId, blobSize}) {\n    const cleanCorrelationId = sanitize(correlationId);\n    const cleanBlobSize = sanitize(blobSize);\n\n    const result = await operator.findOneAndUpdate({\n      correlationId: cleanCorrelationId\n    }, {\n      $set: {\n        blobSize: cleanBlobSize,\n        modificationTime: moment().toDate()\n      }\n    }, {projection: {_id: 0}, returnDocument: 'after'});\n\n    logger.verbose(`SetBlobSizeResult in mongo: ${JSON.stringify(result)}`);\n    debugDev(`${JSON.stringify(result)}`);\n    return result;\n  }\n\n}\n"],
+  "mappings": "AAEA,SAAQ,aAAa,cAAc,wBAAuB;AAC1D,OAAO,YAAY;AACnB,OAAO,gBAAgB;AACvB,OAAO,cAAc;AACrB,OAAO,uBAAuB;AAC9B,SAAQ,oBAAmB;AAC3B,SAAQ,SAAS,UAAU,oBAAmB;AAC9C,SAAQ,kBAAkB,kBAAkB,kBAAiB;AAC7D,SAAQ,gBAAe;AAiCvB,8BAA+B,WAAW,YAAY,KAAK,YAAY;AACrE,QAAM,SAAS,aAAa;AAC5B,QAAM,QAAQ,kBAAkB,0CAA0C;AAC1E,QAAM,WAAW,MAAM,OAAO,MAAM;AAGpC,QAAM,SAAS,MAAM,YAAY,QAAQ,SAAS;AAClD,QAAM,eAAe,OAAO,GAAG,EAAE;AACjC,QAAM,eAAe,IAAI,aAAa,cAAc,EAAC,YAAY,WAAU,CAAC;AAC5E,QAAM,WAAW,aAAa,WAAW,UAAU;AACnD,WAAS,UAAU,SAAS,SAAS,EAAE,iBAAiB,UAAU,EAAE;AAEpE,SAAO,EAAC,YAAY,YAAY,kBAAkB,2BAA2B,OAAO,kBAAkB,WAAW,cAAc,QAAQ,aAAa,eAAe,QAAQ,WAAW,UAAU,mBAAmB,SAAS,cAAc,cAAc,eAAe,aAAa,YAAW;AAE/R,iBAAe,WAAW,EAAC,eAAe,WAAW,cAAc,WAAW,kBAAiB,GAAG;AAChG,aAAS,YAAY;AACrB,UAAM,OAAO,OAAO,EAAE,OAAO;AAC7B,UAAM,eAAe;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY,CAAC,SAAS;AAAA,MACtB;AAAA,MACA,mBAAmB;AAAA,QACjB,MAAM;AAAA,QACN,GAAG;AAAA,MACL;AAAA,MACA,gBAAgB,iBAAiB,UAAU;AAAA,MAC3C,UAAU;AAAA,MACV,gBAAgB;AAAA,QACd,QAAQ,iBAAiB;AAAA,QACzB,QAAQ,iBAAiB;AAAA,QACzB,KAAK,iBAAiB;AAAA,MACxB;AAAA,MACA,SAAS,CAAC;AAAA,MACV,cAAc;AAAA,MACd,aAAa;AAAA,MACb,cAAc;AAAA,MACd,kBAAkB;AAAA,IACpB;AACA,QAAI;AACF,eAAS,cAAc,KAAK,UAAU,YAAY,CAAC,OAAO,UAAU,GAAG;AACvE,YAAM,SAAS,MAAM,SAAS,UAAU,YAAY;AACpD,eAAS,KAAK,UAAU,MAAM,CAAC;AAC/B,UAAI,OAAO,cAAc;AACvB,eAAO,KAAK,2BAA2B,SAAS,IAAI,aAAa,qBAAqB,UAAU,EAAE;AAClG;AAAA,MACF;AAEA,YAAM,IAAI,SAAS,WAAW,qBAAqB;AAAA,IACrD,SAAS,OAAO;AAEd,YAAM,eAAe,MAAM,WAAW,MAAM,WAAW;AACvD,eAAS,KAAK;AACd,YAAM,IAAI,SAAS,WAAW,uBAAuB,kBAAkB,YAAY,EAAE;AAAA,IACvF;AAAA,EACF;AAEA,iBAAe,WAAW,EAAC,eAAe,WAAW,cAAc,WAAW,aAAa,kBAAkB,QAAQ,kBAAiB,GAAG;AACvI,UAAM,OAAO,OAAO,EAAE,OAAO;AAC7B,UAAM,eAAe;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY,CAAC,SAAS;AAAA,MACtB,mBAAmB;AAAA,QACjB,MAAM;AAAA,QACN,GAAG;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA,gBAAgB,SAAS,iBAAiB,UAAU,YAAY,iBAAiB,UAAU;AAAA,MAC3F,UAAU;AAAA,MACV,gBAAgB;AAAA,QACd,QAAQ,iBAAiB;AAAA,QACzB,QAAQ,iBAAiB;AAAA,QACzB,KAAK,iBAAiB;AAAA,MACxB;AAAA,MACA,SAAS,CAAC;AAAA,MACV,cAAc;AAAA,MACd,aAAa;AAAA,MACb,cAAc;AAAA,MACd,kBAAkB;AAAA,IACpB;AAEA,QAAI,QAAQ;AACV,YAAM,qBAAqB;AAC3B,UAAI;AACF,cAAM,IAAI,QAAQ,CAAC,SAAS,WAAW;AACrC,gBAAM,eAAe,aAAa,iBAAiB,aAAa;AAEhE,iBACG,GAAG,SAAS,MAAM,EAClB,GAAG,QAAQ,WAAS,aAAa,MAAM,KAAK,CAAC,EAC7C,GAAG,OAAO,MAAM,aAAa,IAAI,QAAW,QAAW,MAAM;AAC5D,oBAAQ,aAAa;AAAA,UACvB,CAAC,CAAC;AAAA,QACN,CAAC;AACD,eAAO,KAAK,2BAA2B,SAAS,IAAI,aAAa,qBAAqB,UAAU,GAAG;AACnG,cAAM,2BAA2B,SAAS,IAAI,aAAa,qBAAqB,UAAU,GAAG;AAC7F,eAAO,SAAS,UAAU,YAAY;AAAA,MACxC,SAAS,OAAO;AACd,cAAM,eAAe,MAAM,WAAW,MAAM,WAAW;AACvD,iBAAS,KAAK;AACd,cAAM,IAAI,SAAS,WAAW,uBAAuB,kBAAkB,YAAY,EAAE;AAAA,MACvF;AAAA,IACF;AAEA,WAAO,MAAM,WAAW;AACxB,UAAM,uBAAuB;AAC7B,QAAI,CAAC,QAAQ;AACX,UAAI;AACF,cAAM,SAAS,MAAM,SAAS,UAAU,YAAY;AACpD,YAAI,OAAO,cAAc;AACvB,iBAAO,KAAK,oCAAoC,SAAS,IAAI,aAAa,qBAAqB,UAAU,EAAE;AAC3G,gBAAM,oCAAoC,SAAS,IAAI,aAAa,qBAAqB,UAAU,EAAE;AACrG,iBAAO,EAAC,eAAe,gBAAgB,iBAAiB,UAAU,oBAAmB;AAAA,QACvF;AAEA,cAAM,IAAI,SAAS,WAAW,qBAAqB;AAAA,MACrD,SAAS,OAAO;AAEd,cAAM,eAAe,MAAM,WAAW,MAAM,WAAW;AACvD,iBAAS,KAAK;AACd,cAAM,IAAI,SAAS,WAAW,uBAAuB,kBAAkB,YAAY,EAAE;AAAA,MACvF;AAAA,IACF;AAAA,EACF;AAIA,iBAAe,iBAAiB,EAAC,eAAe,OAAO,eAAe,QAAW,cAAc,OAAS,GAAG;AAGzG,UAAM,UAAU,MAAM,aAAa,EAAC,cAAa,CAAC;AAClD,QAAI,SAAS;AACX,aAAO,SAAS,EAAC,eAAe,OAAO,cAAc,YAAW,CAAC;AAAA,IACnE;AACA,WAAO;AAAA,EACT;AAIA,iBAAe,0BAA0B,EAAC,eAAe,WAAW,eAAc,GAAG;AAGnF,WAAO,MAAM,GAAG,aAAa,KAAK,cAAc,KAAK,SAAS,EAAE;AAChE,UAAM,UAAU,MAAM,aAAa,EAAC,eAAe,WAAW,eAAc,CAAC;AAC7E,QAAI,SAAS;AACX,aAAO,kBAAkB,EAAC,eAAe,WAAW,eAAc,CAAC;AAAA,IACrE;AACA,WAAO;AAAA,EACT;AAEA,iBAAe,MAAM,QAAQ,aAAa,CAAC,GAAG;AAC5C,WAAO,MAAM,aAAa,KAAK,UAAU,MAAM,CAAC,KAAK,KAAK,UAAU,UAAU,CAAC,EAAE;AACjF,UAAM,aAAa,KAAK,UAAU,MAAM,CAAC,KAAK,KAAK,UAAU,UAAU,CAAC,EAAE;AAC1E,UAAM,EAAC,QAAQ,KAAM,OAAO,GAAG,GAAG,KAAI,IAAI;AAE1C,UAAM,SAAS,MAAM,SAAS,KAAK,MAAM,EAAC,YAAY,iBAAiB,UAAU,EAAC,CAAC,EAChF,MAAM,SAAS,OAAO,EAAE,CAAC,EACzB,KAAK,SAAS,MAAM,EAAE,CAAC,EACvB,QAAQ;AACX,WAAO,MAAM,iBAAiB,OAAO,SAAS,IAAI,WAAW,YAAY,EAAE;AAC3E,WAAO,MAAM,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACxC,UAAM,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACjC,WAAO;AAAA,EACT;AAMA,WAAS,iBAAiB,aAAa,CAAC,GAAG;AACzC,WAAO,MAAM,kCAAkC,KAAK,UAAU,UAAU,CAAC,EAAE;AAC3E,UAAM,kCAAkC,KAAK,UAAU,UAAU,CAAC,EAAE;AACpE,UAAM,EAAC,UAAU,GAAG,iBAAiB,GAAG,wBAAwB,GAAG,uBAAuB,GAAG,qBAAqB,EAAC,IAAI;AACvH,UAAM,qBAAqB;AAAA,MACzB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AACA,WAAO,MAAM,YAAY,OAAO,qBAAqB,cAAc,4BAA4B,qBAAqB,2BAA2B,oBAAoB,yBAAyB,kBAAkB,EAAE;AAChN,UAAM,YAAY,OAAO,qBAAqB,cAAc,4BAA4B,qBAAqB,2BAA2B,oBAAoB,yBAAyB,kBAAkB,EAAE;AACzM,UAAM,GAAG,KAAK,UAAU,kBAAkB,CAAC,EAAE;AAG7C,QAAI,aAAa,OAAO,GAAG;AAEzB,YAAM,YAAY,OAAO,EAAE;AAC3B,aAAO;AAAA,QACL,KAAK;AAAA,MACP;AAAA,IACF;AAEA,UAAM,mBAAmB;AAAA,MACvB,kBAAkB;AAAA,MAClB,yBAAyB;AAAA,MACzB,wBAAwB;AAAA,MACxB,sBAAsB;AAAA,IACxB;AAEA,UAAM,SAAS,OAAO,KAAK,kBAAkB,EAE1C,OAAO,WAAS,UAAU,aAAa,CAAC,aAAa,mBAAmB,KAAK,CAAC,CAAC,EAC/E,OAAO,WAAS,iBAAiB,KAAK,CAAC,EACvC,IAAI,CAAC,UAAU,iBAAiB,KAAK,CAAC;AACzC,WAAO,MAAM,uCAAuC,KAAK,UAAU,MAAM,CAAC,EAAE;AAC5E,UAAM,uCAAuC,KAAK,UAAU,MAAM,CAAC,EAAE;AAGrE,UAAM,gBAAgB,OAAO,YAAY,OAAO,IAAI,WAAS,CAAC,OAAO,CAAC,CAAC,CAAC;AACxE,WAAO,MAAM,uCAAuC,KAAK,UAAU,aAAa,CAAC,EAAE;AACnF,UAAM,uCAAuC,KAAK,UAAU,aAAa,CAAC,EAAE;AAG5E,WAAO;AAAA,MACL,KAAK;AAAA,MACL,GAAG;AAAA,IACL;AAAA,EACF;AAIA,iBAAe,UAAU,EAAC,eAAe,eAAe,MAAK,GAAG;AAC9D,QAAI,cAAc;AAChB,YAAM,2BAA2B,YAAY,EAAE;AAC/C,YAAM,aAAa,EAAC,cAAa,CAAC;AAClC,aAAO,UAAU,EAAC,cAAa,CAAC;AAAA,IAClC;AACA,UAAM,SAAS,MAAM,SAAS,QAAQ,EAAC,cAAa,GAAG,EAAC,YAAY,EAAC,KAAK,EAAC,EAAC,CAAC;AAC7E,WAAO;AAAA,EACT;AAOA,iBAAe,aAAa,EAAC,cAAa,GAAG;AAC3C,UAAM,EAAC,kBAAkB,gBAAgB,UAAU,eAAc,IAAI,MAAM,SAAS,QAAQ,EAAC,cAAa,CAAC;AAG3G,QAAI,CAAC,iBAAiB,OAAO,iBAAiB,OAAO,iBAAiB,IAAI,EAAE,SAAS,QAAQ,GAAG;AAC9F,aAAO,MAAM,GAAG,aAAa,uBAAuB,QAAQ,EAAE;AAC9D,aAAO;AAAA,IACT;AAEA,UAAM,cAAc,OAAO,gBAAgB,EAAE,IAAI,GAAG,GAAG;AACvD,WAAO,MAAM,GAAG,aAAa,cAAc,WAAW,EAAE;AAExD,QAAI,YAAY,SAAS,GAAG;AAC1B,YAAM,uBAAuB,CAAC,iBAAiB,OAAO,iBAAiB,MAAM,iBAAiB,OAAO,iBAAiB,KAAK;AAC3H,UAAI,CAAC,qBAAqB,SAAS,eAAe,MAAM,GAAG;AACzD,cAAM,kBAAkB,EAAC,eAAe,WAAW,UAAU,gBAAgB,iBAAiB,MAAK,CAAC;AAAA,MACtG;AAEA,UAAI,CAAC,qBAAqB,SAAS,eAAe,MAAM,GAAG;AACzD,cAAM,kBAAkB,EAAC,eAAe,WAAW,UAAU,gBAAgB,iBAAiB,MAAK,CAAC;AAAA,MACtG;AAEA,UAAI,CAAC,qBAAqB,SAAS,eAAe,GAAG,GAAG;AACtD,cAAM,kBAAkB,EAAC,eAAe,WAAW,OAAO,gBAAgB,iBAAiB,MAAK,CAAC;AAAA,MACnG;AAEA,YAAM,SAAS,EAAC,eAAe,OAAO,iBAAiB,OAAO,aAAa,WAAW,iBAAiB,cAAc,cAAc,QAAQ,GAAE,CAAC;AAE9I,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,EACT;AAEA,iBAAe,OAAO,QAAQ;AAC5B,WAAO,MAAM,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACxC,WAAO,KAAK,wBAAwB,UAAU,SAAS,OAAO,aAAa,EAAE;AAC7E,UAAM,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACjC,UAAM,wBAAwB,UAAU,SAAS,OAAO,aAAa,EAAE;AACvE,UAAM,QAAQ,SAAS,OAAO,aAAa;AAC3C,WAAO,MAAM,wBAAwB,KAAK,UAAU,KAAK,CAAC,EAAE;AAC5D,UAAM,wBAAwB,KAAK,UAAU,KAAK,CAAC,EAAE;AAGrD,QAAI;AAGF,YAAM,YAAY,MAAM,cAAc,EAAC,eAAe,OAAO,cAAa,CAAC;AAC3E,YAAM,qCAAqC,KAAK,UAAU,SAAS,CAAC,EAAE;AACtE,UAAI,WAAW;AACb,cAAM,eAAe,MAAM,SAAS,UAAU,EAAC,eAAe,MAAK,CAAC;AACpE,cAAM,iBAAiB,KAAK,UAAU,YAAY,CAAC,EAAE;AACrD,eAAO;AAAA,MACT;AAAA,IACF,SAAS,KAAK;AAEZ,UAAI,eAAe,kBAAkB;AACnC,cAAM,GAAG;AACT,YAAI,IAAI,QAAQ,QAAQ,uBAAuB,MAAM,IAAI;AACvD,iBAAO,MAAM,oDAAoD,KAAK,UAAU,KAAK,CAAC,SAAS,UAAU,EAAE;AAC3G,gBAAM,oDAAoD,KAAK,UAAU,KAAK,CAAC,SAAS,UAAU,EAAE;AACpG,gBAAM,SAAS,UAAU,EAAC,eAAe,MAAK,CAAC;AAC/C,iBAAO;AAAA,QACT;AACA,eAAO,MAAM,IAAI,OAAO;AACxB,cAAM,IAAI,SAAS,WAAW,uBAAuB,IAAI,OAAO;AAAA,MAClE;AACA,YAAM,IAAI,SAAS,WAAW,uBAAuB,IAAI,OAAO;AAAA,IAClE;AAAA,EACF;AAEA,iBAAe,YAAY,eAAe;AACxC,WAAO,KAAK,sCAAsC,aAAa,OAAO,UAAU,EAAE;AAClF,UAAM,sCAAsC,aAAa,OAAO,UAAU,EAAE;AAE5E,UAAM,QAAQ,SAAS,aAAa;AACpC,UAAM,KAAK;AAEX,UAAM,SAAS,MAAM,SAAS,QAAQ,EAAC,eAAe,MAAK,CAAC;AAC5D,UAAM,MAAM;AACZ,QAAI,QAAQ;AACV,YAAM,EAAC,kBAAiB,IAAI;AAC5B,UAAI,kBAAkB,SAAS,SAAS,kBAAkB,aAAa,OAAO;AAC5E,eAAO;AAAA,UACL,aAAa,OAAO;AAAA,UACpB,YAAY,aAAa,yBAAyB,KAAK;AAAA,QACzD;AAAA,MACF;AACA,aAAO,MAAM,yBAAyB,aAAa,KAAK,KAAK,UAAU,iBAAiB,CAAC,EAAE;AAC3F,YAAM,yBAAyB,aAAa,KAAK,KAAK,UAAU,iBAAiB,CAAC,EAAE;AACpF,YAAM,kDAAkD,aAAa,2BAA2B;AAChG,YAAM,IAAI,SAAS,WAAW,aAAa,EAAC,SAAS,kDAAkD,aAAa,4BAA2B,CAAC;AAAA,IAClJ;AAEA,UAAM,IAAI,SAAS,WAAW,WAAW,EAAC,SAAS,OAAO,aAAa,cAAa,CAAC;AAAA,EACvF;AAEA,iBAAe,cAAc,QAAQ;AACnC,WAAO,KAAK,uCAAuC,OAAO,aAAa,OAAO,UAAU,EAAE;AAC1F,UAAM,uCAAuC,OAAO,aAAa,OAAO,UAAU,EAAE;AACpF,UAAM,QAAQ,SAAS,OAAO,aAAa;AAE3C,UAAM,SAAS,MAAM,aAAa,KAAK,EAAC,UAAU,MAAK,CAAC,EAAE,QAAQ;AAClE,UAAM,2CAA2C,KAAK,UAAU,MAAM,CAAC,EAAE;AAGzE,QAAI,OAAO,SAAS,GAAG;AAErB,YAAM,CAAC,iBAAiB,IAAI;AAC5B,YAAM,SAAS,kBAAkB;AAEjC,YAAM,aAAa,OAAO,MAAM;AAChC,aAAO;AAAA,IACT;AAEA,WAAO,MAAM,gBAAgB,OAAO,aAAa,YAAY;AAC7D,WAAO;AAAA,EACT;AAEA,WAAS,OAAO,EAAC,gBAAgB,iBAAiB,OAAS,GAAG;AAE5D,WAAO,MAAM,mBAAmB,cAAc,qBAAqB,KAAK,UAAU,cAAc,CAAC,EAAE;AAEnG,UAAM,sBAAsB,iBAAiB,EAAC,gBAAgB,SAAS,cAAc,EAAC,IAAI;AAC1F,UAAM,UAAU,EAAC,YAAY,EAAC,KAAK,EAAC,EAAC;AAErC,QAAI;AAGF,UAAI,kBAAkB,mBAAmB,QAAW;AAClD,eAAO,MAAM,eAAe,UAAU,aAAa,KAAK,UAAU,oBAAoB,cAAc,CAAC,EAAE;AAEvG,eAAO,SAAS,QAAQ,EAAC,GAAG,oBAAmB,GAAG,OAAO;AAAA,MAC3D;AAGA,UAAI,kBAAkB,mBAAmB,QAAW;AAClD,eAAO,MAAM,eAAe,UAAU,QAAQ,KAAK,UAAU,cAAc,CAAC,EAAE;AAE9E,eAAO,SAAS,QAAQ,EAAC,GAAG,eAAc,GAAG,OAAO;AAAA,MACtD;AAGA,UAAI,kBAAkB,gBAAgB;AACpC,eAAO,MAAM,eAAe,UAAU,QAAQ,cAAc,QAAQ,KAAK,UAAU,cAAc,CAAC,EAAE;AAEpG,eAAO,SAAS,QAAQ,EAAC,GAAG,qBAAqB,GAAG,eAAc,GAAG,OAAO;AAAA,MAC9E;AAEA,aAAO,MAAM,qBAAqB;AAClC,YAAM,IAAI,SAAS,WAAW,uBAAuB,0CAA0C;AAAA,IAEjG,SAAS,OAAO;AACd,YAAM,eAAe,MAAM,WAAW,MAAM,WAAW;AACvD,eAAS,KAAK;AACd,YAAM,IAAI,SAAS,WAAW,uBAAuB,kBAAkB,YAAY,EAAE;AAAA,IACvF;AAAA,EACF;AAEA,WAAS,UAAU,eAAe;AAChC,WAAO,KAAK,iCAAiC,aAAa,OAAO,UAAU,EAAE;AAC7E,UAAM,QAAQ,SAAS,aAAa;AACpC,QAAI;AAEF,aAAO,aAAa,yBAAyB,KAAK;AAAA,IACpD,SAAS,OAAO;AAEd,YAAM,eAAe,MAAM,WAAW,MAAM,WAAW;AACvD,eAAS,KAAK;AACd,YAAM,IAAI,SAAS,WAAW,uBAAuB,kBAAkB,YAAY,EAAE;AAAA,IACvF;AAAA,EACF;AAEA,iBAAe,QAAQ,EAAC,eAAe,YAAY,YAAW,GAAG;AAC/D,WAAO,QAAQ,aAAa,WAAW,MAAM,sBAAsB,YAAY,MAAM,KAAK,aAAa,OAAO,UAAU,EAAE;AAC1H,WAAO,MAAM,QAAQ,WAAW,MAAM,MAAM,KAAK,UAAU,UAAU,CAAC,iBAAiB,YAAY,MAAM,KAAK,KAAK,UAAU,WAAW,CAAC,EAAE;AAC3I,UAAM,QAAQ,SAAS,aAAa;AACpC,UAAM,SAAS,UAAU;AAAA,MACvB,eAAe;AAAA,IACjB,GAAG;AAAA,MACD,MAAM;AAAA,QACJ,kBAAkB,OAAO,EAAE,OAAO;AAAA,MACpC;AAAA,MACA,OAAO;AAAA,QACL,YAAY,EAAC,OAAO,WAAU;AAAA,QAC9B,aAAa,EAAC,OAAO,YAAW;AAAA,MAClC;AAAA,IACF,CAAC;AAAA,EACH;AAEA,iBAAe,aAAa,EAAC,eAAe,UAAU,eAAe,WAAU,GAAG;AAChF,WAAO,QAAQ,kBAAkB,SAAS,MAAM,QAAQ,YAAY,IAAI,aAAa,OAAO,UAAU,EAAE;AACxG,WAAO,MAAM,aAAa,SAAS,MAAM,MAAM,KAAK,UAAU,QAAQ,CAAC,GAAG;AAC1E,UAAM,QAAQ,SAAS,aAAa;AACpC,UAAM,oBAAoB,SAAS,YAAY;AAC/C,UAAM,GAAG,YAAY,OAAO,iBAAiB,EAAE;AAC/C,UAAM,SAAS,UAAU;AAAA,MACvB,eAAe;AAAA,IACjB,GAAG;AAAA,MACD,MAAM;AAAA,QACJ,kBAAkB,OAAO,EAAE,OAAO;AAAA,MACpC;AAAA,MACA,OAAO;AAAA,QACL,CAAC,iBAAiB,GAAG,EAAC,OAAO,SAAQ;AAAA,MACvC;AAAA,IACF,CAAC;AAAA,EACH;AAEA,WAAS,SAAS,EAAC,eAAe,OAAO,eAAe,QAAW,cAAc,OAAS,GAAG;AAC3F,UAAM,cAAc,gBAAgB,cAAc,qBAAqB,KAAK,UAAU,YAAY,KAAK,EAAE,qBAAqB,eAAe,EAAE,MAAM;AACrJ,WAAO,KAAK,4BAA4B,KAAK,QAAQ,aAAa,GAAG,WAAW,OAAO,UAAU,EAAE;AAEnG,UAAM,yBAAyB,OAAO,OAAO,gBAAgB,EAAE,QAAQ,KAAK,IAAI;AAChF,UAAM,kCAAkC,OAAO,OAAO,iBAAiB,SAAS,EAAE,QAAQ,KAAK,IAAI;AACnG,UAAM,iCAAiC,OAAO,OAAO,iBAAiB,QAAQ,EAAE,QAAQ,KAAK,IAAI;AACjG,QAAI,CAAC,0BAA0B,CAAC,mCAAmC,CAAC,gCAAgC;AAClG,YAAM,+BAA+B,KAAK,EAAE;AAC5C,YAAM,IAAI,SAAS,WAAW,aAAa,6BAA6B;AAAA,IAC1E;AAEA,UAAM,QAAQ,SAAS,aAAa;AACpC,UAAM,eAAe;AAAA,MACnB,gBAAgB;AAAA,MAChB,kBAAkB,OAAO,EAAE,OAAO;AAAA,MAClC;AAAA,MACA;AAAA,IACF;AAGA,WAAO,KAAK,YAAY,EAAE,QAAQ,SAAO,aAAa,GAAG,MAAM,UAAa,OAAO,aAAa,GAAG,CAAC;AAEpG,WAAO,SACJ;AAAA,MACC,EAAC,eAAe,MAAK;AAAA,MACrB,EAAC,MAAM,aAAY;AAAA,MACnB,EAAC,YAAY,EAAC,KAAK,EAAC,GAAG,gBAAgB,QAAO;AAAA,IAChD;AAAA,EACJ;AAEA,WAAS,kBAAkB,EAAC,eAAe,WAAW,eAAc,GAAG;AAGrE,WAAO,KAAK,uCAAuC,SAAS,KAAK,cAAc,SAAS,aAAa,EAAE;AACvG,UAAM,uCAAuC,SAAS,KAAK,cAAc,SAAS,aAAa,EAAE;AACjG,UAAM,qBAAqB,SAAS,aAAa;AACjD,UAAM,sBAAsB,SAAS,cAAc;AAEnD,QAAI,EAAE,aAAa,aAAa;AAC9B,YAAM,UAAU,SAAS,cAAc,KAAK,UAAU,UAAU,CAAC,EAAE;AACnE,YAAM,IAAI,SAAS,WAAW,aAAa,wCAAwC;AAAA,IACrF;AAEA,QAAI,EAAE,uBAAuB,mBAAmB;AAC9C,YAAM,UAAU,mBAAmB,cAAc,KAAK,UAAU,gBAAgB,CAAC,EAAE;AACnF,YAAM,IAAI,SAAS,WAAW,aAAa,0BAA0B;AAAA,IACvE;AAEA,UAAM,cAAc,eAAe,WAAW,mBAAmB;AAGjE,aAAS,eAAeA,YAAWC,sBAAqB;AACtD,UAAID,eAAc,WAAW,QAAQ;AACnC,eAAO,EAAC,yBAAyBC,qBAAmB;AAAA,MACtD;AACA,UAAID,eAAc,WAAW,QAAQ;AACnC,eAAO,EAAC,yBAAyBC,qBAAmB;AAAA,MACtD;AACA,UAAID,eAAc,WAAW,KAAK;AAChC,eAAO,EAAC,sBAAsBC,qBAAmB;AAAA,MACnD;AAGA,YAAM,IAAI,SAAS,OAAO,wCAAwC;AAAA,IACpE;AAEA,WAAO,SAAS,iBAAiB;AAAA,MAC/B,eAAe;AAAA,IACjB,GAAG;AAAA,MACD,MAAM;AAAA,QACJ,GAAG;AAAA,QACH,kBAAkB,OAAO,EAAE,OAAO;AAAA,MACpC;AAAA,IACF,GAAG,EAAC,YAAY,EAAC,KAAK,EAAC,GAAG,gBAAgB,QAAO,CAAC;AAAA,EACpD;AAEA,iBAAe,aAAa,EAAC,eAAe,UAAS,GAAG;AACtD,UAAM,eAAe;AACrB,UAAM,EAAC,WAAW,aAAY,IAAI,MAAM,SAAS,QAAQ,EAAC,cAAa,CAAC;AACxE,WAAO,KAAK,qCAAqC,YAAY,OAAO,YAAY,QAAQ,aAAa,OAAO,UAAU,EAAE;AACxH,UAAM,qBAAqB,SAAS,aAAa;AACjD,UAAM,oBAAoB,SAAS,YAAY;AAE/C,UAAM,SAAS,MAAM,SAAS,iBAAiB;AAAA,MAC7C,eAAe;AAAA,IACjB,GAAG;AAAA,MACD,MAAM;AAAA,QACJ,WAAW;AAAA,QACX,kBAAkB,OAAO,EAAE,OAAO;AAAA,MACpC;AAAA,IACF,GAAG,EAAC,YAAY,EAAC,KAAK,EAAC,GAAG,gBAAgB,QAAO,CAAC;AAClD,aAAS,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACpC,WAAO,OAAO,MAAM;AAAA,EAEtB;AAEA,iBAAe,cAAc,EAAC,eAAe,cAAc,kBAAkB,OAAS,GAAG;AACvF,UAAM,qBAAqB,SAAS,aAAa;AAEjD,UAAM,YAAY,MAAM,SAAS,QAAQ,EAAC,eAAe,mBAAkB,CAAC;AAC5E,WAAO,MAAM,yBAAyB,KAAK,UAAU,SAAS,CAAC,EAAE;AAEjE,UAAM,gBAAgB,UAAU;AAEhC,WAAO,KAAK,sCAAsC,KAAK,UAAU,aAAa,CAAC,cAAc,YAAY,iBAAiB,eAAe,QAAQ,aAAa,OAAO,UAAU,EAAE;AAEjL,UAAM,wBAAwB,cAAc,OAAO,eAAa,cAAc,eAAe;AAC7F,UAAM,8BAA8B,sBAAsB,SAAS,YAAY,IAAI,wBAAwB,CAAC,GAAG,uBAAuB,YAAY;AAElJ,WAAO,MAAM,0BAA0B,qBAAqB,EAAE;AAC9D,WAAO,MAAM,+BAA+B,2BAA2B,EAAE;AAEzE,UAAM,SAAS,MAAM,SAAS,iBAAiB;AAAA,MAC7C,eAAe;AAAA,IACjB,GAAG;AAAA,MACD,MAAM;AAAA,QACJ,YAAY;AAAA,QACZ,kBAAkB,OAAO,EAAE,OAAO;AAAA,MACpC;AAAA,IACF,GAAG,EAAC,YAAY,EAAC,KAAK,EAAC,GAAG,gBAAgB,QAAO,CAAC;AAClD,aAAS,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACpC,WAAO,OAAO,MAAM;AAAA,EAEtB;AAEA,iBAAe,YAAY,EAAC,cAAa,GAAG;AAC1C,UAAM,qBAAqB,SAAS,aAAa;AAEjD,UAAM,SAAS,MAAM,SAAS,iBAAiB;AAAA,MAC7C,eAAe;AAAA,MACf,gBAAgB,iBAAiB,UAAU;AAAA,IAC7C,GAAG;AAAA,MACD,MAAM;AAAA,QACJ,UAAU;AAAA,MACZ;AAAA,MACA,MAAM;AAAA,QACJ,kBAAkB,OAAO,EAAE,OAAO;AAAA,MACpC;AAAA,IACF,GAAG,EAAC,YAAY,EAAC,KAAK,EAAC,GAAG,gBAAgB,QAAO,CAAC;AAElD,WAAO,QAAQ,+BAA+B,KAAK,UAAU,MAAM,CAAC,EAAE;AACtE,aAAS,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACpC,WAAO;AAAA,EACT;AAEA,iBAAe,YAAY,EAAC,eAAe,SAAQ,GAAG;AACpD,UAAM,qBAAqB,SAAS,aAAa;AACjD,UAAM,gBAAgB,SAAS,QAAQ;AAEvC,UAAM,SAAS,MAAM,SAAS,iBAAiB;AAAA,MAC7C,eAAe;AAAA,IACjB,GAAG;AAAA,MACD,MAAM;AAAA,QACJ,UAAU;AAAA,QACV,kBAAkB,OAAO,EAAE,OAAO;AAAA,MACpC;AAAA,IACF,GAAG,EAAC,YAAY,EAAC,KAAK,EAAC,GAAG,gBAAgB,QAAO,CAAC;AAElD,WAAO,QAAQ,+BAA+B,KAAK,UAAU,MAAM,CAAC,EAAE;AACtE,aAAS,GAAG,KAAK,UAAU,MAAM,CAAC,EAAE;AACpC,WAAO;AAAA,EACT;AAEF;",
+  "names": ["operation", "cleanImportJobState"]
+}