@izara_project/izara-market-library-service-schemas 1.0.55 → 1.0.56

package/src/reStructure/TemplateData/flowSchema/components/upload/processFile/mainFunction/ProcessCsvtemplate.ejs

@@ -0,0 +1,391 @@
+/*
+Copyright (C) 2021 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict';
+
+const callingFlowSharedLib = require('@izara_project/izara-core-library-calling-flow');
+const snsSharedLib = require('@izara_project/izara-core-library-sns');
+const dynamodbSharedLib = require('@izara_project/izara-core-library-dynamodb');
+const lambdaSharedLib = require('@izara_project/izara-core-library-lambda');
+
+const externalRequest = require('@izara_project/izara-core-library-external-request')
+const lambda = externalRequest.lambda;
+const sns = externalRequest.sns;
+const NoRetryError = require('@izara_project/izara-core-library-core').NoRetryError;
+const utils = require('@izara_project/izara-market-library-service-schemas/src/MainLibs/src/Utils');
+
+const { S3Client, GetObjectCommand, DeleteObjectCommand, PutObjectCommand, CopyObjectCommand } = require('@aws-sdk/client-s3');
+const s3 = new S3Client({ region: 'us-east-2' });
+
+const hash = require('@izara_project/izara-shared-core').objectHash;
+const Readable = require('stream');
+const { isValidCsv, getNestObject } = require('../../../../libs/source/GenerateCodeLibs');
+const consts = require('@izara_project/izara-market-library-service-schemas/src/GenerateCodeLibs/src/Consts')
+const coreConsts = require('@izara_project/izara-core-library-core/src/Consts');
+
+
+/**
+ * description of function.
+ * @param {Object} _izContext
+ * @param {CorrelationIds} _izContext.correlationIds - property of _izContext
+ * @param {Logger} _izContext.logger - property of _izContext
+ * @param {Logger} _izContext.logger.debug - property of _izContext
+ * @param {Object} bucket - bucket's data
+ * @param {Array} field - image's data
+ * @param {Array} importBy - 'webSocket' || 'importData'
+ * @param {Array} imageImportId - from importData
+ * @param {Array} realImageSize - from importData
+ * @param {Array} passBackProperties - from importData
+ *
+ * @returns {object} returnResponse
+ */
+
+module.exports.processAfterUpload = async (
+  _izContext,
+  bucket,
+  field,
+  importBy = '',
+  importBatchId = '',
+  realImageSize = 0,
+  passBackProperties = {}
+) => {
+
+  try {
+
+    _izContext.logger.debug('----- processImageFile event params ----- ', {
+      bucket: bucket,
+      field: field,
+      importBy: importBy,
+      importBatchId: importBatchId,
+      realImageSize: realImageSize,
+      passBackProperties: passBackProperties
+    })
+
+    let errorsFound = [];
+
+    const objectS3Param = {
+      Bucket: bucket.name,
+      Key: field.key
+    }
+    _izContext.logger.debug("params before getObjectParam::", objectS3Param)
+
+    const fileObjectS3 = await s3.send(new GetObjectCommand(objectS3Param));
+    _izContext.logger.debug("get fileObjectS3", fileObjectS3);
+
+    importBatchId = fileObjectS3.Metadata.importbatchid
+
+    let getImportBatchMain = await lambda.invokeSync(_izContext,
+      await lambdaSharedLib.lambdaFunctionName(_izContext, "ImportBatchMainGetHdrInv"),
+      {
+        identifiers: {
+          importBatchId: importBatchId
+        }
+      }
+    )
+
+    _izContext.logger.debug("getImportBatchMain", getImportBatchMain)
+    _izContext.correlationIds.set(coreConsts.BASE_USER_ID, getImportBatchMain.fields.userId);
+    _izContext.correlationIds.set(coreConsts.TARGET_ID, getImportBatchMain.fields.submittedByUserId);
+
+    let userId = _izContext.correlationIds.get(coreConsts.BASE_USER_ID);
+    let targetId = _izContext.correlationIds.get(coreConsts.TARGET_ID);
+
+    if (Date.now() > getImportBatchMain.fields.expiryTimestamp) {
+      errorsFound.push("url expired can't upload")
+      let sendMessageUpdateUrlExpired = {
+        objInstanceFull: {
+          identifiers: {
+            importBatchId: importBatchId
+          },
+          fields: {
+            importBatchStatus: "urlExpired",
+            errorsFound: "url expired can't upload",
+            processingError: true
+          }
+        }
+      }
+
+      let sendMessageToInImportBatchUpdate = {
+        Message: JSON.stringify(sendMessageUpdateUrlExpired),
+        TopicArn: await snsSharedLib.snsTopicArn(_izContext, "InImportBatchMainUpdate")
+      };
+
+      _izContext.logger.debug("message param before send to updateImportBatchMain", sendMessageToInImportBatchUpdate);
+      await sns.publishAsync(_izContext, sendMessageToInImportBatchUpdate);
+
+      let deleteFileObjectS3 = await s3.send(new DeleteObjectCommand(objectS3Param));
+      _izContext.logger.debug("Delete object response: ", deleteFileObjectS3)
+
+      return
+    }
+
+    if (fileObjectS3) {
+      if (!importBatchId) {
+        errorsFound.push("cant't get importBatchId from S3 uploaded file");
+      }
+    } else {
+      errorsFound.push("cant't get fileObjectS3 from S3 uploaded file");
+    };
+
+    let fileS3Body = fileObjectS3.Body;
+    _izContext.logger.debug("fileS3Body", fileS3Body);
+
+    let fileS3Buffer = null;
+
+    if (fileS3Body instanceof Readable) {
+      fileS3Buffer = Buffer.concat(await fileS3Body.toArray());
+      _izContext.logger.debug("fileS3Buffer", fileS3Buffer)
+    } else {
+      errorsFound.push('unknown object stream type');
+    }
+
+    // check file type
+    // can be strict in function.yml that only trigger if it specific type of file
+    const srcKey = field.key;
+    const importFieldType = srcKey.match(/\.([^.]*)$/);
+    _izContext.logger.debug("importFieldType", importFieldType);
+    if (!importFieldType) {
+      throw new NoRetryError("error fileType")
+    };
+
+    const importKeyType = utils.firstLetterLowerCase(importFieldType[1]);
+    _izContext.logger.debug("importKeyType", importKeyType);
+
+    if (importKeyType === "csv") {
+      // check valid CSV file
+      let [validCsv, errorCheckCsv] = await isValidCsv(_izContext, fileObjectS3)
+      _izContext.logger.debug("validCsv check::", { validCsv, errorsFound })
+      if (errorCheckCsv.length) {
+        errorsFound.push(errorCheckCsv[0])
+      }
+
+      // check file size
+      let reservedFileSize = getImportBatchMain.fields.fileSize;
+      _izContext.logger.debug("initialFileSize::", reservedFileSize)
+      let s3FileSize = fileObjectS3.ContentLength / 1048576 // convert back to byte unit
+      _izContext.logger.debug("s3FileSize in MiB::", s3FileSize)
+
+      if (s3FileSize < 0.5) { // round up to 1
+        s3FileSize = 1
+      }
+
+      let reservedData = await dynamodbSharedLib.getItem(_izContext,
+        await dynamodbSharedLib.tableName(_izContext, "ReservedDataMain"),
+        {
+          servicePendingId: importBatchId,
+          reservedDataId: `${consts.ACCOUNTLIMIT_TAG.reservedDynamic}_${getImportBatchMain.fields.userId}_${consts.LIMIT_TAG.uploadData}`
+        }
+      )
+      _izContext.logger.debug("reservedData::", reservedData);
+
+      let sizeDiff;
+
+      if (s3FileSize > reservedFileSize) {
+        sizeDiff = s3FileSize - reservedFileSize;
+
+
+        if (sizeDiff < 1) {
+
+          // send message to confirmUsage
+          let confirmUsageMessage = {
+            userId: userId,
+            limitTag: consts.LIMIT_TAG.uploadData,
+            uniqueIdentify: reservedData.uniqueIdentify,
+          }
+
+          let sendMessageToConfirmUsage = {
+            Message: JSON.stringify(confirmUsageMessage),
+            TopicArn: await snsSharedLib.snsTopicArn(_izContext, 'InConfirmUsage', consts.EXTERNAL_SERVICE_NAME.accountLimits)
+          };
+          _izContext.logger.debug("sendMessageToConfirmUsage", sendMessageToConfirmUsage);
+          await sns.publishAsync(_izContext, sendMessageToConfirmUsage);
+
+          let sendMessageToUpdateImportBatchStatus = {
+            objInstanceFull: {
+              identifiers: {
+                importBatchId: servicePendingId
+              },
+              fields: {
+                importBatchStatus: 'uploadComplete'
+              }
+            }
+          }
+
+          let sendMessageToUpdateImportBatch = {
+            Message: JSON.stringify(sendMessageToUpdateImportBatchStatus),
+            TopicArn: await snsSharedLib.snsTopicArn(_izContext, `InImportBatchMainUpdate`)
+          };
+          _izContext.logger.debug("sendMessageToUpdateImportBatch", sendMessageToUpdateImportBatch);
+          await sns.publishAsync(_izContext, sendMessageToUpdateImportBatch);
+
+        }
+      } else if (s3FileSize < reservedFileSize) {
+        sizeDiff = reservedFileSize - s3FileSize;
+
+        if (sizeDiff < 1) {
+          let confirmUsageMessage = {
+            userId: userId,
+            limitTag: consts.LIMIT_TAG.uploadData,
+            uniqueIdentify: reservedData.uniqueIdentify,
+          }
+
+          let sendMessageToConfirmUsage = {
+            Message: JSON.stringify(confirmUsageMessage),
+            TopicArn: await snsSharedLib.snsTopicArn(_izContext, 'InConfirmUsage', consts.EXTERNAL_SERVICE_NAME.accountLimits)
+          };
+          _izContext.logger.debug("sendMessageToConfirmUsage", sendMessageToConfirmUsage);
+          await sns.publishAsync(_izContext, sendMessageToConfirmUsage);
+
+          let sendMessageToUpdateImportBatchStatus = {
+            objInstanceFull: {
+              identifiers: {
+                importBatchId: servicePendingId
+              },
+              fields: {
+                importBatchStatus: 'uploadComplete'
+              }
+            }
+          }
+
+          let sendMessageToUpdateImportBatch = {
+            Message: JSON.stringify(sendMessageToUpdateImportBatchStatus),
+            TopicArn: await snsSharedLib.snsTopicArn(_izContext, `InImportBatchMainUpdate`)
+          };
+          _izContext.logger.debug("sendMessageToUpdateImportBatch", sendMessageToUpdateImportBatch);
+          await sns.publishAsync(_izContext, sendMessageToUpdateImportBatch);
+
+        } else {
+          let realFileSize = Math.round(s3FileSize);
+          // cancel reserved -> new reserved -> confirm
+          // cancel
+          let sendMessageToCancelReserved = {
+            Message: JSON.stringify({
+              userId: userId,
+              limitTag: consts.LIMIT_TAG.uploadData,
+              uniqueIdentify: reservedData.uniqueIdentify
+            }),
+            TopicArn: await snsSharedLib.snsTopicArn(_izContext, 'InCancelUsage', consts.EXTERNAL_SERVICE_NAME.accountLimits)
+          };
+          _izContext.logger.debug("sendMessageToCancelReserved", sendMessageToCancelReserved);
+          await sns.publishAsync(_izContext, sendMessageToCancelReserved);
+
+          // new reserved
+          let originTimestamp = Date.now();
+          let messageObjReserved = {
+            userId: userId,
+            limitTag: consts.LIMIT_TAG.uploadData,
+            value: realFileSize,
+            rbacType: consts.RBAC_TYPE.appLevel,
+            usedTimestamp: originTimestamp,
+            targetUserId: targetId
+          }
+
+          let callingFlowProperties = {
+            identifier: {
+              servicePendingId: importBatchId,
+              reservedDataId: `${consts.ACCOUNTLIMIT_TAG.reservedDynamic}_${getImportBatchMain.fields.userId}_${consts.LIMIT_TAG.uploadData}`,
+              userId: getImportBatchMain.fields.userId,
+              realSize: realFileSize
+            }
+          }
+
+          messageObjReserved = callingFlowSharedLib.addCallingFlowToSnsRequestMessageObject(
+            messageObjReserved,
+            callingFlowSharedLib.createCallingFlowConfig(
+              await lambdaSharedLib.lambdaFunctionName(_izContext, "ConfirmNewReservedHdrSqs"),
+              callingFlowProperties
+            )
+          )
+          _izContext.logger.debug('after add callingFlow: ', messageObjReserved)
+
+          let sendMessageToReservedDynamic = {
+            Message: JSON.stringify(messageObjReserved),
+            TopicArn: await snsSharedLib.snsTopicArn(_izContext, 'InReservedDynamicUsage', consts.EXTERNAL_SERVICE_NAME.accountLimits)
+          };
+
+          _izContext.logger.debug("sendMessageToReservedDynamic", sendMessageToReservedDynamic);
+          await sns.publishAsync(_izContext, sendMessageToReservedDynamic);
+        };
+      } else if (s3FileSize === reservedFileSize) {
+        let confirmUsageMessage = {
+          userId: getImportBatchMain.fields.userId,
+          limitTag: consts.LIMIT_TAG.uploadData,
+          uniqueIdentify: reservedData.uniqueIdentify,
+        }
+
+        let sendMessageToConfirmUsage = {
+          Message: JSON.stringify(confirmUsageMessage),
+          TopicArn: await snsSharedLib.snsTopicArn(_izContext, 'InConfirmUsage', consts.EXTERNAL_SERVICE_NAME.accountLimits)
+        };
+        _izContext.logger.debug("sendMessageToConfirmUsage", sendMessageToConfirmUsage);
+        await sns.publishAsync(_izContext, sendMessageToConfirmUsage);
+
+        let sendMessageToUpdateImportBatchStatus = {
+          objInstanceFull: {
+            identifiers: {
+              importBatchId: servicePendingId
+            },
+            fields: {
+              importBatchStatus: 'uploadComplete'
+            }
+          }
+        }
+
+        let sendMessageToUpdateImportBatch = {
+          Message: JSON.stringify(sendMessageToUpdateImportBatchStatus),
+          TopicArn: await snsSharedLib.snsTopicArn(_izContext, `InImportBatchMainUpdate`)
+        };
+        _izContext.logger.debug("sendMessageToUpdateImportBatch", sendMessageToUpdateImportBatch);
+        await sns.publishAsync(_izContext, sendMessageToUpdateImportBatch);
+
+      }
+
+    } else {
+      errorsFound.push("error importKeyType")
+    }
+
+    if (errorsFound.length) { // if error send message to update error in importBatchMain
+      let updateImportBatchMainError = {
+        objInstanceFull: {
+          identifiers: {
+            importBatchId: importBatchId
+          },
+          fields: {
+            importBatchStatus: "errorToUpload",
+            errorsFound: errorsFound,
+            processingError: true
+          }
+        }
+      }
+
+      let sendMessageToUpdateImportBatchMain = {
+        Message: JSON.stringify(updateImportBatchMainError),
+        TopicArn: await snsSharedLib.snsTopicArn(_izContext, "InImportBatchMainUpdate")
+      };
+      _izContext.logger.debug('sendMessageToUpdateImportBatchMain::', sendMessageToUpdateImportBatchMain);
+      await sns.publishAsync(_izContext, sendMessageToUpdateImportBatchMain);
+
+      let deleteFileObjectS3 = await s3.send(new DeleteObjectCommand(objectS3Param));
+      _izContext.logger.debug("Delete object response: ", deleteFileObjectS3)
+
+    }
+
+  } catch (err) {
+    _izContext.logger.error('error ProcessImageAfterUploadMain: ', err)
+    throw (err)
+  }
+}