@izara_project/izara-market-library-service-schemas 1.0.54 → 1.0.56

package/src/reStructure/TemplateData/flowSchema/components/upload/preSignUrl/reservedLimit/mainFunction/template.ejs

@@ -20,7 +20,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 const callingFlowSharedLib = require('@izara_project/izara-core-library-calling-flow');
 const lambdaSharedLib = require('@izara_project/izara-core-library-lambda');
 const snsSharedLib = require('@izara_project/izara-core-library-sns');
-
+const asyncFlowSharedLib = require("@izara_project/izara-core-library-asynchronous-flow")
 const externalRequest = require('@izara_project/izara-core-library-external-request')
 const sns = externalRequest.sns;
 const lambda = externalRequest.lambda
@@ -45,6 +45,8 @@ module.exports.reservedLimit = async (
   _izContext,
   objType,
   objInstanceFull,
+  //(<additionalProperties>)
+  //(</additionalProperties>)
   callingFlowConfig = {}
 ) => {
 
@@ -55,19 +57,9 @@ module.exports.reservedLimit = async (
       objInstanceFull,
       callingFlowConfig
     });
-    const {
-      userId,
-      submittedByUserId,
-      importConfigId,
-      importBatchStatus,
-      importType,
-      fileName,
-      fileSize,
-      startTime
-    } = objInstanceFull.fields
-    const {
-      importBatchId
-    } = objInstanceFull.identifiers
+
+    //(<beforeReserved>)
+    //(</beforeReserved>)
 
     let connectionId = _izContext.correlationIds.get(coreConsts.CONNECTION_ID)
     //* hard code
@@ -75,23 +67,16 @@ module.exports.reservedLimit = async (
 
     // get static
     let payloadBeforeInvokeStaticLimit = {
-      userId: userId,
-      limitTag: consts.LIMIT_TAG.uploadData,
-      value: fileSize,
-      rbacType: consts.RBAC_TYPE.appLevel,
-      targetUserId: submittedByUserId,
-      callingFlow: "BasDev",
-      callingFlowProperties: {}
+      userId: _izContext.correlationIds.get(coreConsts.BASE_USER_ID),
+      targetUserId: _izContext.correlationIds.get(coreConsts.TARGET_ID),
+      //(<additionalPropertiesStaticLimit>)
+      //(</additionalPropertiesStaticLimit>)
     }
 
-    let res = await importDataLibs.getDataFromInvokeFunctionWithCache(_izContext,
+    let res = await importDataLibs.processStaticLimit(_izContext,
       await lambdaSharedLib.lambdaFunctionName(_izContext, "StaticLimitProcessHdrInv", consts.EXTERNAL_SERVICE_NAME.accountLimits),
       payloadBeforeInvokeStaticLimit
     )
-    // let res = await lambda.invokeSync(_izContext,
-    //   await lambdaSharedLib.lambdaFunctionName(_izContext, "StaticLimitProcessHdrInv", consts.EXTERNAL_SERVICE_NAME.accountLimits),
-    //   payloadBeforeInvokeStaticLimit
-    // )
 
     _izContext.logger.debug("res::", res);
 
@@ -99,27 +84,20 @@ module.exports.reservedLimit = async (
       importDataLibs.postToConnection({ message: "static limit over" }, connectionId)
     }
 
-
     // reservedDynamic
 
     let sendMsgReservedDynamic = {
-      userId: userId,
-      limitTag: consts.LIMIT_TAG.uploadData,
-      value: fileSize,
-      rbacType: consts.RBAC_TYPE.appLevel,
-      usedTimestamp: startTime,
-      targetUserId: submittedByUserId
+      userId: _izContext.correlationIds.get(coreConsts.BASE_USER_ID),
+      targetUserId: _izContext.correlationIds.get(coreConsts.TARGET_ID)
+      //(<additionalPropertiesReservedDynamic>)
+      //(</additionalPropertiesReservedDynamic>)
     }
 
     const reservedDataId = `${consts.ACCOUNTLIMIT_TAG.reservedDynamic}_${userId}_${consts.LIMIT_TAG.uploadData}`
 
     let callingFlowProperties = {
-      identifier: {
-        servicePendingId: importBatchId,
-        reservedDataId: reservedDataId
-      },
-      importConfigId: importConfigId,
-      objType: objType
+      //(<baseCallingFlowProperties>)
+      //(</baseCallingFlowProperties>)
     }
 
     _izContext.logger.debug("callingFlowProperties", callingFlowProperties);
@@ -130,7 +108,9 @@ module.exports.reservedLimit = async (
         callingFlowSharedLib.addParentCallingFlowConfig(
           callingFlowConfig,
           callingFlowSharedLib.createCallingFlowConfig(
+            //(<callingFlowProperties>)
             await lambdaSharedLib.lambdaFunctionName(_izContext, "ReservedLimitCompleteHdrSqs"),
+            //(</callingFlowProperties>)
             callingFlowProperties
           )
         )
@@ -139,15 +119,30 @@ module.exports.reservedLimit = async (
       sendMsgReservedDynamic = callingFlowSharedLib.addCallingFlowToSnsRequestMessageObject(
         sendMsgReservedDynamic,
         callingFlowSharedLib.createCallingFlowConfig(
+          //(<callingFlowProperties>)
           await lambdaSharedLib.lambdaFunctionName(_izContext, "ReservedLimitCompleteHdrSqs"),
+          //(</callingFlowProperties>)
           callingFlowProperties
         )
       );
     }
 
+    // create awaitingSteps
+    await asyncFlowSharedLib.createAwaitingStep(_izContext,
+      asyncFlowSharedLib.createAwaitingStepId(
+        //(<createAwaitStepId>)
+        //(</createAwaitStepId>)
+      ),
+      asyncFlowSharedLib.createPendingStepId(
+        //(<createPendingStepId>)
+        //(</createPendingStepId>)
+      )
+    )
+
+
     let sendMessageToReservedDynamic = {
       Message: JSON.stringify(sendMsgReservedDynamic),
-      TopicArn: await snsSharedLib.snsTopicArn(_izContext, "InReservedDynamicUsage", consts.EXTERNAL_SERVICE_NAME.accountLimits)
+      TopicArn: await snsSharedLib.snsTopicArn(_izContext, "ReservedDynamicUsage_In", consts.EXTERNAL_SERVICE_NAME.accountLimits)
     };
     _izContext.logger.debug("sendMessageToReservedDynamic", sendMessageToReservedDynamic);
     await sns.publishAsync(_izContext, sendMessageToReservedDynamic);

package/src/reStructure/TemplateData/flowSchema/components/upload/processFile/handler/handlerS3/template.ejs

@@ -37,6 +37,8 @@ module.exports.main = middleware.wrap(async (event, context, callback) => {
       event._izContext,       // correlationId/logger/integrationTestDetail/uniqueRequestId/userId
       event.Records[0].s3.bucket,
       event.Records[0].s3.object,
+      //(<additionalProperties>)
+      //(</additionalProperties>)
     );
 
     if (processImageResponse instanceof Error) {
@@ -49,20 +51,4 @@ module.exports.main = middleware.wrap(async (event, context, callback) => {
     event._izContext.logger.error('Error, processAfterUploadHdrS3: ', err);
     return (izara.response.failure(err));
   }
-});
-
-//
-// s3: {
-//   s3SchemaVersion: '1.0',
-//   configurationId: 'ImageStandardTestProcessImageFileHdrS3-68a737133cefb25bff959852b8f04754',
-//   bucket: {
-//     name: 'image-test-importdata',
-//     ownerIdentity: { principalId: 'A3CDE97VJKTOJ0' },
-//     arn: 'arn:aws:s3:::image-test-importdata'
-//   },
-//   object: {
-//     key: 'image_8733685.png',
-//     size: 2974746,
-//     eTag: '93e940d895c26ef90b7542da54d61898',
-//     sequencer: '00650D4EEAD0F63EB5'
-//   }
+});

package/src/reStructure/TemplateData/flowSchema/components/upload/processFile/mainFunction/ProcessCsvtemplate.ejs

@@ -0,0 +1,391 @@
+/*
+Copyright (C) 2021 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict';
+
+const callingFlowSharedLib = require('@izara_project/izara-core-library-calling-flow');
+const snsSharedLib = require('@izara_project/izara-core-library-sns');
+const dynamodbSharedLib = require('@izara_project/izara-core-library-dynamodb');
+const lambdaSharedLib = require('@izara_project/izara-core-library-lambda');
+
+const externalRequest = require('@izara_project/izara-core-library-external-request')
+const lambda = externalRequest.lambda;
+const sns = externalRequest.sns;
+const NoRetryError = require('@izara_project/izara-core-library-core').NoRetryError;
+const utils = require('@izara_project/izara-market-library-service-schemas/src/MainLibs/src/Utils');
+
+const { S3Client, GetObjectCommand, DeleteObjectCommand, PutObjectCommand, CopyObjectCommand } = require('@aws-sdk/client-s3');
+const s3 = new S3Client({ region: 'us-east-2' });
+
+const hash = require('@izara_project/izara-shared-core').objectHash;
+const Readable = require('stream');
+const { isValidCsv, getNestObject } = require('../../../../libs/source/GenerateCodeLibs');
+const consts = require('@izara_project/izara-market-library-service-schemas/src/GenerateCodeLibs/src/Consts')
+const coreConsts = require('@izara_project/izara-core-library-core/src/Consts');
+
+
+/**
+ * description of function.
+ * @param {Object} _izContext
+ * @param {CorrelationIds} _izContext.correlationIds - property of _izContext
+ * @param {Logger} _izContext.logger - property of _izContext
+ * @param {Logger} _izContext.logger.debug - property of _izContext
+ * @param {Object} bucket - bucket's data
+ * @param {Array} field - image's data
+ * @param {Array} importBy - 'webSocket' || 'importData'
+ * @param {Array} imageImportId - from importData
+ * @param {Array} realImageSize - from importData
+ * @param {Array} passBackProperties - from importData
+ *
+ * @returns {object} returnResponse
+ */
+
+module.exports.processAfterUpload = async (
+  _izContext,
+  bucket,
+  field,
+  importBy = '',
+  importBatchId = '',
+  realImageSize = 0,
+  passBackProperties = {}
+) => {
+
+  try {
+
+    _izContext.logger.debug('----- processImageFile event params ----- ', {
+      bucket: bucket,
+      field: field,
+      importBy: importBy,
+      importBatchId: importBatchId,
+      realImageSize: realImageSize,
+      passBackProperties: passBackProperties
+    })
+
+    let errorsFound = [];
+
+    const objectS3Param = {
+      Bucket: bucket.name,
+      Key: field.key
+    }
+    _izContext.logger.debug("params before getObjectParam::", objectS3Param)
+
+    const fileObjectS3 = await s3.send(new GetObjectCommand(objectS3Param));
+    _izContext.logger.debug("get fileObjectS3", fileObjectS3);
+
+    importBatchId = fileObjectS3.Metadata.importbatchid
+
+    let getImportBatchMain = await lambda.invokeSync(_izContext,
+      await lambdaSharedLib.lambdaFunctionName(_izContext, "ImportBatchMainGetHdrInv"),
+      {
+        identifiers: {
+          importBatchId: importBatchId
+        }
+      }
+    )
+
+    _izContext.logger.debug("getImportBatchMain", getImportBatchMain)
+    _izContext.correlationIds.set(coreConsts.BASE_USER_ID, getImportBatchMain.fields.userId);
+    _izContext.correlationIds.set(coreConsts.TARGET_ID, getImportBatchMain.fields.submittedByUserId);
+
+    let userId = _izContext.correlationIds.get(coreConsts.BASE_USER_ID);
+    let targetId = _izContext.correlationIds.get(coreConsts.TARGET_ID);
+
+    if (Date.now() > getImportBatchMain.fields.expiryTimestamp) {
+      errorsFound.push("url expired can't upload")
+      let sendMessageUpdateUrlExpired = {
+        objInstanceFull: {
+          identifiers: {
+            importBatchId: importBatchId
+          },
+          fields: {
+            importBatchStatus: "urlExpired",
+            errorsFound: "url expired can't upload",
+            processingError: true
+          }
+        }
+      }
+
+      let sendMessageToInImportBatchUpdate = {
+        Message: JSON.stringify(sendMessageUpdateUrlExpired),
+        TopicArn: await snsSharedLib.snsTopicArn(_izContext, "InImportBatchMainUpdate")
+      };
+
+      _izContext.logger.debug("message param before send to updateImportBatchMain", sendMessageToInImportBatchUpdate);
+      await sns.publishAsync(_izContext, sendMessageToInImportBatchUpdate);
+
+      let deleteFileObjectS3 = await s3.send(new DeleteObjectCommand(objectS3Param));
+      _izContext.logger.debug("Delete object response: ", deleteFileObjectS3)
+
+      return
+    }
+
+    if (fileObjectS3) {
+      if (!importBatchId) {
+        errorsFound.push("cant't get importBatchId from S3 uploaded file");
+      }
+    } else {
+      errorsFound.push("cant't get fileObjectS3 from S3 uploaded file");
+    };
+
+    let fileS3Body = fileObjectS3.Body;
+    _izContext.logger.debug("fileS3Body", fileS3Body);
+
+    let fileS3Buffer = null;
+
+    if (fileS3Body instanceof Readable) {
+      fileS3Buffer = Buffer.concat(await fileS3Body.toArray());
+      _izContext.logger.debug("fileS3Buffer", fileS3Buffer)
+    } else {
+      errorsFound.push('unknown object stream type');
+    }
+
+    // check file type
+    // can be strict in function.yml that only trigger if it specific type of file
+    const srcKey = field.key;
+    const importFieldType = srcKey.match(/\.([^.]*)$/);
+    _izContext.logger.debug("importFieldType", importFieldType);
+    if (!importFieldType) {
+      throw new NoRetryError("error fileType")
+    };
+
+    const importKeyType = utils.firstLetterLowerCase(importFieldType[1]);
+    _izContext.logger.debug("importKeyType", importKeyType);
+
+    if (importKeyType === "csv") {
+      // check valid CSV file
+      let [validCsv, errorCheckCsv] = await isValidCsv(_izContext, fileObjectS3)
+      _izContext.logger.debug("validCsv check::", { validCsv, errorsFound })
+      if (errorCheckCsv.length) {
+        errorsFound.push(errorCheckCsv[0])
+      }
+
+      // check file size
+      let reservedFileSize = getImportBatchMain.fields.fileSize;
+      _izContext.logger.debug("initialFileSize::", reservedFileSize)
+      let s3FileSize = fileObjectS3.ContentLength / 1048576 // convert back to byte unit
+      _izContext.logger.debug("s3FileSize in MiB::", s3FileSize)
+
+      if (s3FileSize < 0.5) { // round up to 1
+        s3FileSize = 1
+      }
+
+      let reservedData = await dynamodbSharedLib.getItem(_izContext,
+        await dynamodbSharedLib.tableName(_izContext, "ReservedDataMain"),
+        {
+          servicePendingId: importBatchId,
+          reservedDataId: `${consts.ACCOUNTLIMIT_TAG.reservedDynamic}_${getImportBatchMain.fields.userId}_${consts.LIMIT_TAG.uploadData}`
+        }
+      )
+      _izContext.logger.debug("reservedData::", reservedData);
+
+      let sizeDiff;
+
+      if (s3FileSize > reservedFileSize) {
+        sizeDiff = s3FileSize - reservedFileSize;
+
+
+        if (sizeDiff < 1) {
+
+          // send message to confirmUsage
+          let confirmUsageMessage = {
+            userId: userId,
+            limitTag: consts.LIMIT_TAG.uploadData,
+            uniqueIdentify: reservedData.uniqueIdentify,
+          }
+
+          let sendMessageToConfirmUsage = {
+            Message: JSON.stringify(confirmUsageMessage),
+            TopicArn: await snsSharedLib.snsTopicArn(_izContext, 'InConfirmUsage', consts.EXTERNAL_SERVICE_NAME.accountLimits)
+          };
+          _izContext.logger.debug("sendMessageToConfirmUsage", sendMessageToConfirmUsage);
+          await sns.publishAsync(_izContext, sendMessageToConfirmUsage);
+
+          let sendMessageToUpdateImportBatchStatus = {
+            objInstanceFull: {
+              identifiers: {
+                importBatchId: servicePendingId
+              },
+              fields: {
+                importBatchStatus: 'uploadComplete'
+              }
+            }
+          }
+
+          let sendMessageToUpdateImportBatch = {
+            Message: JSON.stringify(sendMessageToUpdateImportBatchStatus),
+            TopicArn: await snsSharedLib.snsTopicArn(_izContext, `InImportBatchMainUpdate`)
+          };
+          _izContext.logger.debug("sendMessageToUpdateImportBatch", sendMessageToUpdateImportBatch);
+          await sns.publishAsync(_izContext, sendMessageToUpdateImportBatch);
+
+        }
+      } else if (s3FileSize < reservedFileSize) {
+        sizeDiff = reservedFileSize - s3FileSize;
+
+        if (sizeDiff < 1) {
+          let confirmUsageMessage = {
+            userId: userId,
+            limitTag: consts.LIMIT_TAG.uploadData,
+            uniqueIdentify: reservedData.uniqueIdentify,
+          }
+
+          let sendMessageToConfirmUsage = {
+            Message: JSON.stringify(confirmUsageMessage),
+            TopicArn: await snsSharedLib.snsTopicArn(_izContext, 'InConfirmUsage', consts.EXTERNAL_SERVICE_NAME.accountLimits)
+          };
+          _izContext.logger.debug("sendMessageToConfirmUsage", sendMessageToConfirmUsage);
+          await sns.publishAsync(_izContext, sendMessageToConfirmUsage);
+
+          let sendMessageToUpdateImportBatchStatus = {
+            objInstanceFull: {
+              identifiers: {
+                importBatchId: servicePendingId
+              },
+              fields: {
+                importBatchStatus: 'uploadComplete'
+              }
+            }
+          }
+
+          let sendMessageToUpdateImportBatch = {
+            Message: JSON.stringify(sendMessageToUpdateImportBatchStatus),
+            TopicArn: await snsSharedLib.snsTopicArn(_izContext, `InImportBatchMainUpdate`)
+          };
+          _izContext.logger.debug("sendMessageToUpdateImportBatch", sendMessageToUpdateImportBatch);
+          await sns.publishAsync(_izContext, sendMessageToUpdateImportBatch);
+
+        } else {
+          let realFileSize = Math.round(s3FileSize);
+          // cancel reserved -> new reserved -> confirm
+          // cancel
+          let sendMessageToCancelReserved = {
+            Message: JSON.stringify({
+              userId: userId,
+              limitTag: consts.LIMIT_TAG.uploadData,
+              uniqueIdentify: reservedData.uniqueIdentify
+            }),
+            TopicArn: await snsSharedLib.snsTopicArn(_izContext, 'InCancelUsage', consts.EXTERNAL_SERVICE_NAME.accountLimits)
+          };
+          _izContext.logger.debug("sendMessageToCancelReserved", sendMessageToCancelReserved);
+          await sns.publishAsync(_izContext, sendMessageToCancelReserved);
+
+          // new reserved
+          let originTimestamp = Date.now();
+          let messageObjReserved = {
+            userId: userId,
+            limitTag: consts.LIMIT_TAG.uploadData,
+            value: realFileSize,
+            rbacType: consts.RBAC_TYPE.appLevel,
+            usedTimestamp: originTimestamp,
+            targetUserId: targetId
+          }
+
+          let callingFlowProperties = {
+            identifier: {
+              servicePendingId: importBatchId,
+              reservedDataId: `${consts.ACCOUNTLIMIT_TAG.reservedDynamic}_${getImportBatchMain.fields.userId}_${consts.LIMIT_TAG.uploadData}`,
+              userId: getImportBatchMain.fields.userId,
+              realSize: realFileSize
+            }
+          }
+
+          messageObjReserved = callingFlowSharedLib.addCallingFlowToSnsRequestMessageObject(
+            messageObjReserved,
+            callingFlowSharedLib.createCallingFlowConfig(
+              await lambdaSharedLib.lambdaFunctionName(_izContext, "ConfirmNewReservedHdrSqs"),
+              callingFlowProperties
+            )
+          )
+          _izContext.logger.debug('after add callingFlow: ', messageObjReserved)
+
+          let sendMessageToReservedDynamic = {
+            Message: JSON.stringify(messageObjReserved),
+            TopicArn: await snsSharedLib.snsTopicArn(_izContext, 'InReservedDynamicUsage', consts.EXTERNAL_SERVICE_NAME.accountLimits)
+          };
+
+          _izContext.logger.debug("sendMessageToReservedDynamic", sendMessageToReservedDynamic);
+          await sns.publishAsync(_izContext, sendMessageToReservedDynamic);
+        };
+      } else if (s3FileSize === reservedFileSize) {
+        let confirmUsageMessage = {
+          userId: getImportBatchMain.fields.userId,
+          limitTag: consts.LIMIT_TAG.uploadData,
+          uniqueIdentify: reservedData.uniqueIdentify,
+        }
+
+        let sendMessageToConfirmUsage = {
+          Message: JSON.stringify(confirmUsageMessage),
+          TopicArn: await snsSharedLib.snsTopicArn(_izContext, 'InConfirmUsage', consts.EXTERNAL_SERVICE_NAME.accountLimits)
+        };
+        _izContext.logger.debug("sendMessageToConfirmUsage", sendMessageToConfirmUsage);
+        await sns.publishAsync(_izContext, sendMessageToConfirmUsage);
+
+        let sendMessageToUpdateImportBatchStatus = {
+          objInstanceFull: {
+            identifiers: {
+              importBatchId: servicePendingId
+            },
+            fields: {
+              importBatchStatus: 'uploadComplete'
+            }
+          }
+        }
+
+        let sendMessageToUpdateImportBatch = {
+          Message: JSON.stringify(sendMessageToUpdateImportBatchStatus),
+          TopicArn: await snsSharedLib.snsTopicArn(_izContext, `InImportBatchMainUpdate`)
+        };
+        _izContext.logger.debug("sendMessageToUpdateImportBatch", sendMessageToUpdateImportBatch);
+        await sns.publishAsync(_izContext, sendMessageToUpdateImportBatch);
+
+      }
+
+    } else {
+      errorsFound.push("error importKeyType")
+    }
+
+    if (errorsFound.length) { // if error send message to update error in importBatchMain
+      let updateImportBatchMainError = {
+        objInstanceFull: {
+          identifiers: {
+            importBatchId: importBatchId
+          },
+          fields: {
+            importBatchStatus: "errorToUpload",
+            errorsFound: errorsFound,
+            processingError: true
+          }
+        }
+      }
+
+      let sendMessageToUpdateImportBatchMain = {
+        Message: JSON.stringify(updateImportBatchMainError),
+        TopicArn: await snsSharedLib.snsTopicArn(_izContext, "InImportBatchMainUpdate")
+      };
+      _izContext.logger.debug('sendMessageToUpdateImportBatchMain::', sendMessageToUpdateImportBatchMain);
+      await sns.publishAsync(_izContext, sendMessageToUpdateImportBatchMain);
+
+      let deleteFileObjectS3 = await s3.send(new DeleteObjectCommand(objectS3Param));
+      _izContext.logger.debug("Delete object response: ", deleteFileObjectS3)
+
+    }
+
+  } catch (err) {
+    _izContext.logger.error('error ProcessImageAfterUploadMain: ', err)
+    throw (err)
+  }
+}