@izara_project/izara-market-library-service-schemas 1.0.29 → 1.0.31

package/src/reStructure/TemplateData/flowSchema/components/upload/preSignUrl/createPresignUrl/mainFunction/template.ejs

@@ -0,0 +1,253 @@
+/*
+Copyright (C) 2021 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict';
+
+const asyncFlowSharedLib = require('@izara_project/izara-core-library-asynchronous-flow');
+const snsSharedLib = require('@izara_project/izara-core-library-sns');
+const callingFlowSharedLib = require('@izara_project/izara-core-library-calling-flow');
+const lambdaSharedLib = require('@izara_project/izara-core-library-lambda');
+const dynamodbSharedLib = require('@izara_project/izara-core-library-dynamodb');
+
+const externalRequest = require('@izara_project/izara-core-library-external-request')
+const lambda = externalRequest.lambda;
+const sns = externalRequest.sns;
+
+const { S3Client } = require('@aws-sdk/client-s3');
+const client = new S3Client({ region: 'us-east-2' });
+const { createPresignedPost } = require('@aws-sdk/s3-presigned-post');
+const consts = require('@izara_project/izara-middleware/src/MiddlewareCore/Consts')
+
+const { postToConnection } = require("../../libs/source/ImportDataLib");
+const hash = require('object-hash');
+const utils = require('@izara_project/izara-market-library-service-schemas/src/MainLibs/src/Utils');
+const coreConsts = require('@izara_project/izara-core-library-core/src/Consts');
+const { getNestObject } = require("../../libs/source/ImportDataLib");
+const { EXTERNAL_SERVICE_NAME } = require('@izara_project/izara-market-library-service-schemas/src/GenerateCodeLibs/src/Consts');
+/**
+ * description of function.
+ * @param {Object} _izContext
+ * @param {CorrelationIds} _izContext.correlationIds - property of _izContext
+ * @param {Logger} _izContext.logger - property of _izContext
+ * @param {Logger} _izContext.logger - property of _izContext
+ * @param {Object} returnMessage - limitValue or limitFound
+ * @param {Array} errorsFound - array
+ * @param {boolean} passStatus - true || false
+ * @param {Object} passBackProperties - identifier
+ *
+ * @returns {object} returnResponse
+ */
+
+module.exports.createPresignUrl = async (
+  _izContext,
+  returnMessage,
+  errorsFound,
+  passStatus,
+  passBackProperties
+) => {
+
+  try {
+
+    _izContext.logger.debug('----- createPresignUrl event params -----', {
+      returnMessage: returnMessage,
+      errorsFound: errorsFound,
+      passStatus: passStatus,
+      passBackProperties: passBackProperties
+    });
+
+    const servicePendingId = passBackProperties.passBackProperties.identifier.servicePendingId; // importBatchId
+    const reservedDataId = passBackProperties.passBackProperties.identifier.reservedDataId;
+    const objType = passBackProperties.passBackProperties.objType
+
+    const correlationId = _izContext.correlationIds.get(consts.X_CORRELATION_ID)
+    const connectionId = _izContext.correlationIds.get(consts.CONNECTION_ID)
+
+    // getImportBatchMain from table
+    let importBatchMain = await lambda.invokeSync(_izContext,
+      await lambdaSharedLib.lambdaFunctionName(_izContext, "ImportBatchMainGetHdrInv"),
+      {
+        identifiers: {
+          importBatchId: servicePendingId
+        }
+      }
+    )
+    _izContext.logger.debug("importBatchMain", importBatchMain);
+
+    const {
+      importBatchId
+    } = importBatchMain.identifiers
+
+    const {
+      importBatchStatus,
+      importConfigId,
+      fileName,
+      fileSize,
+      importType,
+      startTime,
+      submittedByUserId,
+      userId
+    } = importBatchMain.fields
+
+    let uniqueIdentify = getNestObject(
+      _izContext,
+      returnMessage,
+      ['limitFound', 'runningTotal', 'uniqueIdentify', 'usedTimestampId']
+    );
+    _izContext.logger.debug("uniqueIdentify", uniqueIdentify);
+
+    await dynamodbSharedLib.putItem(_izContext,
+      await dynamodbSharedLib.tableName(_izContext, "ReservedDataMain"),
+      {
+        servicePendingId: importBatchId, // importBatchId
+        reservedDataId: reservedDataId,
+        reservedValue: fileSize,
+        uniqueIdentify: uniqueIdentify
+      }
+    );
+
+    if (passStatus === false) { // if passStatus false post message to client
+      // use correlationId to get connectionId and use connectionId to post message to client
+
+      _izContext.logger.debug("reserved has reach limit");
+
+      errorsFound.push("reserved has reach limit")
+      // update importBatchMainTable
+      let updateImportBatchMainMsg = {
+        objInstanceFull: {
+          identifiers: {
+            importBatchId: importBatchId
+          },
+          fields: {
+            importBatchStatus: "error",
+            errorsFound: errorsFound
+          }
+        }
+      }
+
+      let sendMessageToUpdateImportBatch = {
+        Message: JSON.stringify(updateImportBatchMainMsg),
+        TopicArn: await snsSharedLib.snsTopicArn(_izContext, `InImportBatchMainUpdate`)
+      }
+      _izContext.logger.debug("sendMessageToUpdateImportBatch", sendMessageToUpdateImportBatch);
+
+      // cancel reserved
+
+
+      let cancelReservedMsg = {
+        userId: userId,
+        limitTag: returnMessage.limitTag,
+        uniqueIdentify: uniqueIdentify
+      };
+
+      let sendMessageToCancelReserved = {
+        Message: JSON.stringify(cancelReservedMsg),
+        TopicArn: await snsSharedLib.snsTopicArn(_izContext, 'InCancelUsage', EXTERNAL_SERVICE_NAME.accountLimits)
+      };
+
+      _izContext.logger.debug("sendMessageToCancelReserved", sendMessageToCancelReserved);
+      await sns.publishAsync(_izContext, sendMessageToCancelReserved);
+
+      await postToConnection({ message: "reserved has reach limit" }, connectionId)
+      // after post message delete websocket task
+      await dynamodbSharedLib.deleteItem(_izContext,
+        await dynamodbSharedLib.tableName(_izContext, "WebSocketTask"),
+        {
+          taskKey: correlationId,
+          connectionId: connectionId
+        }
+      )
+    }
+
+    // store reservedDataId and servicePendingId in ReservedDataMain
+
+
+    const Key = `${fileName}.${importType}`
+
+    const maximumSize = fileSize * 1048576 // convert to byte unit for restrict
+    _izContext.logger.debug("maximumSize", maximumSize);
+
+    // create PresignUrl
+    let createPresignUrlParam = {
+      Bucket: "generatecodetemplatebas-integration-test",
+      Key: Key,
+      Conditions: [
+        ["content-length-range", 0, maximumSize],
+        ["starts-with", "$Content-Type", "file/csv"],
+        ["eq", "$x-amz-meta-importBatchId", importBatchId]
+      ],
+      fields: {},
+      expires: 86400
+    };
+    _izContext.logger.debug('createPresignUrlParam', createPresignUrlParam);
+
+    const uploadUrl = await createPresignedPost(client, createPresignUrlParam);
+
+    uploadUrl.fields['x-amz-meta-importBatchId'] = importBatchId;
+    uploadUrl.fields['Content-Type'] = `file/${importType}`;
+
+    _izContext.logger.debug('uploadUrlLimitSize: ', uploadUrl) // return as object type = {url, fields}
+
+    // update to waiting to upload
+    if (importBatchStatus === "processing") {
+
+      let updateImportBatchMainMsg = {
+        objInstanceFull: {
+          identifiers: {
+            importBatchId: importBatchId
+          },
+          fields: {
+            importBatchStatus: "waitingToUpload",
+            expiryTimestamp: (startTime + (createPresignUrlParam.expires * 1000))
+          }
+        }
+      }
+
+      updateImportBatchMainMsg = callingFlowSharedLib.addCallingFlowToSnsRequestMessageObject(
+        updateImportBatchMainMsg,
+        callingFlowSharedLib.addParentCallingFlowConfig(
+          returnMessage, // resive parent callingFlowConfig.
+          callingFlowSharedLib.createCallingFlowConfig( // callinfflow own service
+            await lambdaSharedLib.lambdaFunctionName(_izContext, `WebSocketTaskComplete`), {}
+          )
+        )
+      );
+
+      _izContext.logger.debug("after add callingFlow", updateImportBatchMainMsg)
+
+      let sendMessageToUpdateImportBatch = {
+        Message: JSON.stringify(updateImportBatchMainMsg),
+        TopicArn: await snsSharedLib.snsTopicArn(_izContext, `In${objType.objectType}Update`)
+      }
+      _izContext.logger.debug("sendMessageToUpdateImportBatch", sendMessageToUpdateImportBatch);
+      await sns.publishAsync(_izContext, sendMessageToUpdateImportBatch);
+
+      await dynamodbSharedLib.updateItem(_izContext,
+        await dynamodbSharedLib.tableName(_izContext, "WebSocketTask"),
+        {
+          taskKey: correlationId,
+          connectionId: connectionId
+        },
+        {
+          uploadUrl: uploadUrl
+        }
+      )
+    }
+  } catch (err) {
+    _izContext.logger.error('error createPresignUrl: ', err)
+    throw (err)
+  }
+}

package/src/reStructure/TemplateData/flowSchema/components/upload/preSignUrl/{reservedLimit → createPresignUrl}/queue/data.js

@@ -48,7 +48,7 @@ function data(_izContext, srcPath) {
 
   WebSocketSqsQueueNames.push(
     {
-      queueName: TOPIC_NAME_GENERATE_CODE.reservedLimitComplete
+      queueName: TOPIC_NAME_GENERATE_CODE.createPresignUrl
     }
   )
 

package/src/reStructure/TemplateData/flowSchema/components/upload/preSignUrl/reservedLimit/functionYaml/data.js

@@ -36,9 +36,8 @@ const {
   awaitingMultipleStepsRole,
   externalResourceSns,
   externalResourceName,
-  DYNAMO_RESOURCE,
-  LAMBDA_RESOURCE,
-  SOURCE_GENERATE_IAM_ROLE
+  SOURCE_GENERATE_IAM_ROLE,
+  FUNCTION_NAME
 } = require('../../../../../../../../MainLibs/src/Consts');
 
 const {
@@ -52,12 +51,12 @@ const {
  * @return {{ templatePath, templateData, setting } }
  */
 
-function data(_izContext, srcPath) {
-  return [createSourceParams(_izContext, srcPath)]
+function data(_izContext, flowSchema, srcPath) {
+  return [createSourceParams(_izContext, flowSchema, srcPath)]
 }
 
-function createSourceParams(_izContext, srcPath) {
-  let functionName = "ReservedLimitComplete";
+function createSourceParams(_izContext, flowSchema, srcPath) {
+  let functionName = upperCase(FUNCTION_NAME.reservedLimit);
   let handlerType = upperCase(HANDLER.hdrSqs);
   let additionalResourcePermission = defaultIamRolePerAction();
 
@@ -69,43 +68,16 @@ function createSourceParams(_izContext, srcPath) {
           SQS_RESOURCE.sendMessage,
           SQS_RESOURCE.deleteMessage,
           SQS_RESOURCE.getQueueAttributes
+        ],
+        [RESOURCE_CLASSES.sns]: [
+          SNS_RESOURCE.publish,
+          SNS_RESOURCE.subscribe
         ]
       },
       [
         resourceNames(RESOURCE_CLASSES.sqs, functionName + handlerType),
-        resourceNames(RESOURCE_CLASSES.sqs, functionName + "DLQ"),
-      ]
-    ),
-    createIamRole(
-      {
-        [RESOURCE_CLASSES.dynamoDbTable]: [
-          DYNAMO_RESOURCE.query,
-          DYNAMO_RESOURCE.deleteItem,
-          DYNAMO_RESOURCE.updateItem,
-          DYNAMO_RESOURCE.putItem
-        ]
-      },
-      [
-        resourceNames(RESOURCE_CLASSES.dynamoDbTable, "WebSocketTask"),
-        resourceNames(RESOURCE_CLASSES.dynamoDbTable, "ReservedDataMain"),
-      ]
-    ),
-    createIamRole(
-      {
-        "execute-api": "ManageConnections"
-      },
-      [
-        resourceNames(RESOURCE_CLASSES.webSocket)
-      ]
-    ),
-    createIamRole(
-      {
-        [RESOURCE_CLASSES.lambda]: [LAMBDA_RESOURCE.invokeFunction],
-        [RESOURCE_CLASSES.sns]: [SNS_RESOURCE.publish]
-      },
-      [
-        resourceNames(RESOURCE_CLASSES.lambda, "ImportBatchMainGetHdrInv"),
-        resourceNames(RESOURCE_CLASSES.sns, "InImportBatchMainUpdate")
+        resourceNames(RESOURCE_CLASSES.sqs, functionName + handlerType + "DLQ"),
+        resourceNames(RESOURCE_CLASSES.sns, TOPIC_NAME_GENERATE_CODE.outCreateNodeComplete)
       ]
     )
   )
@@ -116,9 +88,9 @@ function createSourceParams(_izContext, srcPath) {
       functionName,
       handlerType,
       functionNameConfig: functionName + upperCase(handlerType),
-      resourceLocation: SOURCE_PATH.resourceLocationWebSocket,
+      resourceLocation: path.join(SOURCE_PATH.resourceLocationWebSocket, flowSchema.flowTag, "source/"),
       additionalResourcePermission,
-      functionNameConfig: upperCase("ReservedLimitComp") + upperCase(shortNameHandler(handlerType)),
+      functionNameConfig: upperCase(functionName) + upperCase(shortNameHandler(handlerType)),
       roleName: SOURCE_GENERATE_IAM_ROLE.FlowSchemaUploadS3Role
     },
     setting: {

package/src/reStructure/TemplateData/flowSchema/components/upload/preSignUrl/reservedLimit/functionYaml/template.ejs

@@ -6,9 +6,6 @@
     - sqs:
         arn: arn:aws:sqs:${self:custom.iz_region}:${self:custom.iz_accountId}:${self:custom.iz_resourcePrefix}<%- functionName %><%- firstLetterUpperCase(handlerType) %>
         batchSize: 10
-        filterPatterns: #**** need to update serverless framwork upper v.2.69.1
-        - body: {"MessageAttributes":{"callingFlow":{"Value":["${self:custom.iz_resourcePrefix}<%- firstLetterUpperCase(functionName) %><%- firstLetterUpperCase(handlerType) %>"]}}} # functionName of callingFlow
-        - body: {"MessageAttributes":{"callingFlow":{"Value":[{"exists":false}]}}}
   role: <%- roleName %>Role
     #<#<%- functionName %><%- handlerType %>IamRole#>
     #<#/<%- functionName %><%- handlerType %>IamRole#>

package/src/reStructure/TemplateData/flowSchema/components/upload/preSignUrl/reservedLimit/handler/data.js

@@ -22,7 +22,7 @@ const path = require('path');
 const templatePath = path.join(__dirname, "./template.ejs")
 const { firstLetterLowerCase: lowerCase, firstLetterUpperCase: upperCase } = require("../../../../../../../../MainLibs/src/Utils")
 
-const { HANDLER, SOURCE_PATH } = require('../../../../../../../../MainLibs/src/Consts');
+const { HANDLER, SOURCE_PATH, FUNCTION_NAME } = require('../../../../../../../../MainLibs/src/Consts');
 
 /**
  * receive objectSchema
@@ -31,12 +31,12 @@ const { HANDLER, SOURCE_PATH } = require('../../../../../../../../MainLibs/src/C
  * @return {{ templatePath, templateData, setting } }
  */
 
-function data(_izContext, srcPath) {
-  return [createSourceParams(_izContext, srcPath)]
+function data(_izContext, flowSchema, srcPath) {
+  return [createSourceParams(_izContext, flowSchema, srcPath)]
 }
 
-function createSourceParams(_izContext, srcPath) {
-  let functionName = "ReservedLimitComplete";
+function createSourceParams(_izContext, flowSchema, srcPath) {
+  let functionName = upperCase(FUNCTION_NAME.reservedLimit);
   let handlerType = upperCase(HANDLER.hdrSqs);
 
   return {
@@ -47,7 +47,7 @@ function createSourceParams(_izContext, srcPath) {
     },
     setting: {
       saveFileName: `${functionName}_${handlerType}`,
-      savePath: path.join(srcPath, SOURCE_PATH.webSocket),
+      savePath: path.join(srcPath, SOURCE_PATH.webSocket, flowSchema.flowTag, "source/"),
       fileExtension: ".js",
       isAppend: true
     }

package/src/reStructure/TemplateData/flowSchema/components/upload/preSignUrl/reservedLimit/handler/template.ejs

@@ -17,56 +17,51 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 'use strict';
 
-// const AWS = require('aws-sdk');
-// const sqs = new AWS.SQS({ apiVersion: '2012-11-05' });
-
 const izara = require("@izara_project/izara-middleware");
 const middleware = izara.middlewareHandler;
+const izaraShared = require('@izara_project/izara-shared');
 const recordHandlerSharedLib = require('@izara_project/izara-core-library-record-handler');
-const Logger = require('@izara_project/izara-core-library-logger');
 const callingFlowSharedLib = require('@izara_project/izara-core-library-calling-flow');
+const Logger = require('@izara_project/izara-core-library-logger');
 
-const reservedLimitComplete = require('./ReservedLimitComplete_Main');
+const reservedLimit = require('./ReservedLimit_Main');
 
-// validate event properties in body.Message of sqs event
-middleware.setValidatorSchema(recordHandlerSharedLib.baseValidatorSchema());
+middleware.setServiceSchema(recordHandlerSharedLib.baseValidatorSchema());
 
-// set schema for record.body.Message
 let perRecordsValidatorSchema = {
   type: "object",
-  required: [
-    'returnMessage',
-    'errorsFound',
-    'passStatus'
-  ],
+  required: ["objType", "objInstanceFull"],
   properties: {
-    returnMessage: {
-      type: "object"
-    },
-    errorsFound: {
-      type: "array"
+    objType: {
+      type: "object",
+      required: ["objectType", "serviceTag"],
+      properties: {
+        objectType: {
+          type: "string",
+          pattern: "^[a-zA-Z0-9_-]+$"
+        },
+        serviceTag: {
+          type: "string",
+          pattern: "^[a-zA-Z0-9_-]+$"
+        }
+      },
     },
-    passStatus: {
-      type: 'boolean'
+    objInstaceFull: {
+      type: "object",
+      properties: {
+        identifiers: {
+          type: "object",
+          minProperties: 1
+        },
+        fields: {
+          type: "object"
+        }
+      }
     }
   }
 };
 
-// set schema for record.body.MessageAttributes
-// const messageAttributeValidatorSchema = {
-//   type: "object",
-//   required: ['msgAtrrParam1', 'msgAtrrParam2'],
-//   properties: {
-//     msgAtrrParam1: {
-//       type: "string"
-//     },
-//     msgAtrrParam2: {
-//       type: "object"
-//     }
-//   }
-// };
-
-perRecordsValidatorSchema = callingFlowSharedLib.addOptionalPassBackPropertiesToValidatorSchema(perRecordsValidatorSchema)
+perRecordsValidatorSchema = callingFlowSharedLib.addOptionalPassBackPropertiesToValidatorSchema(perRecordsValidatorSchema);
 
 module.exports.main = middleware.wrap(async (event, context, callback) => {
 
@@ -80,26 +75,26 @@ module.exports.main = middleware.wrap(async (event, context, callback) => {
       let passOnProperties = []
       record._izContext.logger.debug('record ReceiveMsgOutHdrSqs', record);
 
-      // validate message (and MessageAttributes)
+      //validate message (and MessageAttributes)
       await recordHandlerSharedLib.validateRecord(
         record, // one record will send to mainFunction
-        "ReservedLimitCompleteHdrSqs", // queue name that need to retry or send to dlq
+        "GetPresignUrlHdrSqs", // queue name that need to retry or send to dlq
         perRecordsValidatorSchema, // schema for record.Message
         // messageAttributeValidatorSchema  // ----- for msgAttr default is null -> do not send this parameter if not want to validate msgAtt
       );
 
       // add argument (to invoke lambda) to passOnProperties[]
-      passOnProperties.push(record.body.Message.returnMessage)
-      passOnProperties.push(record.body.Message.errorsFound)
-      passOnProperties.push(record.body.Message.passStatus)
-      passOnProperties.push(callingFlowSharedLib.addPassBackPropertiesToPassOnProperties(record.body.Message))
+      passOnProperties.push(record.body.Message.objType)
+      passOnProperties.push(record.body.Message.objInstanceFull)
+      passOnProperties.push(callingFlowSharedLib.addPassBackPropertiesToPassOnProperties(record.body.Message)) // is Ok PassBackProperties
+      // passOnProperties.push(callingFlowSharedLib.addCallingFlowToPassOnProperties(record.body.Message)) // is Ok callinfFlow
       record._izContext.logger.debug('passOnProperties in handler', passOnProperties);
 
       // call recordHandlerSharedLib.recordHandler with 3 parameters and return promise(resolve)
       let recordPromise = recordHandlerSharedLib.recordHandler(
         record, // one record will send to mainFunction
-        reservedLimitComplete.reservedLimitComplete, // mainFunction that need to invoke.
-        "ReservedLimitCompleteHdrSqs", // queue name that need to retry or send to dlq
+        reservedLimit.reservedLimit, // mainFunction that need to invoke.
+        "GetPresignUrlHdrSqs", // queue name that need to retry or send to dlq
         passOnProperties, // all parameters that mainFunction needed.
       );
       record._izContext.logger.debug('after recordPromise in handler');
@@ -111,7 +106,7 @@ module.exports.main = middleware.wrap(async (event, context, callback) => {
       // --- main await all promises
       await Promise.all(recordPromises); // await all promises
 
-      return event.Records // return all for local test
+      return event.Records // return all for local testing
 
     } catch {
       Logger.debug('Promise.all(recordPromises) in handler threw error (at least one record did no resolve)');
@@ -119,7 +114,7 @@ module.exports.main = middleware.wrap(async (event, context, callback) => {
     Logger.debug('after Promise.all(recordPromises) in handler');
 
   } catch (err) {
-    Logger.error('Unhandled Error, ReservedImageLimitCompleteHdrSqs: ', err);
+    Logger.error('Unhandled Error, updateNodeCompleteHdrSqs: ', err);
     throw (err);
   }
 });

package/src/reStructure/TemplateData/flowSchema/components/upload/preSignUrl/reservedLimit/mainFunction/data.js

@@ -20,8 +20,8 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 const path = require('path');
 
 const templatePath = path.join(__dirname, "./template.ejs")
-
-const { SOURCE_PATH } = require('../../../../../../../../MainLibs/src/Consts');
+const { firstLetterUpperCase: upperCase } = require('../../../../../../../../MainLibs/src/Utils')
+const { SOURCE_PATH, FUNCTION_NAME } = require('../../../../../../../../MainLibs/src/Consts');
 
 /**
  * receive objectSchema
@@ -35,16 +35,13 @@ function data(_izContext, flowSchema, srcPath) {
 }
 
 function createSourceParams(_izContext, flowSchema, srcPath) {
-  let functionName = "ReservedLimitComplete"
-  let bucketName = flowSchema.bucketName
+  let functionName = upperCase(FUNCTION_NAME.reservedLimit)
   return {
     templatePath: templatePath,
-    templateData: {
-      bucketName
-    },
+    templateData: {},
     setting: {
       saveFileName: `${functionName}_Main`,
-      savePath: path.join(srcPath, SOURCE_PATH.webSocket),
+      savePath: path.join(srcPath, SOURCE_PATH.webSocket, flowSchema.flowTag, "source/"),
       fileExtension: ".js",
       isAppend: false
     }