@izara_project/izara-market-library-service-schemas 1.0.34 → 1.0.36

package/src/reStructure/TemplateData/flowSchema/CreateRecordComplete/handler/template.ejs

@@ -0,0 +1,85 @@
+/*
+Copyright (C) 2020 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict';
+
+// const AWS = require('aws-sdk');
+// const sqs = new AWS.SQS({ apiVersion: '2012-11-05' });
+
+const izara = require("@izara_project/izara-middleware");
+const middleware = izara.middlewareHandler;
+const recordHandlerSharedLib = require("@izara_project/izara-core-library-record-handler")
+const Logger = require('@izara_project/izara-core-library-logger');
+
+const FlowSchemaOwnTopicEndpoint = require('./FlowSchemaOwnTopicEndpoint_Main')
+
+// validate event properties in body.Message of sqs event
+let perRecordsValidatorSchema = {}
+middleware.setValidatorSchema(recordHandlerSharedLib.baseValidatorSchema());
+
+module.exports.main = middleware.wrap(async (event, context, callback) => {
+
+  try {
+
+    let recordPromises = [];
+
+    // loop each record and send to mainFunction
+    await Promise.all(event.Records.map(async record => { // promise.all for map() function
+
+      let passOnProperties = []
+      record._izContext.logger.debug('record ReceiveMsgOutHdrSqs', record);
+
+      //validate message (and MessageAttributes)
+      await recordHandlerSharedLib.validateRecord(
+        record, // one record will send to mainFunction
+        "<%- queueName %>", // queue name that need to retry or send to dlq
+        perRecordsValidatorSchema, // schema for record.Message
+        // messageAttributeValidatorSchema   // ----- for msgAttr default is null -> do not send this parameter if not want to validate msgAtt
+      );
+
+      // add argument (to invoke lambda) to passOnProperties[]
+      passOnProperties.push(record.body.Message)
+      record._izContext.logger.debug('passOnProperties in handler', passOnProperties);
+
+      // call recordHandlerSharedLib.recordHandler with 3 parameters and return promise(resolve)
+      let recordPromise = recordHandlerSharedLib.recordHandler(
+        record, // one record will send to mainFunction
+        <%- functionName %>.<%- functionName %>, // mainFunction that need to invoke.
+        "<%- queueName %>", // queue name that need to retry or send to dlq
+        passOnProperties, // all parameters that mainFunction needed.
+      );
+      record._izContext.logger.debug('after recordPromise in handler');
+      recordPromises.push(recordPromise); // push promise to recordPromises
+    }))
+
+    Logger.debug('before Promise.all(recordPromises) in handler');
+    try {
+      // --- main await all promises
+      await Promise.all(recordPromises); // await all promises
+
+      return event.Records // return all for local testing
+
+    } catch {
+      Logger.debug('Promise.all(recordPromises) in handler threw error (at least one record did no resolve)');
+    }
+    Logger.debug('after Promise.all(recordPromises) in handler');
+
+  } catch (err) {
+    Logger.error('Unhandled Error, LambdaFunctionHdrSqs: ', err);
+    throw (err);
+  }
+});

package/src/reStructure/TemplateData/flowSchema/CreateRecordComplete/mainFunction/data.js

@@ -0,0 +1,68 @@
+/*
+Copyright (C) 2020 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict';
+const path = require('path');
+
+const { SOURCE_PATH, FUNCTION_NAME, FLOW_SCHEMA_EVENT_TYPE } = require('../../../../../MainLibs/src/Consts');
+const { firstLetterLowerCase: lowerCase, firstLetterUpperCase: upperCase } = require("../../../../../MainLibs/src/Utils")
+const templatePath = path.join(__dirname, "template.ejs");
+const getObjectSchema = require('@izara_project/izara-core-library-service-schemas/src/GetObjectSchema');
+
+/**
+ * receive objectSchema
+ * create data for WebScoket handler template
+ *
+ * @param {Object} objectSchema
+ * @return {{templatePath, templateData,setting}}
+ */
+
+
+function data(_izContext, flowSchema, srcPath) {
+  return [createParamsForCreateSource(_izContext, flowSchema, srcPath)]
+}
+
+function createParamsForCreateSource(_izContext, flowSchema, srcPath) {
+  let functionName = upperCase(FUNCTION_NAME.createRecordComplete)
+  let flowSteps = Object.keys(flowSchema.flowSteps)
+  let splitFlowSteps = flowSteps[0].split("_")
+  let topicArn;
+  let serviceTag;
+  for (const event of flowSchema.event) {
+    if (event.includes(FLOW_SCHEMA_EVENT_TYPE.ownTopic)) {
+      topicArn = flowSteps[0]
+    } else if (event.includes(FLOW_SCHEMA_EVENT_TYPE.extTopic)) {
+      topicArn = splitFlowSteps[3] + "_" + splitFlowSteps[4]
+      serviceTag = splitFlowSteps[0]
+    }
+  }
+  return {
+    templatePath: templatePath,
+    templateData: {
+      topicArn,
+      serviceTag
+    },
+    setting: {
+      savePath: path.join(srcPath, SOURCE_PATH.createRecordComplete, flowSchema.flowTag, 'source/'),
+      saveFileName: `${functionName}_Main`,
+      fileExtension: ".js",
+      isAppend: false
+    }
+  }
+}
+
+module.exports = data

package/src/reStructure/TemplateData/flowSchema/CreateRecordComplete/mainFunction/template.ejs

@@ -0,0 +1,131 @@
+/*
+Copyright (C) 2020 Sven Mason <http: //izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see
+<http: //www.gnu.org/licenses />.
+*/
+
+'use strict';
+
+const hash = require("object-hash");
+const { getObjectSchema } = require('@izara_project/izara-core-library-service-schemas');
+
+const dynamodbSharedLib = require('@izara_project/izara-core-library-dynamodb');
+const snsSharedLib = require('@izara_project/izara-core-library-sns');
+const asyncFlowSharedLib = require('@izara_project/izara-core-library-asynchronous-flow');
+const callingFlowSharedLib = require('@izara_project/izara-core-library-calling-flow');
+const lambdaSharedLib = require('@izara_project/izara-core-library-lambda');
+
+const externalRequest = require('@izara_project/izara-core-library-external-request');
+const sns = externalRequest.sns
+const { postToConnection } = require("../../../libs/source/GenerateCodeLibs");
+
+const utils = require('@izara_project/izara-market-library-service-schemas/src/MainLibs/src/Utils')
+const { NoRetryError } = require('@izara_project/izara-core-library-core')
+const { v4: uuidv4 } = require('uuid')
+const coreConsts = require("@izara_project/izara-core-library-core/src/Consts");
+// const { TOPIC_NAME_GENERATE_CODE } = require('@izara_project/izara-market-library-service-schemas/src/GenerateCodeLibs/src/Consts')
+const { generateCodeLibs: { consts: TOPIC_NAME_GENERATE_CODE } } = require('@izara_project/izara-market-library-service-schemas')
+
+/**
+ * description of function.
+ * @param {Object} _izContext
+ * @param {CorrelationIds} _izContext.correlationIds - property of _izContext
+ * @param {Logger} _izContext.logger - property of _izContext
+ * @param {Object} requestParams - request params
+ * @param {Object} requestParams.identifiers - identifiers for get data
+ * @param {Object} requestParams.additionalRequest - additionalRequest
+ *
+ * @returns {object} description of return value
+ */
+module.exports.<%- functionMain%> = async (
+  _izContext,
+  requestParams,
+  callingFlowConfig = {},
+  //(<additionalParams>)
+  //(</additionalParams>)
+) => {
+  try {
+    _izContext.logger.debug("WebSocketInvoke _izContext", _izContext)
+    _izContext.logger.debug("WebSocketInvoke requestParams", requestParams)
+    _izContext.logger.debug("WebSocketInvoke callingFlowConfig", callingFlowConfig)
+
+    const connectionId = requestContext.connectionId;
+    let correlationId = _izContext.correlationIds.get(consts.X_CORRELATION_ID);
+
+    const getRegisterRecords = await dynamodbSharedLib.getItem(
+      _izContext,
+      await dynamodbSharedLib.tableName(_izContext, "RegisterRecords"),
+      {
+        taskKey: correlationId,
+        connectionId: connectionId,
+      }
+    );
+    _izContext.logger.debug("getRegisterRecords", getRegisterRecords)
+    if (!getRegisterRecords) {
+      throw new NoRetryError(`connectionId not found ${connectionId}`)
+    }
+
+    const flowType = getRegisterRecords.flowType;
+    const flowTag = flowType.split('_')[0];
+    const getFlowSchema = await getObjectSchema.getFlowSchemaS3WithCache(_izContext, {
+      flowTag: flowTag,
+      serviceTag: process.env.iz_serviceTag
+    })
+    _izContext.logger.debug("getFlowSchema", getFlowSchema)
+    if (!getFlowSchema) {
+      throw new NoRetryError(`flowTag not found ${flowTag}`)
+    }
+
+    switch (flowSchema.statusType) {
+      case "statusField":
+        let payload = {
+          identifiers: requestParams.identifiers,
+          objectType: flowSchema.objType.objectType
+        }
+        let result = await externalRequest.lambda.invokeSync(
+          _izContext,
+          await lambdaSharedLib.lambdaFunctionName(_izContext, "GetHdrInv", flowSchema.objType.serviceTag),
+          payload
+        )
+        if (!result) {
+          throw new NoRetryError(`statusField not found ${requestParams.identifiers}`)
+        }
+        return result
+        case "storedCache":
+        // if statusType=storedCache then query storedCache > post message to client -> remove connectionId if status = error || complete
+        const getStoredCache = await dynamodbSharedLib.getItem(
+          _izContext,
+          await dynamodbSharedLib.tableName(_izContext, flowSchema.flowTag + "Main"),
+          {
+            taskKey: requestParams.identifiers,
+            connectionId: requestParams.connectionId
+          }
+        )
+        if (!getStoredCache) {
+          throw new NoRetryError(`storedCache not found ${requestParams.identifiers}`)
+        }
+        return getStoredCache
+      case "none":
+        // if statusType=none -> post message to client ---- not remove connectionId when post message, can remove connectionId when cannot use connectionId only(user disconnect)
+        break
+      default:
+        throw new NoRetryError(`statusType not found ${flowSchema.statusType}`)
+    }
+
+  } catch (err) {
+    _izContext.logger.error('error WebSocketInvoke: ', err)
+    throw (err)
+  }
+}

package/src/reStructure/TemplateData/flowSchema/CreateRecordComplete/queue/data.js

@@ -0,0 +1,77 @@
+/*
+Copyright (C) 2020 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict';
+
+const path = require("path")
+
+
+// const {
+//   HANDLER,
+// STORAGE_TYPES
+// } = require('@izara_project/izara-core-library-service-schemas/src/Consts');
+
+const { firstLetterLowerCase: lowerCase, firstLetterUpperCase: upperCase } = require("../../../../../MainLibs/src/Utils.js");
+const STORAGE_TYPES = {
+  dynamoDB: "dynamoDB",
+  graph: "graph"
+}
+const templatePath = path.join(__dirname, "./template.ejs");
+const { SOURCE_PATH, SAVE_FILE_NAME, FUNCTION_NAME, HANDLER } = require('../../../../../MainLibs/src/Consts.js');
+const { TOPIC_NAME_GENERATE_CODE } = require("../../../../../GenerateCodeLibs/src/Consts.js");
+
+/**
+ * create data for dynamoDbYaml tempalte from objectSchema
+ * return array of dynamoDb data
+ * because one objectSchema can create multiple dynamoDb table
+ *
+ * @param {Object} _izContext
+ * @param {String} saveFilePath
+ * @returns {Object[]}              - data of multiple dynamoDb template
+ */
+
+function createDataForDefaultSnsInSqs(_izContext, flowSchema, srcPath) {
+  let resultsForCreateDefaultSnsInSqs = [];
+
+  const setting = {
+    initialData: "Resources:\n",
+    savePath: path.join(srcPath, SOURCE_PATH.resourceYaml),
+    saveFileName: SAVE_FILE_NAME.snsInSqsYaml,
+    fileExtension: ".yml",
+    isAppend: true
+  }
+
+  const generatedSnsTopicInForFlowSchema = [
+    {
+      queueName: upperCase(FUNCTION_NAME.createRecordComplete) + upperCase(flowSchema.flowTag) + upperCase(HANDLER.hdrSqs),
+      subScription: upperCase(TOPIC_NAME_GENERATE_CODE.outCreateNodeComplete)
+    },
+  ];
+
+
+  for (let defaultSnsInSqsData of generatedSnsTopicInForFlowSchema) {
+    resultsForCreateDefaultSnsInSqs.push({
+      templatePath: templatePath,
+      templateData: defaultSnsInSqsData,
+      setting: setting
+    })
+  }
+
+  return resultsForCreateDefaultSnsInSqs;
+}
+
+module.exports = createDataForDefaultSnsInSqs;

package/src/reStructure/TemplateData/flowSchema/CreateRecordComplete/queue/template.ejs

@@ -0,0 +1,49 @@
+  Subscription<%- queueName-%>:
+    Type: AWS::SNS::Subscription
+    Properties:
+      TopicArn: !Ref <%- subScription %>
+      Endpoint: "arn:aws:sqs:${self:custom.iz_region}:${self:custom.iz_accountId}:${self:custom.iz_resourcePrefix}<%- queueName %>"
+      Protocol: "sqs"
+  #------- queue ---------
+  <%- queueName %>:
+    Type: "AWS::SQS::Queue"
+    Properties:
+      QueueName: ${self:custom.iz_resourcePrefix}<%- queueName %>
+      RedrivePolicy:
+        deadLetterTargetArn: #!GetAtt
+          Fn::GetAtt:
+            - <%- queueName %>DLQ
+            - Arn
+        maxReceiveCount: 3
+      VisibilityTimeout: 120
+
+  <%- queueName %>DLQ:
+    Type: AWS::SQS::Queue
+    Properties:
+      QueueName: ${self:custom.iz_resourcePrefix}<%- queueName %>DLQ
+
+  <%- queueName %>Policy:
+    Type: AWS::SQS::QueuePolicy
+    Properties:
+      PolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Sid: "allow-sns-messages"
+            Effect: Allow
+            Principal: "*"
+            Resource: #!GetAtt
+              Fn::GetAtt:
+                - <%- queueName %>
+                - Arn
+            Action: "SQS:SendMessage"
+      Queues:
+        - Ref: <%- queueName %>
+      #<#<%- firstLetterUpperCase(queueName) %>QueueSetting#>
+      #<#/<%- firstLetterUpperCase(queueName) %>QueueSetting#>
+
+<%_ function firstLetterUpperCase(text){
+return text.charAt(0).toUpperCase() + text.slice(1)
+} _%>
+<%_ function firstLetterLowerCase(str) {
+return str.charAt(0).toLowerCase() + str.slice(1)
+} _%>

package/src/reStructure/TemplateData/flowSchema/components/upload/createObject/handler/template.ejs

@@ -26,7 +26,7 @@ const callingFlowSharedLib = require('@izara_project/izara-core-library-calling-
 const hash = require('object-hash')
 // const coreConsts = require("@izara_project/izara-core-library-core/src/Consts");
 const consts = require('@izara_project/izara-middleware/src/MiddlewareCore/Consts')
-const { postToConnection } = require("../../libs/source/GenerateCodeLibs");
+const { postToConnection } = require('../../../libs/source/GenerateCodeLibs')
 const createObject = require('./CreateObjectS3_Main');
 module.exports.main = middleware.wrap(async (event, context, callback) => {
 

package/src/reStructure/TemplateData/flowSchema/components/upload/createObjectComplete/mainFunction/template.ejs

@@ -21,7 +21,7 @@ const snsSharedLib = require('@izara_project/izara-core-library-sns');
 const asyncFlowSharedLib = require('@izara_project/izara-core-library-asynchronous-flow');
 let coreConsts = require('@izara_project/izara-middleware/src/MiddlewareCore/Consts')
 
-const { postToConnection } = require('../../libs/source/GenerateCodeLibs')
+const { postToConnection } = require('../../../libs/source/GenerateCodeLibs')
 module.exports.webSocketTaskComplete = async (
   _izContext,
   objType,

package/src/reStructure/TemplateData/flowSchema/components/upload/preSignUrl/createPresignUrl/mainFunction/template.ejs

@@ -32,7 +32,7 @@ const client = new S3Client({ region: 'us-east-2' });
 const { createPresignedPost } = require('@aws-sdk/s3-presigned-post');
 const consts = require('@izara_project/izara-middleware/src/MiddlewareCore/Consts')
 
-const { postToConnection,getNestObject } = require("../../libs/source/GenerateCodeLibs");
+const { postToConnection,getNestObject } = require('../../../../libs/source/GenerateCodeLibs')
 const hash = require('object-hash');
 const utils = require('@izara_project/izara-market-library-service-schemas/src/MainLibs/src/Utils');
 const coreConsts = require('@izara_project/izara-core-library-core/src/Consts');

package/src/reStructure/TemplateData/flowSchema/components/upload/relate/libs/data.js

@@ -22,14 +22,22 @@ const { SOURCE_PATH, SAVE_FILE_NAME } = require('../../../../../../../MainLibs/s
 
 const templatePath = path.join(__dirname, "./template.ejs")
 
-function data(_izContext, srcPath) {
-  return [createSourceParams(_izContext, srcPath)]
+function data(_izContext, flowSchema, srcPath) {
+  return [createSourceParams(_izContext,flowSchema, srcPath)]
 };
 
-function createSourceParams(_izContext, srcPath) {
+function createSourceParams(_izContext, flowSchema, srcPath) {
+  let uploadS3 = false;
+  for (const event of flowSchema.event) {
+    if (event.includes("s3")) {
+      uploadS3 = true
+    }
+  }
   return {
     templatePath: templatePath,
-    templateData: {},
+    templateData: {
+      uploadS3
+    },
     setting: {
       saveFileName: "GenerateCodeLibs",
       savePath: path.join(srcPath, SOURCE_PATH.generateCodeLib),

package/src/reStructure/TemplateData/flowSchema/components/upload/relate/libs/template.ejs

@@ -21,14 +21,15 @@ const externalRequest = require('@izara_project/izara-core-library-external-requ
 const lambda = externalRequest.lambda;
 
 const { ApiGatewayManagementApiClient, PostToConnectionCommand } = require("@aws-sdk/client-apigatewaymanagementapi");
-const { inMemoryCacheLib } = require('@izara_project/izara-shared');
 const hash = require("object-hash")
-const csv = require('csv-parser');
-const streamifier = require('streamifier');
-const NoRetryError = require('@izara_project/izara-core-library-core').NoRetryError;
+<% if (uploadS3) { %>
+  const csv = require('csv-parser');
+  const streamifier = require('streamifier');
+<% } %>
+const { NoRetryError,inMemoryCacheLib } = require('@izara_project/izara-core-library-core');
 
 const client = new ApiGatewayManagementApiClient({
-  endpoint: `https://${process.env.iz_webSocketEndpoint}`
+  endpoint: `${process.env.iz_webSocketEndpoint}`
 })
 
 async function postToConnection(message, connectionId) {
@@ -60,7 +61,7 @@ async function getDataFromInvokeFunction(_izContext, functionName, payload) {
   return await lambda.invokeSync(_izContext, functionName, payload)
 }
 
-const getDataFromInvokeFunctionWithCache = inMemoryCacheLib.inMemoryCacheLib(
+const getDataFromInvokeFunctionWithCache = inMemoryCacheLib(
   getDataFromInvokeFunction,
   {
     max: 100,
@@ -73,6 +74,7 @@ const getDataFromInvokeFunctionWithCache = inMemoryCacheLib.inMemoryCacheLib(
   }
 )
 
+<% if (uploadS3) { %>
 /**
  *
  * @param {Object} _izContext
@@ -108,6 +110,7 @@ async function isValidCsv(_izContext, data) {
 
   return [validCsv, errorCheckCsv]
 }
+<% } %>
 
 /**
  *
@@ -132,6 +135,8 @@ module.exports = {
   postToConnection,
   getDataFromInvokeFunction,
   getDataFromInvokeFunctionWithCache,
-  isValidCsv,
-  getNestObject
+  getNestObject,
+  <% if (uploadS3) { %>
+    isValidCsv,
+  <% } %>
 }

package/src/reStructure/TemplateData/flowSchema/components/websocket/connect/handler/template.ejs

@@ -32,13 +32,13 @@ module.exports.main = middleware.wrap(async (event, context, callback) => {
       switch (eventType) {
         case 'CONNECT':
           console.log('Connect websocket')
-          //(<additionalParams>)
-          //(</additionalParams>)
+          //(<webSocketConnect>)
+          //(</webSocketConnect>)
           break
         case 'DISCONNECT':
           console.log('Websocket are Disconnection')
-          //(<additionalParams>)
-          //(</additionalParams>)
+          //(<webSocketDisconnect>)
+          //(</webSocketDisconnect>)
           break
         case '$default':
           console.log('Received unknown route:', route)