@izara_project/izara-market-library-service-schemas 1.0.17 → 1.0.19

package/src/TemplateManager/src/flowSchema/UploadS3Case/relate/libs/template.ejs

@@ -0,0 +1,137 @@
+/*
+Copyright (C) 2021 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict';
+
+const externalRequest = require('@izara_project/izara-core-library-external-request');
+const lambda = externalRequest.lambda;
+
+const { ApiGatewayManagementApiClient, PostToConnectionCommand } = require("@aws-sdk/client-apigatewaymanagementapi");
+const { inMemoryCacheLib } = require('@izara_project/izara-shared');
+const hash = require("object-hash")
+const csv = require('csv-parser');
+const streamifier = require('streamifier');
+const NoRetryError = require('@izara_project/izara-core-library-core').NoRetryError;
+
+const client = new ApiGatewayManagementApiClient({
+  endpoint: `https://${process.env.iz_webSocketEndpoint}`
+})
+
+async function postToConnection(message, connectionId) {
+  try {
+    const messageParams = JSON.stringify(message)
+    const command = new PostToConnectionCommand({
+      ConnectionId: connectionId,
+      Data: Buffer.from(messageParams)
+    })
+    // console.log("command::", command)
+    // console.log("client::", client)
+    await client.send(command);
+    console.log(`send Message: ${JSON.stringify(message)} to ${connectionId} 1`)
+  } catch (err) {
+    if (err.name === "GoneException") {
+      console.log(`message send to connectionIds ${connectionId} is no longer connect`)
+    } else {
+      console.error(`Error post message: ${err}`)
+    }
+  }
+}
+/**
+ *
+ * @param {Object} _izContext
+ * @param {string} functionName
+ * @param {Object} payload
+ */
+async function getDataFromInvokeFunction(_izContext, functionName, payload) {
+  return await lambda.invokeSync(_izContext, functionName, payload)
+}
+
+const getDataFromInvokeFunctionWithCache = inMemoryCacheLib.inMemoryCacheLib(
+  getDataFromInvokeFunction,
+  {
+    max: 100,
+    maxAge: 86400000,
+    promise: true,
+    profileName: "getDataFromInvokeFunction",
+    normalizer: function (args) {
+      return hash(args[1], args[2])
+    }
+  }
+)
+
+/**
+ *
+ * @param {Object} _izContext
+ * @param {Object} data - S3 data
+ * @returns
+ */
+async function isValidCsv(_izContext, data) {
+  _izContext.logger.debug("function checkValidCsv", { data })
+  let errorCheckCsv = [];
+  let validCsv = true;
+  let rowCount = 0;
+  let columnCount = null;
+
+  streamifier.createReadStream(data.Body).pipe(csv()).on('data', (row) => {
+    rowCount++;
+    if (columnCount === null) {
+      columnCount = Object.keys(row).length
+    } else if (Object.keys(row).length !== columnCount) {
+      validCsv = false
+      errorCheckCsv.push(`Row ${rowCount} has inconsistent columns: ${JSON.stringify(row)}`)
+    }
+  })
+    .on('end', () => {
+      if (rowCount === 0) {
+        errorCheckCsv.push("csv file not have data row")
+      } else {
+        _izContext.logger.debug("validCsv")
+      }
+    })
+    .on('error', (error) => {
+      errorCheckCsv.push(`error:: ${error}`)
+    })
+
+  return [validCsv, errorCheckCsv]
+}
+
+/**
+ *
+ * @param {Object} _izContext
+ * @param {Object} nestedObj - source Object
+ * @param {String[]} paths - path to get value
+ */
+function getNestObject(_izContext, nestedObj, paths) {
+  _izContext.logger.debug("function getNestObject", { nestedObj, paths });
+  let data;
+
+  if (typeof nestedObj === "object" && Array.isArray(paths)) {
+    data = paths.reduce((obj, key) => obj && typeof obj[key] !== 'undefined' ? obj[key] : undefined, nestedObj);
+  } else {
+    throw new NoRetryError("Type error: Invalid type in getNestObject")
+  }
+
+  return data;
+}
+
+module.exports = {
+  postToConnection,
+  getDataFromInvokeFunction,
+  getDataFromInvokeFunctionWithCache,
+  isValidCsv,
+  getNestObject
+}

package/src/TemplateManager/src/libs/Consts.js

@@ -54,7 +54,7 @@ function resourceNames(resourceClass, resourceName) {
     [RESOURCE_CLASSES.s3]: "arn:aws:s3:::" + resourceName,
     [RESOURCE_CLASSES.dynamoDbTable]: "arn:aws:dynamodb:${self:custom.iz_region}:${self:custom.iz_accountId}:table/${self:custom.iz_resourcePrefix}" + resourceName,
     [RESOURCE_CLASSES.sns]: "arn:aws:sns:${self:custom.iz_region}:${self:custom.iz_accountId}:${self:custom.iz_resourcePrefix}" + resourceName,
-    [RESOURCE_CLASSES.lambda]: "arn:aws:lambda:${self:custom.iz_region}:${self:custom.iz_accountId}:function:" + resourceName,
+    [RESOURCE_CLASSES.lambda]: "arn:aws:lambda:${self:custom.iz_region}:${self:custom.iz_accountId}:function:${self:custom.iz_resourcePrefix}" + resourceName,
     [RESOURCE_CLASSES.webSocket]: "arn:aws:execute-api:${self:custom.iz_region}:${self:custom.iz_accountId}:${self:custom.iz_webSocketHostId}/Test/POST/@connections/{connectionId}"
   }[resourceClass];
 }
@@ -67,13 +67,24 @@ function resourceNames(resourceClass, resourceName) {
  * @returns {{ effect, action, resource}}
  */
 function createIamRole(action, resource = [], effect = "Allow") {
-  for (const resourceAction in action) {
-    if (typeof (action[resourceAction]) === 'string') {
-      action[resourceAction] = [action[resourceAction]]
-    }
+  if (Array.isArray(action)) {
+    const actionObj = {};
+    action.forEach(act => {
+      if (act.includes(":")) {
+        const [serviceAction, resourceAction] = act.split(":")
+        actionObj[serviceAction] = [resourceAction]
+      }
+    });
+    action = actionObj
+  } else {
+    for (const resourceAction in action) {
+      if (typeof (action[resourceAction]) === 'string') {
+        action[resourceAction] = [action[resourceAction]]
+      }
 
-    if (!Array.isArray(action[resourceAction])) {
-      throw new Error(`resourceAction:${resourceAction} should be string or array`);
+      if (!Array.isArray(action[resourceAction])) {
+        throw new Error(`resourceAction:${resourceAction} should be string or array`);
+      }
     }
   }
 
@@ -101,7 +112,11 @@ const SOURCE_PATH = {
   resourceLocationFindData: 'src/generatedCode/FindData/source',
   resourceLocationRelationshipPerAction: 'src/generatedCode/RelationshipPerAction/source/',
   resourceLocationTranslateId: 'src/generatedCode/TranslateId/source/',
-  resourceLocationProcessLogical: "src/generatedCode/ProcessLogical/source/"
+  resourceLocationProcessLogical: "src/generatedCode/ProcessLogical/source/",
+  webSocket: "../src/generatedCode/WebSocket/source/",
+  resourceLocationWebSocket: "src/generatedCode/WebSocket/source/",
+  generateCodeLib: "../src/generatedCode/libs/source/",
+  webSocketHook: "../src/generatedCode/WebSocket/source/hookFunction/source/"
 };
 
 
@@ -122,7 +137,7 @@ const DEFAULT_LAMBDA_ROLE_PER_ACTION = {
     dynamoDbPermission: 'DeleteItem',
     httpMethod: 'delete',
   }
-}
+};
 
 const SQS_RESOURCE = {
   sendMessage: "SendMessage",
@@ -131,12 +146,12 @@ const SQS_RESOURCE = {
   getQueueAttributes: "GetQueueAttributes",
   deleteMessageBatch: "DeleteMessageBatch",
   getQueueUrl: "GetQueueUrl"
-}
+};
 
 const SNS_RESOURCE = {
   publish: "Publish",
-  subscription: "Subscription"
-}
+  subscribe: "Subscribe"
+};
 
 const DYNAMO_RESOURCE = {
   putItem: "PutItem",
@@ -144,11 +159,21 @@ const DYNAMO_RESOURCE = {
   query: "Query",
   updateItem: "UpdateItem",
   deleteItem: "DeleteItem"
-}
+};
 
 const LAMBDA_RESOURCE = {
   invokeFunction: "InvokeFunction"
-}
+};
+
+const S3_RESOURCE = {
+  getObject: "GetObject",
+  getObjectTagging: "GetObjectTagging",
+  deleteObject: "DeleteObject",
+  deleteObjectTagging: "DeleteObjectTagging",
+  putObject: "PutObject",
+  putObjectTagging: "PutObjectTagging",
+};
+
 function externalResourceName(resourceClass, resourceName, serviceTag) {
   return {
     [RESOURCE_CLASSES.dynamoDbTable]: "arn:aws:dynamodb:${process.env.iz_region}:${process.env.iz_accountId}:table/${await dynamoSharedLib.tableName(_izContext, " + `"${resourceName}" ,"${serviceTag}")}`,
@@ -156,11 +181,11 @@ function externalResourceName(resourceClass, resourceName, serviceTag) {
     [RESOURCE_CLASSES.dynamoDbAcrossService]: "arn:aws:dynamodb:${process.env.iz_region}:${process.env.iz_accountId}:table/*" + `${resourceName}`,
     [RESOURCE_CLASSES.lambda]: "arn:aws:lambda:${process.env.iz_region}:${process.env.iz_accountId}:function:${await lambdaSharedLib.lambdaFunctionName(_izContext, " + `"${resourceName}" ,"${serviceTag}")}`,
   }[resourceClass]
-}
+};
 
 function externalResourceSns(resourceName, serviceTag) {
   return "arn:aws:sns:${process.env.iz_region}:${process.env.iz_accountId}:${await snsSharedLib.snsTopicName(_izContext, " + `"${resourceName}", "${serviceTag}")}`
-}
+};
 
 const SAVE_FILE_NAME = {
   dynamoDbYaml: "generated-dynamoDB-table",
@@ -172,8 +197,10 @@ const SAVE_FILE_NAME = {
   snsSubscription: "generatedSnsServiceConfig",
   translateId: "TranslateIdsRequest",
   findData: "findData",
-  processLogicalYaml: "processLogical"
-}
+  processLogicalYaml: "processLogical",
+  flowSchema: "flowSchema",
+  S3Bucket: "generated-s3-bucket"
+};
 
 const FUNCTION_NAME = {
   updateRel: "updateRelationship",
@@ -192,8 +219,11 @@ const FUNCTION_NAME = {
   getNodeComplete: "getNodeComplete",
   paginateProcessLogical: "paginateProcessLogical",
   deleteNode: "deleteNode",
-  deleteNodeComplete: "deleteNodeComplete"
-}
+  deleteNodeComplete: "deleteNodeComplete",
+  createObjectS3: "createObjectS3",
+  createObjectS3Complete: "webSocketTaskComplete",
+  processAfterUploadS3: "processAfterUploadS3"
+};
 
 const SHORT_FUNCTION_NAME = {
   updateRel: "updateRel",
@@ -212,7 +242,8 @@ const SHORT_FUNCTION_NAME = {
   getNodeComplete: "getNodeComp",
   paginateProcessLogical: "paginateProLogical",
   deleteNode: "deleteNode",
-  deleteNodeComplete: "deleteNodeComp"
+  deleteNodeComplete: "deleteNodeComp",
+  createObjectS3Complete: "webSocketTaskComp",
 }
 
 const getGraphServiceNameFromAllRelSchema = async (_izContext, allRelSchemas) => {
@@ -230,7 +261,7 @@ const getGraphServiceNameFromAllRelSchema = async (_izContext, allRelSchemas) =>
   }))
   const graphServiceName = [...graphServiceNameSet]
   return graphServiceName
-}
+};
 
 function defaultIamRolePerAction() {
 
@@ -260,7 +291,7 @@ function defaultIamRolePerAction() {
   );
 
   return defaultIamRole
-}
+};
 
 function getComplexFilterServiceTag(allObjectSchemas) {
   const complexFilterServiceTagSet = new Set();
@@ -269,7 +300,7 @@ function getComplexFilterServiceTag(allObjectSchemas) {
   }
   let complexFilterServiceTags = [...complexFilterServiceTagSet];
   return complexFilterServiceTags
-}
+};
 
 async function getGraphServiceNameFromObjectSchema(_izContext, allObjSchemas) {
   const graphServiceName = new Set();
@@ -284,7 +315,7 @@ async function getGraphServiceNameFromObjectSchema(_izContext, allObjSchemas) {
   let graphServiceNames = [...graphServiceName]
   // console.log("graphServiceName", graphServiceNames)
   return graphServiceNames
-}
+};
 
 function awaitingMultipleStepsRole() {
   let awaitingMultipleStepsRole = createIamRole(
@@ -303,11 +334,11 @@ function awaitingMultipleStepsRole() {
     ]
   )
   return awaitingMultipleStepsRole
-}
+};
 
 function shortNameHandler(fullNameHandler) {
   return fullNameHandler.slice(3)
-}
+};
 
 
 module.exports = {
@@ -333,5 +364,6 @@ module.exports = {
   getGraphServiceNameFromObjectSchema,
   awaitingMultipleStepsRole,
   shortNameHandler,
-  SHORT_FUNCTION_NAME
+  SHORT_FUNCTION_NAME,
+  S3_RESOURCE
 }

package/src/TemplateManager/src/flowTag/webSocket/webSocketConnect/functionYaml/template.ejs

@@ -1,10 +0,0 @@
-WebSocketConnect:
-  handler: src/generatedCode/ObjectTypePerActionEndpoint/source/WebSocketConnect.main
-  name: ${self:custom.iz_resourcePrefix}WebSocketConnect
-  events:
-    - websocket:
-        route: $connect
-    - websocket:
-        route: $disconnect
-#<#<WebSocketConnectIamRole#>
-#<#/<WebSocketConnectIamRole#>

package/src/TemplateManager/src/flowTag/webSocket/websocketHandler/lambda/template.ejs

@@ -1,131 +0,0 @@
-/*
-Copyright (C) 2021 Sven Mason <http://izara.io>
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-'use strict';
-
-const izara = require("@izara_project/izara-middleware");
-const middleware = izara.middlewareHandler;
-const AWS = require('aws-sdk');
-const api = new AWS.ApiGatewayManagementApi({ endpoint: process.env.iz_webSocketEndpoint });
-const izaraShared = require('@izara_project/izara-shared')
-const dynamodbSharedLib = izaraShared.dynamodbSharedLib
-const sqsSharedLib = izaraShared.sqsSharedLib;
-const snsSharedLib = izaraShared.snsSharedLib;
-const callingFlowSharedLib = izaraShared.callingFlowSharedLib;
-
-const externalRequest = require('@izara_project/izara-core-library-external-request');
-const sns = externalRequest.sns;
-const getObjectSchema = require('@izara_project/izara-core-library-service-schemas/src/GetObjectSchema')
-
-const utils = require("@izara_project/izara-market-library-service-schemas/src/MainLibs/src/Utils");
-const NoRetryError = require("@izara_project/izara-core-library-core/src/NoRetryError");
-
-module.exports.main = middleware.wrap(async (event, context, callback) => {
-
-  event._izContext.logger.debug('Event:', event);
-
-  try {
-
-    let eventParams = JSON.parse(event.body)
-
-    if (event.requestContext) {
-      const route = event.requestContext.routeKey;
-      const connectionId = event.requestContext.connectionId;
-
-      // const time = Date.now();
-      // console.log(`socketxx: ${time}`);
-
-      switch (route) {
-        case null:
-          console.log('Received unknown route:', route)
-          break
-        case 'webScoket':
-          console.log("event in route", event)
-          let callingFlow = eventParams?.callingFlow ? eventParams.callingFlow : {}
-
-          let getFlowSchema = await getObjectSchema.getLocalFlowSchemas(event._izContext, eventParams.flowTag).then(res => res.records)
-          event._izContext.logger.debug("getFlowSchema", getFlowSchema)
-          let flowSchema = getFlowSchema[0]
-          event._izContext.logger.debug("flowSchema", flowSchema);
-
-          if (!flowSchema) {
-            throw new NoRetryError(`flowTag not found ${flowSchema.flowTag}`)
-          }
-
-          // create messageObject send to topic
-          let messageObject = {
-            propertiesA: eventParams.propertiesA,
-          }
-
-          messageObject = callingFlowSharedLib.addPassBackPropertiesToSnsResponseMessageObject(callingFlow, messageObject);
-
-          let messageAttributes = callingFlowSharedLib.addCallingFlowToSnsResponseMessageAttributes(callingFlow, {});
-
-          let sendMessageToTopic = {
-            Message: JSON.stringify(messageObject),
-            MessageAttributes: messageAttributes,
-            TopicArn: await snsSharedLib.snsTopicArn(event._izContext, `InProcess${utils.firstLetterLowerCase(flowSchema.flowTag)}`)
-          }
-
-          event._izContext.logger.debug(`message Params before send toInProcess${utils.firstLetterUpperCase(flowSchema.flowTag)}`, sendMessageToTopic);
-          await sns.publishAsync(event._izContext, sendMessageToTopic)
-
-
-            await dynamodbSharedLib.putItem(
-              event._izContext,
-              await dynamodbSharedLib.tableName(event._izContext, "WebSocketTask"),
-              {
-                taskKey: flowSchema.flowTag + connectionId,
-                connectionid: connectionId
-              }
-            )
-
-          postMessage(sendMessageToTopic, connectionId)
-          break
-      }
-    } else {
-      const connId = event.connId;
-      await postMessage({ message: event.message }, connId);
-    }
-
-    return (izara.response.webSocketSuccess());
-
-  } catch (err) {
-    console.log("event", event)
-    const connId = event.requestContext.connectionId
-    event._izContext.logger.error('Error, WebSocket: ', err);
-    await sendErrorMessage({ message: err.message }, connId);
-    return (izara.response.failure(err));
-  }
-});
-
-async function postMessage(data, connectionId) {
-  const params = {
-    ConnectionId: connectionId,
-    Data: JSON.stringify(data),
-    // Data: Buffer.from(JSON.stringify(data)),
-  }
-  return api.postToConnection(params).promise();
-};
-
-async function sendErrorMessage(errorMessage, connectionId) {
-  const params = {
-    ConnectionId: connectionId,
-    Data: JSON.stringify(errorMessage)
-  }
-  return api.postToConnection(params).promise();
-}