@izara_project/izara-market-library-service-schemas 1.0.18 → 1.0.19

package/src/TemplateManager/src/flowSchema/UploadS3Case/relate/libs/template.ejs

@@ -0,0 +1,137 @@
+/*
+Copyright (C) 2021 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict';
+
+const externalRequest = require('@izara_project/izara-core-library-external-request');
+const lambda = externalRequest.lambda;
+
+const { ApiGatewayManagementApiClient, PostToConnectionCommand } = require("@aws-sdk/client-apigatewaymanagementapi");
+const { inMemoryCacheLib } = require('@izara_project/izara-shared');
+const hash = require("object-hash")
+const csv = require('csv-parser');
+const streamifier = require('streamifier');
+const NoRetryError = require('@izara_project/izara-core-library-core').NoRetryError;
+
+const client = new ApiGatewayManagementApiClient({
+  endpoint: `https://${process.env.iz_webSocketEndpoint}`
+})
+
+async function postToConnection(message, connectionId) {
+  try {
+    const messageParams = JSON.stringify(message)
+    const command = new PostToConnectionCommand({
+      ConnectionId: connectionId,
+      Data: Buffer.from(messageParams)
+    })
+    // console.log("command::", command)
+    // console.log("client::", client)
+    await client.send(command);
+    console.log(`send Message: ${JSON.stringify(message)} to ${connectionId} 1`)
+  } catch (err) {
+    if (err.name === "GoneException") {
+      console.log(`message send to connectionIds ${connectionId} is no longer connect`)
+    } else {
+      console.error(`Error post message: ${err}`)
+    }
+  }
+}
+/**
+ *
+ * @param {Object} _izContext
+ * @param {string} functionName
+ * @param {Object} payload
+ */
+async function getDataFromInvokeFunction(_izContext, functionName, payload) {
+  return await lambda.invokeSync(_izContext, functionName, payload)
+}
+
+const getDataFromInvokeFunctionWithCache = inMemoryCacheLib.inMemoryCacheLib(
+  getDataFromInvokeFunction,
+  {
+    max: 100,
+    maxAge: 86400000,
+    promise: true,
+    profileName: "getDataFromInvokeFunction",
+    normalizer: function (args) {
+      return hash(args[1], args[2])
+    }
+  }
+)
+
+/**
+ *
+ * @param {Object} _izContext
+ * @param {Object} data - S3 data
+ * @returns
+ */
+async function isValidCsv(_izContext, data) {
+  _izContext.logger.debug("function checkValidCsv", { data })
+  let errorCheckCsv = [];
+  let validCsv = true;
+  let rowCount = 0;
+  let columnCount = null;
+
+  streamifier.createReadStream(data.Body).pipe(csv()).on('data', (row) => {
+    rowCount++;
+    if (columnCount === null) {
+      columnCount = Object.keys(row).length
+    } else if (Object.keys(row).length !== columnCount) {
+      validCsv = false
+      errorCheckCsv.push(`Row ${rowCount} has inconsistent columns: ${JSON.stringify(row)}`)
+    }
+  })
+    .on('end', () => {
+      if (rowCount === 0) {
+        errorCheckCsv.push("csv file not have data row")
+      } else {
+        _izContext.logger.debug("validCsv")
+      }
+    })
+    .on('error', (error) => {
+      errorCheckCsv.push(`error:: ${error}`)
+    })
+
+  return [validCsv, errorCheckCsv]
+}
+
+/**
+ *
+ * @param {Object} _izContext
+ * @param {Object} nestedObj - source Object
+ * @param {String[]} paths - path to get value
+ */
+function getNestObject(_izContext, nestedObj, paths) {
+  _izContext.logger.debug("function getNestObject", { nestedObj, paths });
+  let data;
+
+  if (typeof nestedObj === "object" && Array.isArray(paths)) {
+    data = paths.reduce((obj, key) => obj && typeof obj[key] !== 'undefined' ? obj[key] : undefined, nestedObj);
+  } else {
+    throw new NoRetryError("Type error: Invalid type in getNestObject")
+  }
+
+  return data;
+}
+
+module.exports = {
+  postToConnection,
+  getDataFromInvokeFunction,
+  getDataFromInvokeFunctionWithCache,
+  isValidCsv,
+  getNestObject
+}

package/src/TemplateManager/src/libs/Consts.js

@@ -54,7 +54,7 @@ function resourceNames(resourceClass, resourceName) {
     [RESOURCE_CLASSES.s3]: "arn:aws:s3:::" + resourceName,
     [RESOURCE_CLASSES.dynamoDbTable]: "arn:aws:dynamodb:${self:custom.iz_region}:${self:custom.iz_accountId}:table/${self:custom.iz_resourcePrefix}" + resourceName,
     [RESOURCE_CLASSES.sns]: "arn:aws:sns:${self:custom.iz_region}:${self:custom.iz_accountId}:${self:custom.iz_resourcePrefix}" + resourceName,
-    [RESOURCE_CLASSES.lambda]: "arn:aws:lambda:${self:custom.iz_region}:${self:custom.iz_accountId}:function:" + resourceName,
+    [RESOURCE_CLASSES.lambda]: "arn:aws:lambda:${self:custom.iz_region}:${self:custom.iz_accountId}:function:${self:custom.iz_resourcePrefix}" + resourceName,
     [RESOURCE_CLASSES.webSocket]: "arn:aws:execute-api:${self:custom.iz_region}:${self:custom.iz_accountId}:${self:custom.iz_webSocketHostId}/Test/POST/@connections/{connectionId}"
   }[resourceClass];
 }
@@ -67,13 +67,24 @@ function resourceNames(resourceClass, resourceName) {
  * @returns {{ effect, action, resource}}
  */
 function createIamRole(action, resource = [], effect = "Allow") {
-  for (const resourceAction in action) {
-    if (typeof (action[resourceAction]) === 'string') {
-      action[resourceAction] = [action[resourceAction]]
-    }
+  if (Array.isArray(action)) {
+    const actionObj = {};
+    action.forEach(act => {
+      if (act.includes(":")) {
+        const [serviceAction, resourceAction] = act.split(":")
+        actionObj[serviceAction] = [resourceAction]
+      }
+    });
+    action = actionObj
+  } else {
+    for (const resourceAction in action) {
+      if (typeof (action[resourceAction]) === 'string') {
+        action[resourceAction] = [action[resourceAction]]
+      }
 
-    if (!Array.isArray(action[resourceAction])) {
-      throw new Error(`resourceAction:${resourceAction} should be string or array`);
+      if (!Array.isArray(action[resourceAction])) {
+        throw new Error(`resourceAction:${resourceAction} should be string or array`);
+      }
     }
   }
 
@@ -103,7 +114,9 @@ const SOURCE_PATH = {
   resourceLocationTranslateId: 'src/generatedCode/TranslateId/source/',
   resourceLocationProcessLogical: "src/generatedCode/ProcessLogical/source/",
   webSocket: "../src/generatedCode/WebSocket/source/",
-  resourceLocationWebSocket: "src/generatedCode/WebSocket/source/"
+  resourceLocationWebSocket: "src/generatedCode/WebSocket/source/",
+  generateCodeLib: "../src/generatedCode/libs/source/",
+  webSocketHook: "../src/generatedCode/WebSocket/source/hookFunction/source/"
 };
 
 
@@ -124,7 +137,7 @@ const DEFAULT_LAMBDA_ROLE_PER_ACTION = {
     dynamoDbPermission: 'DeleteItem',
     httpMethod: 'delete',
   }
-}
+};
 
 const SQS_RESOURCE = {
   sendMessage: "SendMessage",
@@ -133,12 +146,12 @@ const SQS_RESOURCE = {
   getQueueAttributes: "GetQueueAttributes",
   deleteMessageBatch: "DeleteMessageBatch",
   getQueueUrl: "GetQueueUrl"
-}
+};
 
 const SNS_RESOURCE = {
   publish: "Publish",
-  subscription: "Subscription"
-}
+  subscribe: "Subscribe"
+};
 
 const DYNAMO_RESOURCE = {
   putItem: "PutItem",
@@ -146,11 +159,21 @@ const DYNAMO_RESOURCE = {
   query: "Query",
   updateItem: "UpdateItem",
   deleteItem: "DeleteItem"
-}
+};
 
 const LAMBDA_RESOURCE = {
   invokeFunction: "InvokeFunction"
-}
+};
+
+const S3_RESOURCE = {
+  getObject: "GetObject",
+  getObjectTagging: "GetObjectTagging",
+  deleteObject: "DeleteObject",
+  deleteObjectTagging: "DeleteObjectTagging",
+  putObject: "PutObject",
+  putObjectTagging: "PutObjectTagging",
+};
+
 function externalResourceName(resourceClass, resourceName, serviceTag) {
   return {
     [RESOURCE_CLASSES.dynamoDbTable]: "arn:aws:dynamodb:${process.env.iz_region}:${process.env.iz_accountId}:table/${await dynamoSharedLib.tableName(_izContext, " + `"${resourceName}" ,"${serviceTag}")}`,
@@ -158,11 +181,11 @@ function externalResourceName(resourceClass, resourceName, serviceTag) {
     [RESOURCE_CLASSES.dynamoDbAcrossService]: "arn:aws:dynamodb:${process.env.iz_region}:${process.env.iz_accountId}:table/*" + `${resourceName}`,
     [RESOURCE_CLASSES.lambda]: "arn:aws:lambda:${process.env.iz_region}:${process.env.iz_accountId}:function:${await lambdaSharedLib.lambdaFunctionName(_izContext, " + `"${resourceName}" ,"${serviceTag}")}`,
   }[resourceClass]
-}
+};
 
 function externalResourceSns(resourceName, serviceTag) {
   return "arn:aws:sns:${process.env.iz_region}:${process.env.iz_accountId}:${await snsSharedLib.snsTopicName(_izContext, " + `"${resourceName}", "${serviceTag}")}`
-}
+};
 
 const SAVE_FILE_NAME = {
   dynamoDbYaml: "generated-dynamoDB-table",
@@ -175,8 +198,9 @@ const SAVE_FILE_NAME = {
   translateId: "TranslateIdsRequest",
   findData: "findData",
   processLogicalYaml: "processLogical",
-  flowSchema: "flowSchema"
-}
+  flowSchema: "flowSchema",
+  S3Bucket: "generated-s3-bucket"
+};
 
 const FUNCTION_NAME = {
   updateRel: "updateRelationship",
@@ -195,8 +219,11 @@ const FUNCTION_NAME = {
   getNodeComplete: "getNodeComplete",
   paginateProcessLogical: "paginateProcessLogical",
   deleteNode: "deleteNode",
-  deleteNodeComplete: "deleteNodeComplete"
-}
+  deleteNodeComplete: "deleteNodeComplete",
+  createObjectS3: "createObjectS3",
+  createObjectS3Complete: "webSocketTaskComplete",
+  processAfterUploadS3: "processAfterUploadS3"
+};
 
 const SHORT_FUNCTION_NAME = {
   updateRel: "updateRel",
@@ -215,7 +242,8 @@ const SHORT_FUNCTION_NAME = {
   getNodeComplete: "getNodeComp",
   paginateProcessLogical: "paginateProLogical",
   deleteNode: "deleteNode",
-  deleteNodeComplete: "deleteNodeComp"
+  deleteNodeComplete: "deleteNodeComp",
+  createObjectS3Complete: "webSocketTaskComp",
 }
 
 const getGraphServiceNameFromAllRelSchema = async (_izContext, allRelSchemas) => {
@@ -233,7 +261,7 @@ const getGraphServiceNameFromAllRelSchema = async (_izContext, allRelSchemas) =>
   }))
   const graphServiceName = [...graphServiceNameSet]
   return graphServiceName
-}
+};
 
 function defaultIamRolePerAction() {
 
@@ -263,7 +291,7 @@ function defaultIamRolePerAction() {
   );
 
   return defaultIamRole
-}
+};
 
 function getComplexFilterServiceTag(allObjectSchemas) {
   const complexFilterServiceTagSet = new Set();
@@ -272,7 +300,7 @@ function getComplexFilterServiceTag(allObjectSchemas) {
   }
   let complexFilterServiceTags = [...complexFilterServiceTagSet];
   return complexFilterServiceTags
-}
+};
 
 async function getGraphServiceNameFromObjectSchema(_izContext, allObjSchemas) {
   const graphServiceName = new Set();
@@ -287,7 +315,7 @@ async function getGraphServiceNameFromObjectSchema(_izContext, allObjSchemas) {
   let graphServiceNames = [...graphServiceName]
   // console.log("graphServiceName", graphServiceNames)
   return graphServiceNames
-}
+};
 
 function awaitingMultipleStepsRole() {
   let awaitingMultipleStepsRole = createIamRole(
@@ -306,11 +334,11 @@ function awaitingMultipleStepsRole() {
     ]
   )
   return awaitingMultipleStepsRole
-}
+};
 
 function shortNameHandler(fullNameHandler) {
   return fullNameHandler.slice(3)
-}
+};
 
 
 module.exports = {
@@ -336,5 +364,6 @@ module.exports = {
   getGraphServiceNameFromObjectSchema,
   awaitingMultipleStepsRole,
   shortNameHandler,
-  SHORT_FUNCTION_NAME
+  SHORT_FUNCTION_NAME,
+  S3_RESOURCE
 }

package/src/TemplateManager/src/flowTag/webSocket/webSocketComplete/mainFunction/template.ejs

@@ -1,180 +0,0 @@
-/*
-Copyright (C) 2020 Sven Mason <http://izara.io>
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-'use strict';
-const { NoRetryError } = require('@izara_project/izara-core-library-core');
-const izaraShared = require('@izara_project/izara-shared');
-const dynamodbSharedLib = izaraShared.dynamodbSharedLib;
-const snsSharedLib = izaraShared.snsSharedLib;
-const asyncFlowSharedLib = izaraShared.asyncFlowSharedLib;
-const AWS = require('aws-sdk');
-const api = new AWS.ApiGatewayManagementApi({
-  endpoint: process.env.iz_webSocketEndpoint,
-});
-
-module.exports.webSocketTaskComplete = async (
-  _izContext,
-  passStatus,
-  errorsFound,
-  uploadUrl,
-  imageImportId,
-  maximumSize,
-  cacheStatus,
-  imageImportCacheData,
-  imageImportCacheId,
-  // route,
-  topicArn
-) => {
-  // _izContext.logger.debug('functionGet params: ', params);
-  try {
-
-    // you must have data in table.
-    console.log("topicArn", topicArn)
-    let taskId = imageImportId;
-    let taskTag = null;
-
-    switch (topicArn) {
-      case snsSharedLib.snsTopicArn("OutReservedImageLimitComplete"):
-        taskTag = 'UploadImage';
-        break
-      // case 'message':
-      //   break
-      // default:
-      //   console.log('Received unknown route:', route)
-    }
-
-    // console.log("topicArn", topicArn)
-    // let taskTag = topicArn.split(":")[5];
-    // let taskId = sortResultId;
-    // console.log("taskTag", taskTag)
-    // switch (taskTag) {
-    //   case 'SortedResultsTestSortResultComplete':
-    //     taskTag = 'SortedResultsTestSortResultComplete';
-    //     break
-    //   // case 'message':
-    //   //   break
-    //   // default:
-    //   //   console.log('Received unknown route:', route)
-    // }
-
-    let webSocketTask = null;
-
-    for (let i = 0; i < 4; i++) {
-      // let value = await dynamodbSharedLib.getItem(
-      webSocketTask = await dynamodbSharedLib.query(
-        _izContext,
-        dynamodbSharedLib.tableName("WebSocketTask"),
-        {
-          taskKey: taskTag + "_" + taskId,
-          // connectionid: "ZKPBId2CCYcCG6A="
-        }
-      );
-
-      if (webSocketTask.length < 1) {
-        continue;
-      } else {
-        break;
-      }
-    }
-    console.log("Item: ", webSocketTask.Items)
-
-    for (let itemIter = 0; itemIter < webSocketTask.Items.length; itemIter++) {
-
-      let connectionid = webSocketTask.Items[itemIter].connectionid;
-      let taskKey = webSocketTask.Items[itemIter].taskKey;
-
-      // if (route) {
-      let data = {};
-
-      if (cacheStatus) {
-        data = {
-          connId: connectionid,
-          cacheStatus: cacheStatus,
-          imageImportId: imageImportId,
-          imageImportCacheId: imageImportCacheId,
-          imageImportCacheData: imageImportCacheData
-        }
-      }
-
-      if (passStatus) {
-        if (passStatus === 'complete') {
-          let maxSize = maximumSize / 1048576
-          data = {
-            connId: connectionid,
-            passStatus: passStatus,
-            imageImportId: imageImportId,
-            uploadUrl: uploadUrl,
-            maximumSize: `${maxSize} MiB`
-          }
-        } else if (passStatus === 'error') {
-          data = {
-            connId: connectionid,
-            passStatus: passStatus,
-            imageImportId: imageImportId,
-            errorsFound: errorsFound
-          }
-        } else {
-          throw new NoRetryError('Another passStatus: pass status is not complete or error')
-        }
-      }
-
-      try {
-        _izContext.logger.debug("endpoint: ", process.env.iz_webSocketEndpoint)
-        await postMessage(
-          data,
-          connectionid
-        )
-      } catch (err) {
-        _izContext.logger.info("postMessage error: ", err, connectionid);
-      }
-      // }
-
-
-      await dynamodbSharedLib.deleteItem(
-        _izContext,
-        dynamodbSharedLib.tableName("WebSocketTask"),
-        {
-          taskKey: taskKey,
-          connectionid: connectionid,
-        }
-      );
-      console.log('testdelete and delete item')
-
-      _izContext.logger.debug("connectionid", connectionid, itemIter)
-    }
-
-    async function postMessage(data, connectionId) {
-      const params = {
-        ConnectionId: connectionId,
-        Data: JSON.stringify(data),
-        // Data: Buffer.from(JSON.stringify(data)),
-      }
-      return api.postToConnection(params).promise();
-    };
-
-    //loop value and send message back to client
-    //fix cachetime
-    const time = new Date();
-    _izContext.logger.debug("value:", webSocketTask, Date.now(), new Date(), time, time.toLocaleTimeString())
-    return {
-      value: webSocketTask
-    }
-
-  } catch (err) {
-    _izContext.logger.error('error websocketTaskComplete: ', err)
-    throw (err);
-  }
-};