@izara_project/izara-core-generate-service-code 1.0.6 → 1.0.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@izara_project/izara-core-generate-service-code",
-  "version": "1.0.6",
+  "version": "1.0.7",
   "description": "Code for locally generating per service files",
   "main": "index.js",
   "scripts": {
@@ -19,6 +19,7 @@
   "jest": {
     "testEnvironment": "node"
   },
+  "type": "module",
   "dependencies": {
     "@izara_project/izara-core-library-core": "^1.0.31",
     "@izara_project/izara-core-library-service-schemas": "^1.0.108",

package/src/TemplateData/IntTest/generateTests/events/data.js

@@ -77,9 +77,9 @@ async function data(_izContext, allIntTestConfig, srcPath) {
           // console.log("have dynamoConfig", inputEventTag, eachStage.dynamoDBConfig)
           for (const dynamoDBConfig of eachStage.dynamoDBConfig) {
             let eventDynamo = {
-              keyValues: dynamoDBConfig.dynamoEvent.identifiers,
               forStageMatching: true,
-              ...setEventDynamo(
+              keyValues: dynamoDBConfig.dynamoEvent.identifiers,
+              ...setEvent(
                 dynamoDBConfig.dynamoEvent.fields,
                 dynamoDBConfig.dynamoEventSettings
               )
@@ -137,7 +137,7 @@ async function data(_izContext, allIntTestConfig, srcPath) {
       }
     }
   }
-  console.log("createEventSources", JSON.stringify(createEventSources, null, 2))
+  // console.log("createEventSources", JSON.stringify(createEventSources, null, 2))
   return createEventSources;
 }
 

package/src/TemplateData/IntTest/generateTests/events/templateEvents.ejs

@@ -2,10 +2,10 @@
   "<%- inputEventTag %>": <%- JSON.stringify(inputConfig, null, 2) %>,
   "<%- outputEventTag %>": <%- JSON.stringify(outputConfig, null, 2) %> <%- eventDynamos.length ? "," : "" -%>
   <% eventDynamos.forEach((eventDynamo,idx) => { %>
-      "<%- eventDynamo.eventTag %>": <%- JSON.stringify(eventDynamo.event,null,2) %> <% idx < eventDynamos.length - 1 ? ",": "" -%>
+      "<%- eventDynamo.eventTag %>": <%- JSON.stringify(eventDynamo.event,null,2) %> <%- idx < eventDynamos.length - 1 ? ",": "" -%>
   <% }) %> <%- eventInvokes.length ? "," : "" -%>
   <% eventInvokes.forEach((invokeEventComponent,index) => { %>
   "<%- invokeEventComponent.inputInvokeName %>": <%- JSON.stringify(invokeEventComponent.inputInvoke,null,2) %>,
-  "<%- invokeEventComponent.outputInvokeName %>": <%- JSON.stringify(invokeEventComponent.outputInvoke,null,2) %><%= index < eventInvokes.length - 1 ? ',': "" -%>
+  "<%- invokeEventComponent.outputInvokeName %>": <%- JSON.stringify(invokeEventComponent.outputInvoke,null,2) %><%- index < eventInvokes.length - 1 ? ",": "" -%>
   <% }) %>
 }

package/src/TemplateData/IntTest/generateTests/tests/data.js

@@ -60,11 +60,10 @@ async function data(_izContext, allIntTestConfig, srcPath) {
             //   'testConfig.seedDataConfig[seedDataTag]',
             //   testConfig.seedDataConfig[seedDataTag]
             // );
-            if (
-              testConfig.seedDataConfig.hasOwnProperty(seedDataTag) &&
-              testConfig.seedDataConfig[seedDataTag].length > 0
-            ) {
+            if (testConfig.seedDataConfig.hasOwnProperty(seedDataTag) && testConfig.seedDataConfig[seedDataTag].length > 0) {
               baseIntTest[seedDataTag] = testConfig.seedDataConfig[seedDataTag];
+            } else {
+              baseIntTest[seedDataTag] = [];
             }
           }
         }
@@ -163,16 +162,11 @@ async function data(_izContext, allIntTestConfig, srcPath) {
           }
 
           // generatedSeedData
-          if (
-            stage.generatedSeedData &&
-            Object.keys(stage.generatedSeedData).length > 0
-          ) {
-            for (const generateSeedType of Object.values(
-              GENERATE_SEED_DATA_TYPE
-            )) {
+          if (stage.generatedSeedData && Object.keys(stage.generatedSeedData).length > 0) {
+            for (const generateSeedType of Object.values(GENERATE_SEED_DATA_TYPE)) {
               if (stage.generatedSeedData.hasOwnProperty(generateSeedType)) {
                 // baseIntTest.stage[currentStageIdx]["generateSeedDatas"] = {
-                stages[currentStageIdx]['generateSeedDatas'] = {
+                stages[currentStageIdx]['generateSeedData'] = {
                   [generateSeedType]: [...stage.generatedSeedData[generateSeedType]]
                 };
               }

package/src/TemplateData/flowSchema/generateTemplateData.js

@@ -133,21 +133,25 @@ import createFlowSchemaRegisterDynamoDb from './register/dynamoDB/register.js';
 
 import createProcessTriggerCacheYaml from './triggerCacheComponent/processTriggerCache/functionYaml/data.js';
 import createCheckTriggerCacheYaml from './triggerCacheComponent/checkTriggerCacheComplete/functionYaml/data.js';
-import createTriggerCacheCompleteYaml from './triggerCacheComponent/triggerCacheComplete/functionYaml/data.js';
+import createTriggerCacheCompleteYamlSqs from './triggerCacheComponent/triggerCacheComplete/functionYaml/sqs/data.js';
+import createTriggerCacheCompleteYamlDsq from './triggerCacheComponent/triggerCacheComplete/functionYaml/dsq/data.js';
 const createFlowSchemaTriggerCacheFunctionYaml = {
   createProcessTriggerCacheYaml,
   createCheckTriggerCacheYaml,
-  createTriggerCacheCompleteYaml
+  createTriggerCacheCompleteYamlSqs,
+  createTriggerCacheCompleteYamlDsq
 };
 
 import createProcessTriggerCacheHandler from './triggerCacheComponent/processTriggerCache/handler/data.js';
 import createCheckTriggerCacheHandler from './triggerCacheComponent/checkTriggerCacheComplete/handler/data.js';
-import createTriggerCacheCompleteHandler from './triggerCacheComponent/triggerCacheComplete/handler/data.js';
+import createTriggerCacheCompleteHandlerSqs from './triggerCacheComponent/triggerCacheComplete/handler/sqs/data.js';
+import createTriggerCacheCompleteHandlerDsq from './triggerCacheComponent/triggerCacheComplete/handler/dsq/data.js';
 
 const createFlowSchemaTriggerCacheHandler = {
   createProcessTriggerCacheHandler,
   createCheckTriggerCacheHandler,
-  createTriggerCacheCompleteHandler
+  createTriggerCacheCompleteHandlerSqs,
+  createTriggerCacheCompleteHandlerDsq
 };
 
 import createProcessTriggerCacheMainFunction from './triggerCacheComponent/processTriggerCache/mainFunction/data.js';

package/src/TemplateData/flowSchema/triggerCacheComponent/triggerCacheComplete/functionYaml/dsq/data.js

@@ -0,0 +1,134 @@
+/*
+Copyright (C) 2020 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import path from 'path';
+import { fileURLToPath } from 'url';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+import { NoRetryError } from '@izara_project/izara-core-library-core';
+import { getObjectSchema } from '@izara_project/izara-core-library-service-schemas';
+import consts from '../../../../../../libs/Consts.js';
+import utils from '../../../../../../libs/Utils.js';
+import libs from '../../../../../../libs/Libs.js';
+
+const {
+  SOURCE_PATH,
+  SAVE_FILE_NAME,
+  HANDLER,
+  RESOURCE_CLASSES,
+  SQS_RESOURCE,
+  DYNAMO_RESOURCE,
+  STORAGE_TYPES
+} = consts;
+const { firstLetterLowerCase: lowerCase, firstLetterUpperCase: upperCase } = utils;
+const {
+  awaitingMultipleStepsRole,
+  resourceNames,
+  defaultIamRolePerAction,
+  createIamRole,
+  shortNameHandler
+} = libs;
+const templatePath = path.join(__dirname, './template.ejs');
+
+/**
+ * receive objectSchema
+ *
+ * @param {Object} objectSchema
+ * @return {{templatePath, templateData,setting}}
+ */
+
+async function data(_izContext, flowSchema, srcPath) {
+  return [await createSourceParams(_izContext, flowSchema, srcPath)];
+}
+
+async function createSourceParams(_izContext, flowSchema, srcPath) {
+  let functionName =
+    'TriggerCache' + upperCase(flowSchema.flowTag) + 'Complete';
+  let handlerType = upperCase(HANDLER.hdrDsq);
+  let additionalResourcePermission = defaultIamRolePerAction();
+  let queueName =
+    'TriggerCache' + upperCase(flowSchema.flowTag) + 'Complete' + handlerType;
+
+  additionalResourcePermission.push(
+    createIamRole(
+      {
+        [RESOURCE_CLASSES.sqs]: Object.values(SQS_RESOURCE)
+      },
+      [
+        resourceNames(RESOURCE_CLASSES.sqs, queueName),
+        resourceNames(RESOURCE_CLASSES.sqs, queueName + 'DLQ')
+      ]
+    )
+  );
+
+  let tableTriggerCache = new Set();
+  let objectSchema = await getObjectSchema
+    .getLocalObjectSchemas(
+      _izContext,
+      [flowSchema.objType.objectType],
+      path.join(srcPath, './schemas')
+    )
+    .then(res => res.records[0]);
+  if (!objectSchema) {
+    throw new NoRetryError('ObjectSchema not found');
+  }
+  for (const storageResource of Object.values(objectSchema.storageResources)) {
+    if (storageResource.storageType === STORAGE_TYPES.dynamoDB) {
+      if (!tableTriggerCache.has(storageResource.tableName)) {
+        tableTriggerCache.add(storageResource.tableName);
+      }
+    }
+  }
+
+  additionalResourcePermission.push(
+    createIamRole(
+      {
+        [RESOURCE_CLASSES.dynamoDbTable]: Object.values(DYNAMO_RESOURCE)
+      },
+      [...tableTriggerCache].map(tableName =>
+        resourceNames(RESOURCE_CLASSES.dynamoDbTable, tableName)
+      )
+    )
+  );
+
+  additionalResourcePermission.push(awaitingMultipleStepsRole());
+  return {
+    templatePath: templatePath,
+    templateData: {
+      resourceLocation: path.join(
+        SOURCE_PATH.resourceLocationFlowSchema,
+        upperCase(flowSchema.flowTag),
+        'source/'
+      ),
+      functionName,
+      handlerType,
+      additionalResourcePermission,
+      functionNameConfig:
+        upperCase(functionName) + upperCase(shortNameHandler(handlerType)),
+      roleName: upperCase(flowSchema.flowTag),
+      queueName
+    },
+    setting: {
+      savePath: path.join(srcPath, SOURCE_PATH.appYaml),
+      saveFileName: upperCase(SAVE_FILE_NAME.flowSchema),
+      fileExtension: '.yml',
+      isAppend: true
+    }
+  };
+}
+
+export default data;

package/src/TemplateData/flowSchema/triggerCacheComponent/triggerCacheComplete/functionYaml/{data.js → sqs/data.js}

@@ -21,9 +21,9 @@ const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 import { NoRetryError } from '@izara_project/izara-core-library-core';
 import { getObjectSchema } from '@izara_project/izara-core-library-service-schemas';
-import consts from '../../../../../libs/Consts.js';
-import utils from '../../../../../libs/Utils.js';
-import libs from '../../../../../libs/Libs.js';
+import consts from '../../../../../../libs/Consts.js';
+import utils from '../../../../../../libs/Utils.js';
+import libs from '../../../../../../libs/Libs.js';
 
 const {
   SOURCE_PATH,

package/src/TemplateData/flowSchema/triggerCacheComponent/triggerCacheComplete/functionYaml/sqs/template.ejs

@@ -0,0 +1,21 @@
+<%_ const join = require('path').join; _%>
+<%- firstLetterUpperCase(functionNameConfig) %>:
+  handler: <%- join(resourceLocation, `${firstLetterUpperCase(functionName)}_${firstLetterUpperCase(handlerType)}.main`)%>
+  name: ${self:custom.iz_resourcePrefix}<%- firstLetterUpperCase(functionName) %><%- firstLetterUpperCase(handlerType) %>
+  role: <%- roleName %>Role
+  #<#<%- functionName %><%- handlerType %>FunctionSetting#>
+  #<#/<%- functionName %><%- handlerType %>FunctionSetting#>
+  events:
+    - sqs:
+        arn: arn:aws:sqs:${self:custom.iz_region}:${self:custom.iz_accountId}:${self:custom.iz_resourcePrefix}<%- queueName %>
+        batchSize: 10
+        filterPatterns: #**** need to update serverless framework upper v.2.69.1
+          - body: {"MessageAttributes":{"callingFlow":{"Value":["${self:custom.iz_resourcePrefix}<%- functionName %>"]}}} # functionName of callingFlow
+          - body: {"MessageAttributes":{"callingFlow":{"Value":[{"exists":false}]}}}
+
+<%_ function firstLetterUpperCase(text){
+  return text.charAt(0).toUpperCase() + text.slice(1)
+} _%>
+<%_ function firstLetterLowerCase(str) {
+  return str.charAt(0).toLowerCase() + str.slice(1)
+} _%>

package/src/TemplateData/flowSchema/triggerCacheComponent/triggerCacheComplete/handler/dsq/data.js

@@ -0,0 +1,69 @@
+/*
+Copyright (C) 2020 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import path from 'path';
+import { fileURLToPath } from 'url';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+import consts from '../../../../../../libs/Consts.js';
+import utils from '../../../../../../libs/Utils.js';
+const { SOURCE_PATH, HANDLER } = consts;
+const {
+  firstLetterLowerCase: lowerCase,
+  firstLetterUpperCase: upperCase,
+  getLocalConfig
+} = utils;
+const templatePath = path.join(__dirname, 'template.ejs');
+
+/**
+ * receive objectSchema
+ *
+ * @param {Object} objectSchema
+ * @return {{templatePath, templateData,setting}}
+ */
+
+function data(_izContext, flowSchema, srcPath) {
+  return [createSourceParams(_izContext, flowSchema, srcPath)];
+}
+
+function createSourceParams(_izContext, flowSchema, srcPath) {
+  let functionName =
+    'TriggerCache' + upperCase(flowSchema.flowTag) + 'Complete';
+  let handlerType = upperCase(HANDLER.hdrDsq);
+  let queueName = 'TriggerCache' + upperCase(flowSchema.flowTag) + 'Complete' + handlerType;
+
+  return {
+    templatePath: templatePath,
+    templateData: {
+      functionName,
+      queueName
+    },
+    setting: {
+      savePath: path.join(
+        srcPath,
+        SOURCE_PATH.flowSchema,
+        upperCase(flowSchema.flowTag),
+        'source/'
+      ),
+      saveFileName: `${upperCase(functionName)}_${upperCase(handlerType)}`,
+      fileExtension: '.js',
+      isAppend: true
+    }
+  };
+}
+
+export default data;

package/src/TemplateData/flowSchema/triggerCacheComponent/triggerCacheComplete/handler/dsq/template.ejs

@@ -0,0 +1,104 @@
+/*
+Copyright (C) 2020 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+
+
+import { middlewareHandler as middleware } from "@izara_project/izara-middleware";
+import Logger from '@izara_project/izara-core-library-logger';
+import { recordHandlerSharedLib } from "@izara_project/izara-core-library-record-handler";
+import callingFlowSharedLib from '@izara_project/izara-core-library-calling-flow';
+
+import <%- functionName %> from './<%- functionName %>_Main.js';
+
+// validate event properties in body.Message of sqs event
+let perRecordsValidatorSchema = {
+  //(<validatorSchema>)
+  //(</validatorSchema>)
+}
+
+middleware.setValidatorSchema(recordHandlerSharedLib.baseValidatorSchema(
+{
+  type: "object",
+  required: ['body', 'messageAttributes'],
+  properties: {
+    body: {
+      type: 'object',
+    },
+    messageAttributes: {
+      type: 'object'
+    }
+  }
+}));
+
+middleware.setValidatorSchema(recordHandlerSharedLib.baseValidatorSchema());
+
+export const main = middleware.wrap(async (event, context, callback) => {
+
+  try {
+
+    let recordPromises = [];
+
+    // loop each record and send to mainFunction
+    await Promise.all(event.Records.map(async record => { // promise.all for map() function
+
+      let passOnProperties = []
+      record = recordHandlerSharedLib.reformatDsqMessage(record._izContext, record);
+      record._izContext.logger.debug('record ReceiveMsgOutHdrSqs', record);
+
+      //validate message (and MessageAttributes)
+      await recordHandlerSharedLib.validateRecord(
+        record, // one record will send to mainFunction
+        "<%- queueName %>", // queue name that need to retry or send to dlq
+        perRecordsValidatorSchema, // schema for record.Message
+        // messageAttributeValidatorSchema   // ----- for msgAttr default is null -> do not send this parameter if not want to validate msgAtt
+      );
+
+      // add argument (to invoke lambda) to passOnProperties[]
+      passOnProperties.push(record.body.Message)
+      //(<additionalParams>)
+      //(</additionalParams>)
+      passOnProperties.push(callingFlowSharedLib.addCallingFlowToPassOnProperties(record.body.Message));
+      record._izContext.logger.debug('passOnProperties in handler', passOnProperties);
+
+      // call recordHandlerSharedLib.recordHandler with 3 parameters and return promise(resolve)
+      let recordPromise = recordHandlerSharedLib.recordHandler(
+        record, // one record will send to mainFunction
+        <%- functionName %>, // mainFunction that need to invoke.
+        "<%- queueName %>", // queue name that need to retry or send to dlq
+        passOnProperties, // all parameters that mainFunction needed.
+      );
+      record._izContext.logger.debug('after recordPromise in handler');
+      recordPromises.push(recordPromise); // push promise to recordPromises
+    }))
+
+    Logger.debug('before Promise.all(recordPromises) in handler');
+    try {
+      // --- main await all promises
+      await Promise.all(recordPromises); // await all promises
+
+      return event.Records // return all for local testing
+
+    } catch {
+      Logger.debug('Promise.all(recordPromises) in handler threw error (at least one record did no resolve)');
+    }
+    Logger.debug('after Promise.all(recordPromises) in handler');
+
+  } catch (err) {
+    Logger.error('Unhandled Error, LambdaFunctionHdrSqs: ', err);
+    throw (err);
+  }
+});

package/src/TemplateData/flowSchema/triggerCacheComponent/triggerCacheComplete/handler/{data.js → sqs/data.js}

@@ -19,8 +19,8 @@ import path from 'path';
 import { fileURLToPath } from 'url';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
-import consts from '../../../../../libs/Consts.js';
-import utils from '../../../../../libs/Utils.js';
+import consts from '../../../../../../libs/Consts.js';
+import utils from '../../../../../../libs/Utils.js';
 const { SOURCE_PATH, HANDLER } = consts;
 const {
   firstLetterLowerCase: lowerCase,

package/src/TemplateData/flowSchema/triggerCacheComponent/triggerCacheQueue/data.js

@@ -66,6 +66,9 @@ function createDataForDefaultSnsInSqs(_izContext, flowSchema, srcPath) {
     },
     {
       queueName: 'TriggerCache' + upperCase(flowTag) + 'Complete' + upperCase(HANDLER.hdrSqs)
+    },
+    {
+      queueName: 'TriggerCache' + upperCase(flowTag) + 'Complete' + upperCase(HANDLER.hdrDsq)
     }
   ];
 

package/src/libs/Consts.js

@@ -275,7 +275,9 @@ const GENERATE_SEED_DATA_TYPE = {
 
 const SEED_DATA_TAG = {
   dynamoDBSeedDataTags: "dynamoDBSeedDataTags",
-  graphSeedDataTags: "graphSeedDataTags"
+  graphSeedDataTags: "graphSeedDataTags",
+  cleaningeDynamoOneRecordsTags: "cleaningeDynamoOneRecordsTags",
+  cleaningeDynamoManyRecordsTags: "cleaningeDynamoManyRecordsTags"
 }
 
 const DEFAULT_LAMBDA_ROLE_PER_ACTION = {