@izara_project/izara-core-generate-service-code 1.0.73 → 1.0.75

package/src/emit/custom/emitCustomFlowApplicationArtifacts.js

@@ -13,6 +13,7 @@ import {
   customFlowRoleName,
   flowDsqSteps,
   flowExtTopicSteps,
+  generatedDsqFlows,
   generatedOwnTopicFlows,
   supportedCustomFlows,
   upperFirst
@@ -94,6 +95,7 @@ export function customFlowApplicationManagedPaths(
     }),
     ...(supportedCustomFlows(flows).length > 0 ||
     generatedOwnTopicFlows(flows).length > 0 ||
+    generatedDsqFlows(flows).length > 0 ||
     flowExtTopicSteps(flows).length > 0 ||
     flowDsqSteps(flows).length > 0 ||
     includeYaml
@@ -112,7 +114,7 @@ async function emitFlow(flow, objects) {
     flowStepName: '',
     mainFileName: `${process}_Main`,
     functionName: process,
-    startConfig: startConfig(flow)
+    startConfig: _startConfig(flow)
   };
   const artifacts = needsProcessMain(flow)
     ? [
@@ -319,7 +321,7 @@ async function emitExtTopicStep({ flow, step, flowTag, stepName }) {
   const artifacts = [
     {
       path: `${source}/${mainFileName}.js`,
-      content: await renderStepMain(functionName)
+      content: await renderStepMain(functionName, _startConfig(flow))
     }
   ];
   const functions = [];
@@ -373,7 +375,7 @@ async function emitDsqStep({ flow, step, flowTag, stepName }) {
   if (!stepEvents.includes('extTopic')) {
     artifacts.push({
       path: `${source}/${mainFileName}.js`,
-      content: await renderStepMain(functionName)
+      content: await renderStepMain(functionName, startConfig)
     });
   }
   const functions = [];
@@ -418,7 +420,7 @@ async function emitOwnTopicRoot(flow, objects) {
     flowTag: flow.flowTag,
     flowStepName: '',
     upperCaseFlowTag: tag,
-    startConfig: startConfig(flow)
+    startConfig: _startConfig(flow)
   };
   return {
     artifacts: [
@@ -638,8 +640,20 @@ function _s3HandlerData({ action, flow, functionName, mainFileName, handler }) {
   return { functionName, handlerType: handler };
 }
 
-function startConfig(flow) {
-  const start = flow.flowSteps?.[0];
+function _startConfig(flow) {
+  const steps = flow.flowSteps ?? [];
+  const start =
+    steps.find(
+      (s) =>
+        typeof s?.stepName === 'string' &&
+        s.stepName.toLowerCase() === 'inprocess'
+    ) ??
+    steps.find(
+      (s) =>
+        typeof s?.stepName === 'string' &&
+        s.stepName.toLowerCase() === 'start'
+    ) ??
+    steps[0];
   if (start) {
     return stepConfig(flow, start);
   }
@@ -691,7 +705,8 @@ async function renderEndpointMain(flow, functionName, objects) {
     flowTag: flow.flowTag,
     upperCaseFlowTag: upperFirst(flow.flowTag),
     functionName,
-    tableName: storedCacheTableName(flow, objects)
+    tableName: storedCacheTableName(flow, objects),
+    startConfig: _startConfig(flow)
   });
 }
 
@@ -715,8 +730,8 @@ function storedCacheTableName(flow, objects) {
   );
 }
 
-async function renderStepMain(functionName) {
-  return renderCustom('FlowStep_Main.ejs', { functionName });
+async function renderStepMain(functionName, startConfig = null) {
+  return renderCustom('FlowStep_Main.ejs', { functionName, startConfig });
 }
 
 function renderMain(functionName) {

package/src/emit/custom/templates/FlowEndpoint_Main.ejs

@@ -88,7 +88,7 @@ export default async function <%- typeof functionName !== 'undefined' ? function
         createFlowTypeConcat(_izContext, {
           flowTag: '<%= flowTag %>',
           serviceTag: process.env.iz_serviceTag
-        })
+        }),
         //(<storedCacheConfig>)
         //(</storedCacheConfig>)
       );

package/src/emit/custom/templates/triggerCache/CheckTriggerCache_HdrSqs.ejs

@@ -11,7 +11,8 @@ let perRecordsValidatorSchema = {
   properties: {
 <%_ startConfig.properties.forEach((prop, index) => { _%>
     <%- /^[a-zA-Z_$][a-zA-Z0-9_$]*$/.test(prop.propertyName) ? prop.propertyName : `'${prop.propertyName}'` %>: {
-      type: '<%- prop.type || 'string' %>'
+      type: '<%- prop.type || 'string' %>'<% if (typeof prop.default !== 'undefined') { %>,
+      default: <%- typeof prop.default === 'string' ? `'${prop.default.replace(/'/g, "\\'")}'` : JSON.stringify(prop.default) %><% } %>
     }<%- index < startConfig.properties.length - 1 ? ',' : '' %>
 <%_ }) _%>
   },

package/src/emit/function/functionYamlArtifact.js

@@ -305,17 +305,22 @@ function _functionSetting(functionName) {
   return `  #<#${functionName}FunctionSetting#>\n  #<#/${functionName}FunctionSetting#>`;
 }
 
-function _sqsFilterPatterns(functionName) {
-  return `        filterPatterns:
+function _sqsFilterPatterns(
+  callingFlowName,
+  hookFunctionName = callingFlowName
+) {
+  return `        #<#${hookFunctionName}FilterPatterns#>
+        filterPatterns:
           - body:
               MessageAttributes:
                 callingFlow:
                   Value:
-                    - \${self:custom.iz_resourcePrefix}${functionName}
+                    - \${self:custom.iz_resourcePrefix}${callingFlowName}
           - body:
               MessageAttributes:
                 callingFlow:
                   Value:
                     - exists: false
+        #<#/${hookFunctionName}FilterPatterns#>
 `;
 }

package/src/emit/relationship/emitDynamicRelationshipFlowSchemas.js

@@ -41,6 +41,15 @@ function createDynamicRelationshipFlow(action) {
   const flowSteps = [
     {
       stepName: 'start',
+      ...(outputTopic
+        ? {
+            nextSteps: [
+              {
+                stepName: 'Complete'
+              }
+            ]
+          }
+        : {}),
       properties: [],
       messageAttributes: MESSAGE_ATTRIBUTES,
       ...(outputTopic
@@ -58,7 +67,18 @@ function createDynamicRelationshipFlow(action) {
     flowSteps.push({
       stepName: 'Complete',
       event: ['extTopic'],
-      properties: [],
+      previousSteps: [
+        {
+          stepName: 'start'
+        }
+      ],
+      properties: [
+        'returnValue',
+        'parentFlowId',
+        'status',
+        'errorsFound',
+        'graphServiceTag'
+      ],
       messageAttributes: MESSAGE_ATTRIBUTES
     });
   }

package/src/emit/relationship/emitRelationshipApplicationArtifacts.js

@@ -158,7 +158,7 @@ async function emitAction(action, relationships, flows, serviceTag) {
     },
     {
       path: `${source}/${complete}_HdrSqs.js`,
-      content: await renderShared('FlowHandler_HdrSqs.ejs', {
+      content: await render(`${action}Complete_HdrSqs.ejs`, {
         ...shared,
         flowStepName: 'Complete',
         mainFileName: `${complete}_Main`,

package/src/emit/relationship/templates/ChangeRelationshipComplete_HdrSqs.ejs

@@ -0,0 +1,99 @@
+import { middlewareHandler as middleware } from '@izara_project/izara-middleware';
+import { recordHandlerSharedLib } from '@izara_project/izara-core-library-record-handler';
+import Logger from '@izara_project/izara-core-library-logger';
+import callingFlowSharedLib from '@izara_project/izara-core-library-calling-flow';
+import ChangeRelationshipComplete from './ChangeRelationshipComplete_Main.js';
+
+//(<optionalRequire>)
+//(</optionalRequire>)
+
+// validate event properties in body.Message of sqs event
+let perRecordsValidatorSchema = {
+  type: 'object',
+  properties: {
+    returnValue: {
+      type: 'object'
+    },
+    parentFlowId: {
+      type: 'string'
+    },
+    status: {
+      type: 'string'
+    },
+    errorsFound: {
+      type: 'array'
+    },
+    graphServiceTag: {
+      type: 'string'
+    }
+  },
+  required: [
+    'returnValue',
+    'parentFlowId',
+    'status',
+    'errorsFound',
+    'graphServiceTag'
+  ]
+  //(<validatorSchema>)
+  //(</validatorSchema>)
+};
+
+let messageAttributeValidatorSchema = null;
+middleware.setValidatorSchema(recordHandlerSharedLib.baseValidatorSchema());
+
+export const main = middleware.wrap(async (event, context) => {
+  try {
+    let recordPromises = [];
+
+    // loop each record and send to mainFunction
+    await Promise.all(
+      event.Records.map(async (record) => {
+        let passOnProperties = [];
+        record._izContext.logger.debug('record ReceiveMsgOutHdrSqs', record);
+
+        //validate message (and MessageAttributes)
+        await recordHandlerSharedLib.validateRecord(
+          record, // one record will send to mainFunction
+          'ChangeRelationshipCompleteHdrSqs', // queue name that need to retry or send to dlq
+          perRecordsValidatorSchema, // schema for record.Message
+          messageAttributeValidatorSchema
+        );
+
+        // add argument (to invoke lambda) to passOnProperties[]
+        passOnProperties.push(record.body.Message);
+        passOnProperties.push(
+          callingFlowSharedLib.addPassBackPropertiesToPassOnProperties(
+            record.body.Message
+          )
+        );
+        //(<additionalParams>)
+        //(</additionalParams>)
+        record._izContext.logger.debug(
+          'passOnProperties in handler',
+          passOnProperties
+        );
+
+        // call recordHandlerSharedLib.recordHandler with 3 parameters and return promise(resolve)
+        let recordPromise = recordHandlerSharedLib.recordHandler(
+          record, // one record will send to mainFunction
+          ChangeRelationshipComplete, // mainFunction that need to invoke.
+          'ChangeRelationshipCompleteHdrSqs', // queue name that need to retry or send to dlq
+          passOnProperties // all parameters that mainFunction needed.
+        );
+        record._izContext.logger.debug('after recordPromise in handler');
+        recordPromises.push(recordPromise); // push promise to recordPromises
+      })
+    );
+    try {
+      await Promise.all(recordPromises);
+      return event.Records;
+    } catch {
+      Logger.debug(
+        'Promise.all(recordPromises) in handler threw error (at least one record did no resolve)'
+      );
+    }
+  } catch (error) {
+    Logger.error('Unhandled Error, ChangeRelationshipComplete_HdrSqs: ', error);
+    throw error;
+  }
+});

package/src/emit/relationship/templates/ChangeRelationshipComplete_Main.ejs

@@ -19,7 +19,7 @@ import { objectHash as hash } from '@izara_project/izara-shared-core';
 
 import callingFlowSharedLib from '@izara_project/izara-core-library-calling-flow';
 import snsSharedLib from '@izara_project/izara-core-library-sns';
-import dynamodbSharedLib from '@izara_project/izara-core-library-dynamodb';
+
 import asyncFlowSharedLib from '@izara_project/izara-core-library-asynchronous-flow';
 
 import { sns } from '@izara_project/izara-core-library-external-request';
@@ -36,124 +36,123 @@ const { TOPIC_NAME_GENERATE_CODE, PREFIX } = consts;
 
 export default async function changeRelationshipComplete(
   _izContext,
-  returnValue,
-  status,
-  errorsFound,
-  graphServiceTag,
+  requestParams,
   passBackProperties = {}
 ) {
   try {
+    const {
+      returnValue,
+      parentFlowId,
+      status,
+      errorsFound,
+      graphServiceTag
+    } = requestParams;
+
     _izContext.logger.debug('Function:ChangeRelationshipComplete', {
       returnValue: returnValue,
+      parentFlowId: parentFlowId,
       status: status,
       errorsFound: errorsFound,
       graphServiceTag: graphServiceTag,
       passBackProperties: passBackProperties
     });
 
-    // start create awaitingStepId
-    const linkTypeId = createLinkTypeId(
-      _izContext,
-      returnValue.requestParams.firstObject.objType,
-      returnValue.requestParams.secondObject.objType,
-      returnValue.requestParams.newRelType,
-      returnValue.requestParams.oldRelTypeAndDirection.relationshipDirection
-    );
-
-    let awaitingStepId = asyncFlowSharedLib.createAwaitingStepId(
-      returnValue.parentFlowId, // hash id
+    const awaitingStepId = asyncFlowSharedLib.createAwaitingStepId(
+      parentFlowId || returnValue?.parentFlowId, // hash id
       PREFIX.changeRel // prefix, use constant further
     );
 
-    let createRelPendingObjects = await dynamodbSharedLib.query(
-      _izContext,
-      await dynamodbSharedLib.tableName(_izContext, 'AwaitingMultipleSteps'),
+    const pending =
+      await asyncFlowSharedLib.findPendingStepAwaitingMultipleSteps(
+        _izContext,
+        awaitingStepId
+      );
+    _izContext.logger.debug('[ChangeRelationshipComplete] awaiting step', {
+      awaitingStepId,
+      pending
+    });
+
+    if (!pending) {
+      _izContext.logger.warn(
+        `No pending step found for awaitingStepId: ${awaitingStepId}`
+      );
+      return { returnValue, status, errorsFound };
+    }
+
+    const result =
+      await asyncFlowSharedLib.checkAllAwaitingStepsFinishedWithReturnParams(
+        _izContext,
+        pending.pendingStepId,
+        awaitingStepId,
+        errorsFound
+      );
+    _izContext.logger.debug(
+      '[ChangeRelationshipComplete] all awaiting steps result',
       {
-        awaitingStepId: awaitingStepId
+        isComplete: result.isComplete,
+        awaitingStepId,
+        pendingStepId: pending.pendingStepId
       }
     );
-    _izContext.logger.debug(
-      'record of createRelPendingObjects ::',
-      createRelPendingObjects
-    );
 
-    await Promise.all(
-      createRelPendingObjects.Items.map(async ({ pendingStepId }) => {
-        if (
-          await asyncFlowSharedLib.checkAllAwaitingStepsFinished(
-            _izContext,
-            pendingStepId, // task waiting
-            awaitingStepId,
-            errorsFound
-          )
-        ) {
-          _izContext.logger.debug('finished all awaitingStep');
-
-          let changeRelCompleteMsg = {
-            firstObject: returnValue.requestParams.firstObject,
-            secondObject: returnValue.requestParams.secondObject,
-            relType: returnValue.requestParams.relType,
-            oldRelTypeAndDirection:
-              returnValue.requestParams.oldRelTypeAndDirection,
-            newRelType: returnValue.requestParams.newRelType,
-            parentFlowId: returnValue.parentFlowId,
-            status: status,
-            errorsFound: errorsFound
-          };
-
-          changeRelCompleteMsg =
-            callingFlowSharedLib.addParentPassBackPropertiesToSnsResponseMessageObject(
-              passBackProperties,
-              changeRelCompleteMsg
-            );
-          _izContext.logger.debug(
-            'After addCallingFlowToSnsResponseMessageObject',
-            changeRelCompleteMsg
-          );
-
-          let messageAttributes =
-            callingFlowSharedLib.addParentPassBackCallingFlowToSnsResponseMessageAttributes(
-              passBackProperties,
-              {}
-            );
-          _izContext.logger.debug(
-            'After addCallingFlowToSnsResponseMessageAttributes',
-            changeRelCompleteMsg
-          );
-
-          let sendMessageToOutChangeRelationshipComplete = {
-            Message: JSON.stringify(changeRelCompleteMsg),
-            MessageAttributes: sns.createStringMessageAttributes(
-              _izContext,
-              messageAttributes
-            ),
-            TopicArn: snsSharedLib.snsTopicArnByFlowSchema(
-              _izContext,
-              TOPIC_NAME_GENERATE_CODE.outChangeRelationshipComplete
-            )
-          };
-          _izContext.logger.debug(
-            'Send message to OutChangeRelationshipComplete :::>',
-            sendMessageToOutChangeRelationshipComplete
-          );
-          await sns.publishAsync(
-            _izContext,
-            sendMessageToOutChangeRelationshipComplete
-          );
-
-          //(<afterCheckAwaitingStep>)
-          //(</afterCheckAwaitingStep>)
-        } // check pendingStepId finished
-
-        // remove awaitingMultipleStep after finished
-        await asyncFlowSharedLib.removeAwaitingMultipleStep(
-          _izContext,
-          awaitingStepId,
-          pendingStepId,
-          errorsFound
+    if (result.isComplete) {
+      _izContext.logger.debug('finished all awaitingStep');
+
+      let changeRelCompleteMsg = {
+        firstObject: returnValue.requestParams.firstObject,
+        secondObject: returnValue.requestParams.secondObject,
+        relType: returnValue.requestParams.relType,
+        oldRelTypeAndDirection:
+          returnValue.requestParams.oldRelTypeAndDirection,
+        newRelType: returnValue.requestParams.newRelType,
+        parentFlowId: parentFlowId || returnValue?.parentFlowId,
+        status: status,
+        errorsFound: errorsFound
+      };
+
+      changeRelCompleteMsg =
+        callingFlowSharedLib.addParentPassBackPropertiesToSnsResponseMessageObject(
+          passBackProperties,
+          changeRelCompleteMsg
         );
-      }) // end per pendingStep
-    );
+      _izContext.logger.debug(
+        'After addCallingFlowToSnsResponseMessageObject',
+        changeRelCompleteMsg
+      );
+
+      let messageAttributes =
+        callingFlowSharedLib.addParentPassBackCallingFlowToSnsResponseMessageAttributes(
+          passBackProperties,
+          {}
+        );
+      _izContext.logger.debug(
+        'After addCallingFlowToSnsResponseMessageAttributes',
+        changeRelCompleteMsg
+      );
+
+      let sendMessageToOutChangeRelationshipComplete = {
+        Message: JSON.stringify(changeRelCompleteMsg),
+        MessageAttributes: sns.createStringMessageAttributes(
+          _izContext,
+          messageAttributes
+        ),
+        TopicArn: snsSharedLib.snsTopicArnByFlowSchema(
+          _izContext,
+          TOPIC_NAME_GENERATE_CODE.outChangeRelationshipComplete
+        )
+      };
+      _izContext.logger.debug(
+        'Send message to OutChangeRelationshipComplete :::>',
+        sendMessageToOutChangeRelationshipComplete
+      );
+      await sns.publishAsync(
+        _izContext,
+        sendMessageToOutChangeRelationshipComplete
+      );
+
+      //(<afterCheckAwaitingStep>)
+      //(</afterCheckAwaitingStep>)
+    } // check isComplete finished
 
     //(<afterCheckAllAwaitingStep>)
     //(</afterCheckAllAwaitingStep>)

package/src/emit/relationship/templates/ChangeRelationship_Main.ejs

@@ -390,15 +390,6 @@ export default async function <%- functionName %>(
       return changeRelCompleteMsg;
     }
 
-    // before create awaitingStepId per graph storageResource
-    const linkTypeId = createLinkTypeId(
-      _izContext,
-      firstObject.objType,
-      secondObject.objType,
-      newRelType,
-      oldRelTypeAndDirection.relationshipDirection
-    );
-
     let usedGraphServiceTag = []; // collect used graph serviceTag, use to check duplicate graph serviceTag across graph serverTag
     let targetGraphStorageResources = []; // collect used graph storageResources, filtered duplicate graphServiceTag from graphServerTag out
     let awaitingStepIds = [];
@@ -412,12 +403,12 @@ export default async function <%- functionName %>(
         targetGraphStorageResources.push(targetStorageResource);
 
         // awaitingStepId per graphServiceTag
-        awaitingStepIds.push(
-          asyncFlowSharedLib.createAwaitingStepId(
+        awaitingStepIds.push({
+          awaitingStepId: asyncFlowSharedLib.createAwaitingStepId(
             parentFlowId, // hash id
             PREFIX.changeRel // prefix, use constant further
           )
-        );
+        });
       }
     }
 
@@ -426,11 +417,11 @@ export default async function <%- functionName %>(
       await asyncFlowSharedLib.createAwaitingMultipleSteps(
         _izContext,
         asyncFlowSharedLib.createPendingStepId(
-          // pendingStepId
           _izContext.uniqueRequestId, // hash id
           PREFIX.changeRel // prefix, use constant further
         ),
-        awaitingStepIds // awaitingStepIds
+        awaitingStepIds,
+        {}
       );
 
       // prepare message body

package/src/emit/relationship/templates/CreateRelationshipComplete_HdrSqs.ejs

@@ -0,0 +1,99 @@
+import { middlewareHandler as middleware } from '@izara_project/izara-middleware';
+import { recordHandlerSharedLib } from '@izara_project/izara-core-library-record-handler';
+import Logger from '@izara_project/izara-core-library-logger';
+import callingFlowSharedLib from '@izara_project/izara-core-library-calling-flow';
+import CreateRelationshipComplete from './CreateRelationshipComplete_Main.js';
+
+//(<optionalRequire>)
+//(</optionalRequire>)
+
+// validate event properties in body.Message of sqs event
+let perRecordsValidatorSchema = {
+  type: 'object',
+  properties: {
+    returnValue: {
+      type: 'object'
+    },
+    parentFlowId: {
+      type: 'string'
+    },
+    status: {
+      type: 'string'
+    },
+    errorsFound: {
+      type: 'array'
+    },
+    graphServiceTag: {
+      type: 'string'
+    }
+  },
+  required: [
+    'returnValue',
+    'parentFlowId',
+    'status',
+    'errorsFound',
+    'graphServiceTag'
+  ]
+  //(<validatorSchema>)
+  //(</validatorSchema>)
+};
+
+let messageAttributeValidatorSchema = null;
+middleware.setValidatorSchema(recordHandlerSharedLib.baseValidatorSchema());
+
+export const main = middleware.wrap(async (event, context) => {
+  try {
+    let recordPromises = [];
+
+    // loop each record and send to mainFunction
+    await Promise.all(
+      event.Records.map(async (record) => {
+        let passOnProperties = [];
+        record._izContext.logger.debug('record ReceiveMsgOutHdrSqs', record);
+
+        //validate message (and MessageAttributes)
+        await recordHandlerSharedLib.validateRecord(
+          record, // one record will send to mainFunction
+          'CreateRelationshipCompleteHdrSqs', // queue name that need to retry or send to dlq
+          perRecordsValidatorSchema, // schema for record.Message
+          messageAttributeValidatorSchema
+        );
+
+        // add argument (to invoke lambda) to passOnProperties[]
+        passOnProperties.push(record.body.Message);
+        passOnProperties.push(
+          callingFlowSharedLib.addPassBackPropertiesToPassOnProperties(
+            record.body.Message
+          )
+        );
+        //(<additionalParams>)
+        //(</additionalParams>)
+        record._izContext.logger.debug(
+          'passOnProperties in handler',
+          passOnProperties
+        );
+
+        // call recordHandlerSharedLib.recordHandler with 3 parameters and return promise(resolve)
+        let recordPromise = recordHandlerSharedLib.recordHandler(
+          record, // one record will send to mainFunction
+          CreateRelationshipComplete, // mainFunction that need to invoke.
+          'CreateRelationshipCompleteHdrSqs', // queue name that need to retry or send to dlq
+          passOnProperties // all parameters that mainFunction needed.
+        );
+        record._izContext.logger.debug('after recordPromise in handler');
+        recordPromises.push(recordPromise); // push promise to recordPromises
+      })
+    );
+    try {
+      await Promise.all(recordPromises);
+      return event.Records;
+    } catch {
+      Logger.debug(
+        'Promise.all(recordPromises) in handler threw error (at least one record did no resolve)'
+      );
+    }
+  } catch (error) {
+    Logger.error('Unhandled Error, CreateRelationshipComplete_HdrSqs: ', error);
+    throw error;
+  }
+});