@izara_project/izara-core-generate-service-code 1.0.69 → 1.0.71

package/src/emit/{relationship/templates/FlowHandler_HdrSqs.ejs → shared/templates/ProcessLogical_HdrSqs.ejs}

@@ -16,47 +16,45 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 import { middlewareHandler as middleware } from '@izara_project/izara-middleware';
-import { recordHandlerSharedLib } from '@izara_project/izara-core-library-record-handler';
 import Logger from '@izara_project/izara-core-library-logger';
+import { recordHandlerSharedLib } from '@izara_project/izara-core-library-record-handler';
 import callingFlowSharedLib from '@izara_project/izara-core-library-calling-flow';
-import <%= functionName %> from './<%= mainFileName %>.js';
+
+import processLogical from './ProcessLogical_Main.js';
 
 // validate event properties in body.Message of sqs event
-let perRecordsValidatorSchema = {
-  //(<validatorSchema>)
-  <% if (typeof startConfig !== 'undefined' && startConfig !== null && startConfig.properties && startConfig.properties.length > 0) { -%>
-  type: 'object',
-  properties: {
-    <% startConfig.properties.forEach((prop, index) => { %> <%- prop.propertyName %>: {
-      type: '<%- prop.type || 'string' %>'
-    } /*__EJS_COMMA_6__*/
-    <% }) %>
-  },
-  required: [<%- startConfig.properties.filter(p => p.required !== false).map(p => `'${p.propertyName}'`).join(', ') %>]
-  <% } -%>
-  //(</validatorSchema>)
-};
+middleware.setValidatorSchema(recordHandlerSharedLib.baseValidatorSchema());
 
-<% if (typeof startConfig !== 'undefined' && startConfig !== null && startConfig.messageAttributes && startConfig.messageAttributes.length > 0) { -%>
-let messageAttributeValidatorSchema = {
+// set schema for record.body.Message
+const perRecordsValidatorSchema = {
   type: 'object',
+  required: ['objType', 'element', 'overWriteCache'],
   properties: {
-    <% startConfig.messageAttributes.forEach((prop, index) => { %> <%- prop.propertyName %>: {
+    objType: {
       type: 'object',
+      required: ['serviceTag', 'objectType'],
       properties: {
-        Type: { type: 'string' },
-        Value: { type: '<%- prop.type || 'string' %>' }
-      },
-      required: ['Type', 'Value']
-    } /*__EJS_COMMA_14__*/
-    <% }) %>
-  },
-  required: [<%- startConfig.messageAttributes.filter(p => p.required !== false).map(p => `'${p.propertyName}'`).join(', ') %>]
+        serviceTag: {
+          type: 'string',
+          enum: [process.env.iz_serviceTag]
+        },
+        objectType: {
+          type: 'string'
+        }
+      }
+    },
+    element: {
+      type: 'object'
+    },
+    overWriteCache: {
+      type: 'boolean'
+    },
+    callingFlow: {
+      type: 'string',
+      default: ''
+    }
+  }
 };
-<% } else { -%>
-let messageAttributeValidatorSchema = null;
-<% } -%>
-middleware.setValidatorSchema(recordHandlerSharedLib.baseValidatorSchema());
 
 export const main = middleware.wrap(async (event, context) => {
   try {
@@ -64,37 +62,24 @@ export const main = middleware.wrap(async (event, context) => {
 
     // loop each record and send to mainFunction
     await Promise.all(
-      event.Records.map(async (record) => {
+      event.Records.map(async record => {
         // promise.all for map() function
 
         let passOnProperties = [];
         record._izContext.logger.debug('record ReceiveMsgOutHdrSqs', record);
 
-        //validate message (and MessageAttributes)
+        // validate message (and MessageAttributes)
         await recordHandlerSharedLib.validateRecord(
           record, // one record will send to mainFunction
-          '<%= queueName %>', // queue name that need to retry or send to dlq
-          perRecordsValidatorSchema, // schema for record.Message
-          messageAttributeValidatorSchema
+          'ProcessLogicalHdrSqs', // queue name that need to retry or send to dlq
+          perRecordsValidatorSchema // schema for record.Message
         );
 
         // add argument (to invoke lambda) to passOnProperties[]
-        passOnProperties.push(record.body.Message);
-        <% if (functionName.endsWith('Complete')) { %>
-        passOnProperties.push(
-          callingFlowSharedLib.addPassBackPropertiesToPassOnProperties(
-            record.body.Message
-          )
-        );
-        <% } else { %>
-        passOnProperties.push(
-          callingFlowSharedLib.addCallingFlowToPassOnProperties(
-            record.body.Message
-          )
-        );
-        <% } %>
-        //(<additionalParams>)
-        //(</additionalParams>)
+        passOnProperties.push(record.body.Message.objType);
+        passOnProperties.push(record.body.Message.element);
+        passOnProperties.push(record.body.Message.overWriteCache);
+        passOnProperties.push(record.body.Message.callingFlow);
         record._izContext.logger.debug(
           'passOnProperties in handler',
           passOnProperties
@@ -103,8 +88,8 @@ export const main = middleware.wrap(async (event, context) => {
         // call recordHandlerSharedLib.recordHandler with 3 parameters and return promise(resolve)
         let recordPromise = recordHandlerSharedLib.recordHandler(
           record, // one record will send to mainFunction
-          <%= functionName %>, // mainFunction that need to invoke.
-          '<%= queueName %>', // queue name that need to retry or send to dlq
+          processLogical, // mainFunction that need to invoke.
+          'ProcessLogicalHdrSqs', // queue name that need to retry or send to dlq
           passOnProperties // all parameters that mainFunction needed.
         );
         record._izContext.logger.debug('after recordPromise in handler');
@@ -125,7 +110,7 @@ export const main = middleware.wrap(async (event, context) => {
     }
     Logger.debug('after Promise.all(recordPromises) in handler');
   } catch (err) {
-    Logger.error('Unhandled Error, LambdaFunctionHdrSqs: ', err);
+    Logger.error('Unhandled Error, ProcessLogicalHdrSqs: ', err);
     throw err;
   }
 });

package/src/emit/shared/templates/ProcessLogical_Main.ejs

@@ -0,0 +1,448 @@
+/*
+Copyright (C) 2020 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import dynamodbSharedLib from '@izara_project/izara-core-library-dynamodb';
+import snsSharedLib from '@izara_project/izara-core-library-sns';
+import logicalResultsSharedLib from '@izara_project/izara-core-library-logical-result';
+import searchResultSharedLib from '@izara_project/izara-core-library-search-result';
+import { sns } from '@izara_project/izara-core-library-external-request';
+
+import {
+  getObjectSchema,
+  serviceConfig,
+  consts,
+  utils
+} from '@izara_project/izara-core-library-service-schemas';
+import { NoRetryError } from '@izara_project/izara-core-library-core';
+const { STORAGE_TYPES } = consts;
+const { getUsedFieldNamesOfIdentifiers } = utils;
+
+//(<additionalModule>)
+import { identifiersObject as identifierObj } from '@izara_project/izara-core-library-service-schemas';
+//(</additionalModule>)
+
+/**
+ * performs the logical operation/filter/search in the element, saves into LogicalResultsData table
+ * @param {Object} _izContext
+ * @param {Object} objType - contain objectType and serviceTag
+ * @param {Object} element - logical element
+ * @param {Object} element.filterType - 'logical'
+ * @param {string} element.logicalType - contain 'identifier' | 'partitionKey'
+ * @param {Object} [element.partitionKeyValues] - value of fieldNames that is partitionKey.
+ * @param {Object} [element.sortKeyValues] - value of fieldNames that is sortKey. (optional)
+ * @param {string} [element.comparison] - use for query in queryElements.sortKeyCondition. (optional)
+ * @param {string} [element.conditionalFieldValues] - user after query for filter data before save to logicalData table. (optional)
+ * @param {boolean} overWriteCache
+ *
+ * @returns {string} description of return value
+ */
+
+export default async function processLogical(
+  _izContext,
+  objType,
+  element,
+  overWriteCache,
+  callingFlow
+) {
+  try {
+    _izContext.logger.debug('----- ProcessLogical event params -----', {
+      objType: objType,
+      element: element,
+      overWriteCache: overWriteCache,
+      callingFlow: callingFlow
+    });
+
+    let removeAttributes = ['paginateProcessLogicalUniqueRequestId'];
+
+    //(<removeAttributes>)
+    //(</removeAttributes>)
+
+    // Check/Create logicalResultsMain
+    let [logicalResultsMainStatus, logicalResultId, logicalResultMain] =
+      await logicalResultsSharedLib.checkLogicalMainStatus(
+        _izContext,
+        element, // for create logicalResultId
+        objType,
+        callingFlow,
+        removeAttributes,
+        overWriteCache
+      );
+    _izContext.logger.debug('After check logicalMainStatus: ', {
+      logicalResultsMainStatus: logicalResultsMainStatus,
+      logicalResultId: logicalResultId,
+      logicalResultMain: logicalResultMain
+    });
+
+    if (logicalResultsMainStatus !== 'process') {
+      _izContext.logger.debug(
+        `logicalResultsMainStatus status isn't process :${logicalResultsMainStatus}`
+      );
+      return {
+        returnResponse: 'logicalResultsMainStatus status is not process'
+      };
+    }
+
+    let errorsFound = [];
+
+    let standardErrorParams = [
+      _izContext,
+      logicalResultId,
+      logicalResultMain.logicalElement,
+      logicalResultMain.objType,
+      overWriteCache,
+      callingFlow,
+      'error'
+    ];
+
+    if (
+      element.comparison &&
+      !searchResultSharedLib.consts.POSSIBLE_COMPARISON_SORTID.includes(
+        element.comparison
+      )
+    ) {
+      errorsFound.push('comparison is not possible comparison');
+    }
+
+    // validate partitionKeyValues
+    if (!element.hasOwnProperty('partitionKeyValues')) {
+      _izContext.logger.debug('element not have partitionKeyValues');
+      errorsFound.push('element not have partitionKeyValues');
+    } else if (Object.keys(element.partitionKeyValues).length === 0) {
+      _izContext.logger.debug('partitionKeyValues is empty');
+      errorsFound.push('partitionKeyValues is empty');
+    }
+
+    const objectSchemas = await getObjectSchema.getObjSchemaS3WithHierarchy(
+      _izContext,
+      objType
+    );
+    _izContext.logger.debug('objectSchemas: ', objectSchemas);
+
+    let partitionKeyFieldName = null;
+    let tableName = null;
+    let sortKeyFieldName = null;
+
+    for (const schemasIdentifier of objectSchemas.identifiers) {
+      if (schemasIdentifier.type === 'partitionKey') {
+        if (partitionKeyFieldName === null) {
+          if (schemasIdentifier.hasOwnProperty('fieldName')) {
+            partitionKeyFieldName = schemasIdentifier.fieldName;
+          } else {
+            partitionKeyFieldName = schemasIdentifier.name;
+          }
+        } else {
+          throw new NoRetryError(
+            'Invalid object schemas have more than one partitionKey'
+          );
+        }
+      }
+
+      if (schemasIdentifier.type === 'sortKey') {
+        if (sortKeyFieldName === null) {
+          if (schemasIdentifier.hasOwnProperty('fieldName')) {
+            sortKeyFieldName = schemasIdentifier.fieldName;
+          } else {
+            sortKeyFieldName = schemasIdentifier.name;
+          }
+        } else {
+          throw new NoRetryError(
+            'Invalid object schemas have more than one sortKey'
+          );
+        }
+        _izContext.logger.debug('have sortKeyFieldName: ', sortKeyFieldName);
+      }
+    }
+
+    const storageResources = objectSchemas.storageResources;
+
+    storageResourcesLoop: for (const storageResourceTag of Object.keys(
+      storageResources
+    )) {
+      _izContext.logger.debug('check storageResourceTag: ', storageResourceTag);
+
+      if (
+        storageResources[storageResourceTag].storageType ===
+        STORAGE_TYPES.dynamoDB
+      ) {
+        if (
+          storageResources[storageResourceTag].hasOwnProperty(
+            'groupByPartitionKeyField'
+          )
+        ) {
+          if (element.hasOwnProperty('sortKeyValues')) {
+            continue;
+          } else {
+            errorsFound.push(
+              'schema error: dynamo storageResource set to groupByPartitionKeyField but no sortKey'
+            );
+            break;
+          }
+        }
+
+        // validate conditionalFieldValues
+        if (
+          element.hasOwnProperty('conditionalFieldValues') &&
+          element.conditionalFieldValues.length > 0
+        ) {
+          conditionalFieldValuesLoop: for (const conditionalFieldValue of element.conditionalFieldValues) {
+            let conditionalFieldName = conditionalFieldValue.fieldName;
+            let storageResourceTagsFieldName =
+              objectSchemas.fieldNames[conditionalFieldName]
+                .storageResourceTags;
+
+            if (storageResourceTagsFieldName.includes(storageResourceTag)) {
+              continue conditionalFieldValuesLoop;
+            } else {
+              continue storageResourcesLoop;
+            }
+          }
+        }
+
+        tableName = storageResources[storageResourceTag].tableName;
+        _izContext.logger.debug('found tableName: ', tableName);
+        break;
+      }
+    }
+
+    if (!tableName) {
+      errorsFound.push('no valid dynamo storageResource found');
+    }
+
+    if (
+      element.hasOwnProperty('sortKeyValues') &&
+      Object.keys(element.sortKeyValues).length > 0 &&
+      !sortKeyFieldName
+    ) {
+      _izContext.logger.debug(
+        'element have sortKeyValues but cannot find sortKey in object schemas'
+      );
+      errorsFound.push(
+        'element have sortKeyValues but cannot find sortKey in object schemas'
+      );
+    }
+
+    if (errorsFound.length > 0) {
+      _izContext.logger.debug('Have errorsFound: ', errorsFound);
+      standardErrorParams.push(errorsFound);
+      await logicalResultsSharedLib.completeLogicalResultsMain(
+        ...standardErrorParams
+      );
+      return {
+        returnResponse: 'Have errorsFound'
+      };
+    }
+
+    // concatenate partitionKey
+    let partitionKeyReqIdentifiers = {};
+    for (const [fieldName, fieldNameValue] of Object.entries(
+      element.partitionKeyValues
+    )) {
+      partitionKeyReqIdentifiers[fieldName] = fieldNameValue.value;
+    }
+
+    // concatenate sortKey
+    if (element.hasOwnProperty('sortKeyValues')) {
+      for (const [fieldName, fieldNameValue] of Object.entries(
+        element.sortKeyValues
+      )) {
+        partitionKeyReqIdentifiers[fieldName] = fieldNameValue.value;
+      }
+    }
+    _izContext.logger.debug(
+      'partitionKeyReqIdentifiers: ',
+      partitionKeyReqIdentifiers
+    );
+
+    let dynamoDbIdentifiers =
+      await identifierObj.dynamoDbIdentifiersByStorageResource(
+        _izContext,
+        objectSchemas,
+        storageResources,
+        partitionKeyReqIdentifiers
+      );
+    _izContext.logger.debug('dynamoDbIdentifiers: ', dynamoDbIdentifiers);
+
+    if (element.hasOwnProperty('sortKeyValues')) {
+      let comparison = 'equals';
+      if (element.comparison && element.comparison !== 'equals') {
+        comparison = element.comparison;
+        let sentMessageToPaginateProcessLogical = {
+          Message: JSON.stringify({
+            logicalResultId: logicalResultId,
+            tableName: tableName,
+            partitionKeyFieldName: partitionKeyFieldName,
+            sortKeyFieldName: sortKeyFieldName,
+            dynamoDbIdentifiers: dynamoDbIdentifiers,
+            comparison: comparison,
+            overWriteCache: overWriteCache,
+            callingFlow: callingFlow
+          }),
+          TopicArn: await snsSharedLib.snsTopicArnByFlowSchema(
+            _izContext,
+            'PaginateProcessLogical_In'
+          )
+        };
+        _izContext.logger.debug(
+          'Request params before send to topic PaginateProcessLogical_In: ',
+          sentMessageToPaginateProcessLogical
+        );
+
+        let resSNS = await sns.publishAsync(
+          _izContext,
+          sentMessageToPaginateProcessLogical
+        );
+        _izContext.logger.debug(
+          'resSNS send to topic ProcessSortResult finish!',
+          resSNS
+        );
+
+        return {
+          returnResponse: 'Send message to PaginateProcessLogical_In'
+        };
+      } else {
+        let queryDatas = await dynamodbSharedLib.query(
+          _izContext,
+          dynamodbSharedLib.tableName(_izContext, tableName),
+          {
+            [partitionKeyFieldName]: dynamoDbIdentifiers[partitionKeyFieldName]
+          },
+          {
+            sortKeyCondition: {
+              comparison: comparison,
+              name: sortKeyFieldName,
+              value: dynamoDbIdentifiers[sortKeyFieldName]
+            }
+          }
+        );
+        _izContext.logger.debug('queryDatas result: ', queryDatas);
+
+        if (queryDatas.Items.length == 0) {
+          _izContext.logger.debug(`can't find data in ${tableName}`);
+          errorsFound.push(`can't find data in ${tableName}`);
+          standardErrorParams.push(errorsFound);
+          await logicalResultsSharedLib.completeLogicalResultsMain(
+            ...standardErrorParams
+          );
+          return {
+            returnResponse: 'Cannot find data in table'
+          };
+        }
+
+        for (const objInstanceBase of queryDatas.Items) {
+          await logicalResultsSharedLib.saveLogicalData(
+            _izContext,
+            logicalResultId,
+            logicalResultMain.objType,
+            objInstanceBase
+          );
+        }
+      }
+    } else {
+      // element don't have sortKey
+
+      if (sortKeyFieldName) {
+        _izContext.logger.debug(
+          'identifiers have sortKey: query dynamo in PaginateProcessLogical'
+        );
+
+        let sentMessageToPaginateProcessLogical = {
+          Message: JSON.stringify({
+            logicalResultId: logicalResultId,
+            tableName: tableName,
+            partitionKeyFieldName: partitionKeyFieldName,
+            sortKeyFieldName: sortKeyFieldName,
+            dynamoDbIdentifiers: dynamoDbIdentifiers,
+            overWriteCache: overWriteCache,
+            callingFlow: callingFlow
+          }),
+          TopicArn: await snsSharedLib.snsTopicArnByFlowSchema(
+            _izContext,
+            'PaginateProcessLogical_In'
+          )
+        };
+        _izContext.logger.debug(
+          'Request params before send to topic PaginateProcessLogical_In: ',
+          sentMessageToPaginateProcessLogical
+        );
+
+        let resSNS = await sns.publishAsync(
+          _izContext,
+          sentMessageToPaginateProcessLogical
+        );
+        _izContext.logger.debug(
+          'resSNS send to topic ProcessSortResult finish!',
+          resSNS
+        );
+
+        return {
+          returnResponse: 'Send message to PaginateProcessLogical_In'
+        };
+      } else {
+        _izContext.logger.debug('identifiers not have sortKey: save dataId');
+
+        let getDataByPartitionKey = await dynamodbSharedLib.getItem(
+          _izContext,
+          dynamodbSharedLib.tableName(_izContext, tableName),
+          {
+            [partitionKeyFieldName]: dynamoDbIdentifiers[partitionKeyFieldName]
+          }
+        );
+
+        if (getDataByPartitionKey) {
+          await logicalResultsSharedLib.saveLogicalData(
+            _izContext,
+            logicalResultId,
+            objType,
+            getDataByPartitionKey
+          );
+        } else {
+          _izContext.logger.debug(`can't find data in ${tableName}`);
+          errorsFound.push(`can't find data in ${tableName}`);
+          standardErrorParams.push(errorsFound);
+          await logicalResultsSharedLib.completeLogicalResultsMain(
+            ...standardErrorParams
+          );
+          return {
+            returnResponse: 'Cannot find data in table'
+          };
+        }
+      }
+    }
+
+    // complete processLogical and send message Out
+    await logicalResultsSharedLib.completeLogicalResultsMain(
+      _izContext,
+      logicalResultId,
+      element,
+      objType,
+      overWriteCache,
+      callingFlow,
+      'complete',
+      []
+    );
+
+    //(<additionalProcessLogical>)
+    //(</additionalProcessLogical>)
+
+    return {
+      returnResponse: 'Function ProcessLogical is Finished'
+    };
+  } catch (err) {
+    _izContext.logger.error('error ProcessLogicalMain: ', err);
+    throw err;
+  }
+}