@izara_project/izara-core-generate-service-code 1.0.69 → 1.0.71

package/src/emit/shared/templates/PaginateProcessLogical_HdrDsq.ejs

@@ -0,0 +1,162 @@
+/*
+Copyright (C) 2020 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { middlewareHandler as middleware } from '@izara_project/izara-middleware';
+import Logger from '@izara_project/izara-core-library-logger';
+import { recordHandlerSharedLib } from '@izara_project/izara-core-library-record-handler';
+import callingFlowSharedLib from '@izara_project/izara-core-library-calling-flow';
+
+import paginateProcessLogical from './PaginateProcessLogical_Main.js';
+
+// validate event properties in body.Message of sqs event
+middleware.setValidatorSchema(
+  recordHandlerSharedLib.baseValidatorSchema({
+    type: 'object',
+    required: ['body', 'messageAttributes'],
+    properties: {
+      body: {
+        type: 'object'
+      },
+      messageAttributes: {
+        type: 'object'
+      }
+    }
+  })
+);
+
+// set schema for record.body.Message
+const perRecordsValidatorSchema = {
+  type: 'object',
+  required: [
+    'logicalResultId',
+    'tableName',
+    'partitionKeyFieldName',
+    'sortKeyFieldName',
+    'dynamoDbIdentifiers',
+    'startKey',
+    'overwriteUniqueRequestId',
+    'overWriteCache'
+  ],
+  properties: {
+    logicalResultId: {
+      type: 'string'
+    },
+    tableName: {
+      type: 'string'
+    },
+    partitionKeyFieldName: {
+      type: 'string'
+    },
+    sortKeyFieldName: {
+      type: 'string'
+    },
+    dynamoDbIdentifiers: {
+      type: 'object'
+    },
+    comparison: {
+      type: 'string',
+      default: ''
+    },
+    startKey: {
+      type: 'object'
+    },
+    overwriteUniqueRequestId: {
+      type: 'string'
+    },
+    overWriteCache: {
+      type: 'boolean'
+    },
+    callingFlow: {
+      type: 'string',
+      default: ''
+    }
+  }
+};
+
+export const main = middleware.wrap(async (event, context) => {
+  try {
+    let recordPromises = [];
+
+    // loop each record and send to mainFunction
+    await Promise.all(
+      event.Records.map(async record => {
+        // promise.all for map() function
+
+        // --- reforming record.body for Dsq request
+        record = recordHandlerSharedLib.reformatDsqMessage(
+          record._izContext,
+          record
+        );
+        record._izContext.logger.debug(
+          'record PaginateProcessLogicalHdrDsq after reform',
+          record
+        );
+
+        let passOnProperties = [];
+
+        // validate message (and MessageAttributes)
+        await recordHandlerSharedLib.validateRecord(
+          record, // one record will send to mainFunction
+          'PaginateProcessLogicalHdrDsq', // queue name that need to retry or send to dlq
+          perRecordsValidatorSchema // schema for record.Message
+        );
+
+        // add argument (to invoke lambda) to passOnProperties[]
+        passOnProperties.push(record.body.Message.logicalResultId);
+        passOnProperties.push(record.body.Message.tableName);
+        passOnProperties.push(record.body.Message.partitionKeyFieldName);
+        passOnProperties.push(record.body.Message.sortKeyFieldName);
+        passOnProperties.push(record.body.Message.dynamoDbIdentifiers);
+        passOnProperties.push(record.body.Message.comparison);
+        passOnProperties.push(record.body.Message.startKey);
+        passOnProperties.push(record.body.Message.overwriteUniqueRequestId);
+        passOnProperties.push(record.body.Message.overWriteCache);
+        passOnProperties.push(record.body.Message.callingFlow);
+        record._izContext.logger.debug(
+          'passOnProperties in handler',
+          passOnProperties
+        );
+
+        // call recordHandlerSharedLib.recordHandler with 3 parameters and return promise(resolve)
+        let recordPromise = recordHandlerSharedLib.recordHandler(
+          record, // one record will send to mainFunction
+          paginateProcessLogical, // mainFunction that need to invoke.
+          'PaginateProcessLogicalHdrDsq', // queue name that need to retry or send to dlq
+          passOnProperties // all parameters that mainFunction needed.
+        );
+        record._izContext.logger.debug('after recordPromise in handler');
+        recordPromises.push(recordPromise); // push promise to recordPromises
+      })
+    );
+
+    Logger.debug('before Promise.all(recordPromises) in handler');
+    try {
+      // --- main await all promises
+      await Promise.all(recordPromises); // await all promises
+
+      return event.Records; // return all for local testing
+    } catch {
+      Logger.debug(
+        'Promise.all(recordPromises) in handler threw error (at least one record did no resolve)'
+      );
+    }
+    Logger.debug('after Promise.all(recordPromises) in handler');
+  } catch (err) {
+    Logger.error('Unhandled Error, PaginateProcessLogicalHdrDsq: ', err);
+    throw err;
+  }
+});

package/src/emit/shared/templates/PaginateProcessLogical_HdrSqs.ejs

@@ -0,0 +1,140 @@
+/*
+Copyright (C) 2020 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { middlewareHandler as middleware } from '@izara_project/izara-middleware';
+import Logger from '@izara_project/izara-core-library-logger';
+import { recordHandlerSharedLib } from '@izara_project/izara-core-library-record-handler';
+import callingFlowSharedLib from '@izara_project/izara-core-library-calling-flow';
+
+import paginateProcessLogical from './PaginateProcessLogical_Main.js';
+
+// validate event properties in body.Message of sqs event
+middleware.setValidatorSchema(recordHandlerSharedLib.baseValidatorSchema());
+
+// set schema for record.body.Message
+const perRecordsValidatorSchema = {
+  type: 'object',
+  required: [
+    'logicalResultId',
+    'tableName',
+    'partitionKeyFieldName',
+    'sortKeyFieldName',
+    'dynamoDbIdentifiers',
+    'overWriteCache'
+  ],
+  properties: {
+    logicalResultId: {
+      type: 'string'
+    },
+    tableName: {
+      type: 'string'
+    },
+    partitionKeyFieldName: {
+      type: 'string'
+    },
+    sortKeyFieldName: {
+      type: 'string'
+    },
+    dynamoDbIdentifiers: {
+      type: 'object'
+    },
+    comparison: {
+      type: 'string',
+      default: ''
+    },
+    startKey: {
+      type: 'object',
+      default: {}
+    },
+    overwriteUniqueRequestId: {
+      type: 'string',
+      default: ''
+    },
+    overWriteCache: {
+      type: 'boolean'
+    },
+    callingFlow: {
+      type: 'string',
+      default: ''
+    }
+  }
+};
+
+export const main = middleware.wrap(async (event, context) => {
+  try {
+    let recordPromises = [];
+
+    // loop each record and send to mainFunction
+    await Promise.all(
+      event.Records.map(async record => {
+        // promise.all for map() function
+
+        let passOnProperties = [];
+        record._izContext.logger.debug('record ReceiveMsgOutHdrSqs', record);
+
+        // validate message (and MessageAttributes)
+        await recordHandlerSharedLib.validateRecord(
+          record, // one record will send to mainFunction
+          'PaginateProcessLogicalHdrSqs', // queue name that need to retry or send to dlq
+          perRecordsValidatorSchema // schema for record.Message
+        );
+
+        // add argument (to invoke lambda) to passOnProperties[]
+        passOnProperties.push(record.body.Message.logicalResultId);
+        passOnProperties.push(record.body.Message.tableName);
+        passOnProperties.push(record.body.Message.partitionKeyFieldName);
+        passOnProperties.push(record.body.Message.sortKeyFieldName);
+        passOnProperties.push(record.body.Message.dynamoDbIdentifiers);
+        passOnProperties.push(record.body.Message.comparison);
+        passOnProperties.push(record.body.Message.startKey);
+        passOnProperties.push(record.body.Message.overwriteUniqueRequestId);
+        passOnProperties.push(record.body.Message.overWriteCache);
+        passOnProperties.push(record.body.Message.callingFlow);
+        record._izContext.logger.debug(
+          'passOnProperties in handler',
+          passOnProperties
+        );
+
+        // call recordHandlerSharedLib.recordHandler with 3 parameters and return promise(resolve)
+        let recordPromise = recordHandlerSharedLib.recordHandler(
+          record, // one record will send to mainFunction
+          paginateProcessLogical, // mainFunction that need to invoke.
+          'PaginateProcessLogicalHdrSqs', // queue name that need to retry or send to dlq
+          passOnProperties // all parameters that mainFunction needed.
+        );
+        record._izContext.logger.debug('after recordPromise in handler');
+        recordPromises.push(recordPromise); // push promise to recordPromises
+      })
+    );
+
+    Logger.debug('before Promise.all(recordPromises) in handler');
+    try {
+      // --- main await all promises
+      await Promise.all(recordPromises); // await all promises
+
+      return event.Records; // return all for local testing
+    } catch {
+      Logger.debug(
+        'Promise.all(recordPromises) in handler threw error (at least one record did no resolve)'
+      );
+    }
+    Logger.debug('after Promise.all(recordPromises) in handler');
+  } catch (err) {
+    Logger.error('Unhandled Error, PaginateProcessLogicalHdrSqs: ', err);
+    throw err;
+  }
+});

package/src/emit/shared/templates/PaginateProcessLogical_Main.ejs

@@ -0,0 +1,268 @@
+/*
+Copyright (C) 2020 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import logicalResultsSharedLib from '@izara_project/izara-core-library-logical-result';
+import sqsSharedLib from '@izara_project/izara-core-library-sqs';
+import dynamodbSharedLib from '@izara_project/izara-core-library-dynamodb';
+import asyncFlowSharedLib from '@izara_project/izara-core-library-asynchronous-flow';
+
+import { sqs } from '@izara_project/izara-core-library-external-request';
+
+import { NoRetryError } from '@izara_project/izara-core-library-core';
+
+// set query limit
+const limit = 2;
+
+/**
+ * performs the logical operation/filter/search in the element, saves into LogicalResultsData table
+ * @param {Object} _izContext
+ * @param {string} logicalResultId
+ * @param {string} tableName
+ * @param {string} partitionKeyFieldName
+ * @param {string} sortKeyFieldName
+ * @param {Object} dynamoDbIdentifiers
+ * @param {string} comparison
+ * @param {Object} startKey
+ * @param {string} overwriteUniqueRequestId
+ * @param {boolean} overwriteCache
+ * @param {string} callingFlow
+ *
+ * @returns {string} description of return value
+ */
+
+export default async function paginateProcessLogical(
+  _izContext,
+  logicalResultId,
+  tableName,
+  partitionKeyFieldName,
+  sortKeyFieldName,
+  dynamoDbIdentifiers,
+  comparison, // optional
+  startKey, // optional
+  overwriteUniqueRequestId, // optional
+  overWriteCache,
+  callingFlow // optional
+) {
+  try {
+    _izContext.logger.debug('----- PaginateProcessLogical event params -----', {
+      logicalResultId: logicalResultId,
+      tableName: tableName,
+      partitionKeyFieldName: partitionKeyFieldName,
+      sortKeyFieldName: sortKeyFieldName,
+      dynamoDbIdentifiers: dynamoDbIdentifiers,
+      comparison: comparison,
+      startKey: startKey,
+      overwriteUniqueRequestId: overwriteUniqueRequestId,
+      overWriteCache: overWriteCache,
+      callingFlow: callingFlow
+    });
+
+    // Check/Create logicalResultsMain
+    let [checkUniqueRequestProcessingStatus, logicalResultsMain] =
+      await asyncFlowSharedLib.checkUniqueRequestProcessing(
+        _izContext,
+        'LogicalResultsMain',
+        {
+          logicalResultId: logicalResultId
+        },
+        'paginateProcessLogical',
+        overwriteUniqueRequestId
+      );
+    _izContext.logger.debug('After check uniqueRequestProcessingStatus:', {
+      checkUniqueRequestProcessingStatus: checkUniqueRequestProcessingStatus,
+      logicalResultsMain: logicalResultsMain
+    });
+
+    if (checkUniqueRequestProcessingStatus == 'stop') {
+      return {
+        returnResponse: 'process already stop'
+      };
+    } else if (checkUniqueRequestProcessingStatus == 'recordNotFound') {
+      // should always have unless another request already completed it, in which case nothing to do
+      return {
+        returnResponse: 'record not found'
+      };
+    }
+
+    let errorsFound = [];
+    let standardErrorParams = [
+      _izContext,
+      logicalResultId,
+      logicalResultsMain.logicalElement,
+      logicalResultsMain.objType,
+      overWriteCache,
+      callingFlow,
+      'error'
+    ];
+
+    let queryElements = {
+      limit: limit
+    };
+
+    //if received startKey validate it, if not received will return null
+    startKey = asyncFlowSharedLib.validateStartKeyParam(
+      _izContext,
+      startKey,
+      partitionKeyFieldName,
+      sortKeyFieldName
+    );
+
+    if (startKey === false) {
+      throw new NoRetryError('split processing received invalid startKey');
+    }
+    _izContext.logger.debug('startKey :', startKey);
+
+    if (startKey) {
+      queryElements['exclusiveStartKey'] = {
+        [partitionKeyFieldName]: startKey[partitionKeyFieldName],
+        [sortKeyFieldName]: startKey[sortKeyFieldName]
+      };
+    }
+
+    if (comparison) {
+      queryElements['sortKeyCondition'] = {
+        comparison: comparison,
+        name: sortKeyFieldName,
+        value: dynamoDbIdentifiers[sortKeyFieldName]
+      };
+    }
+
+    let queryDatas = await dynamodbSharedLib.query(
+      _izContext,
+      await dynamodbSharedLib.tableName(_izContext, tableName),
+      {
+        [partitionKeyFieldName]: dynamoDbIdentifiers[partitionKeyFieldName]
+      },
+      queryElements
+    );
+    _izContext.logger.debug('queryDatas result: ', queryDatas);
+
+    if (!startKey && queryDatas.Items.length == 0) {
+      _izContext.logger.debug(`can't find data in table ${tableName}`);
+      errorsFound.push(`can't find data in table ${tableName}`);
+      standardErrorParams.push(errorsFound);
+      await logicalResultsSharedLib.completeLogicalResultsMain(
+        ...standardErrorParams
+      );
+      return {
+        returnResponse: 'Cannot find data in table'
+      };
+    }
+
+    // For conditionalFieldValues
+    let objInstanceBases = queryDatas.Items;
+    const conditionalFieldValues =
+      logicalResultsMain.logicalElement.conditionalFieldValues;
+
+    if (
+      Array.isArray(conditionalFieldValues) &&
+      conditionalFieldValues.length > 0
+    ) {
+      for (const conditionalFieldValue of conditionalFieldValues) {
+        _izContext.logger.debug(
+          'conditionalFieldValue: ',
+          conditionalFieldValue
+        );
+
+        let conditionalFieldName = conditionalFieldValue.fieldName;
+
+        if (conditionalFieldValue.hasOwnProperty('includeValues')) {
+          let includeValues = conditionalFieldValue.includeValues; // array
+
+          objInstanceBases = objInstanceBases.filter(objInstanceBase => {
+            return includeValues.includes(
+              objInstanceBase[conditionalFieldName]
+            );
+          });
+        } else if (conditionalFieldValue.hasOwnProperty('excludeValues')) {
+          let excludeValues = conditionalFieldValue.excludeValues; // array
+
+          objInstanceBases = objInstanceBases.filter(objInstanceBase => {
+            return !excludeValues.includes(
+              objInstanceBase[conditionalFieldName]
+            );
+          });
+        }
+      }
+      _izContext.logger.debug(
+        'objInstanceBases after conditionalFieldValues: ',
+        objInstanceBases
+      );
+    }
+
+    for (const objInstanceBase of objInstanceBases) {
+      await logicalResultsSharedLib.saveLogicalData(
+        _izContext,
+        logicalResultId,
+        logicalResultsMain.objType,
+        objInstanceBase
+      );
+    }
+
+    if (queryDatas.hasOwnProperty('LastEvaluatedKey')) {
+      if (!overwriteUniqueRequestId) {
+        overwriteUniqueRequestId = _izContext.uniqueRequestId;
+      }
+
+      let continuePaginationMsg = {
+        MessageBody: JSON.stringify({
+          logicalResultId: logicalResultId,
+          tableName: tableName,
+          partitionKeyFieldName: partitionKeyFieldName,
+          sortKeyFieldName: sortKeyFieldName,
+          dynamoDbIdentifiers: dynamoDbIdentifiers,
+          comparison: comparison,
+          startKey: queryDatas.LastEvaluatedKey,
+          overwriteUniqueRequestId: overwriteUniqueRequestId,
+          overWriteCache: overWriteCache,
+          callingFlow: callingFlow
+        }),
+        QueueUrl: await sqsSharedLib.sqsQueueUrl(
+          _izContext,
+          'PaginateProcessLogicalHdrDsq'
+        )
+      };
+
+      _izContext.logger.debug(
+        'continuePaginationMessage: ',
+        continuePaginationMsg
+      );
+      await sqs.sendMessage(_izContext, continuePaginationMsg);
+
+      return {
+        returnResponse: 'Continue PaginateProcessLogical'
+      };
+    } else {
+      await logicalResultsSharedLib.completeLogicalResultsMain(
+        _izContext,
+        logicalResultId,
+        logicalResultsMain.logicalElement,
+        logicalResultsMain.objType,
+        overWriteCache,
+        callingFlow,
+        'complete'
+      );
+    }
+
+    return {
+      returnResponse: 'Function PaginateProcessLogical is Finished'
+    };
+  } catch (err) {
+    _izContext.logger.error('error PaginateProcessLogicalMain: ', err);
+    throw err;
+  }
+}