@izara_project/izara-market-library-service-schemas 1.0.4 → 1.0.6

package/src/TemplateManager/src/ProcessLogicalPagination/Handler/HdrSqs/data.js

@@ -0,0 +1,48 @@
+/*
+Copyright (C) 2020 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict';
+
+const path = require('path');
+const { SOURCE_PATH, FUNCTION_NAME, HANDLER } = require("../../../libs/Consts");
+const { firstLetterUpperCase: upperCase } = require('../../../../../MainLibs/src/Utils');
+const templatePath = path.join(__dirname, "./template.ejs");
+
+
+/**
+ * receive srcPath
+ * not create data for ProcessLogical Main template
+ *
+ * @param {Object} srcPath
+ * @return {{ templatePath, templateData, setting } }
+ */
+
+function data(_izContext, srcPath) {
+
+  return {
+    templatePath: templatePath,
+    templateData: {},
+    setting: {
+      savePath: path.join(srcPath, SOURCE_PATH.processLogical),
+      saveFileName: `${upperCase(FUNCTION_NAME.paginateProcessLogical)}_${upperCase(HANDLER.hdrSqs)}`,
+      fileExtension: '.js',
+      isAppend: false
+    }
+  }
+}
+
+module.exports = data;

package/src/TemplateManager/src/ProcessLogicalPagination/Handler/HdrSqs/template.ejs

@@ -0,0 +1,146 @@
+/*
+Copyright (C) 2020 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict';
+
+// const AWS = require('aws-sdk');
+// const sqs = new AWS.SQS({ apiVersion: '2012-11-05' });
+
+const izara = require("@izara_project/izara-middleware");
+const middleware = izara.middlewareHandler;
+const recordHandlerSharedLib = require("@izara_project/izara-shared").recordHandlerSharedLib;
+const Logger = require('@izara_project/izara-core-library-logger');
+
+const paginateProcessLogical = require('./PaginateProcessLogical_Main');
+
+// validate event properties in body.Message of sqs event
+middleware.setValidatorSchema(recordHandlerSharedLib.baseValidatorSchema());
+
+// set schema for record.body.Message
+const perRecordsValidatorSchema = {
+  type: "object",
+  required: [
+    'logicalResultId',
+    'tableName',
+    'partitionKeyFieldName',
+    'sortKeyFieldName'
+  ],
+  properties: {
+    logicalResultId: {
+      type: "string"
+    },
+    tableName: {
+      type: "string"
+    },
+    partitionKeyFieldName: {
+      type: "string"
+    },
+    sortKeyFieldName: {
+      type: "string"
+    },
+    comparison: {
+      type: "string",
+      default: ""
+    },
+    startKey: {
+      type: "object",
+      default: {}
+    },
+    overwriteUniqueRequestId: {
+      type: "string",
+      default: ""
+    },
+    callingFlow: {
+      type: 'string',
+      default: ""
+    }
+  }
+};
+
+// set schema for record.body.MessageAttributes
+// const messageAttributeValidatorSchema = {
+//   type: "object",
+//   required: ['msgAtrrParam1', 'msgAtrrParam2'],
+//   properties: {
+//     msgAtrrParam1: {
+//       type: "string"
+//     },
+//     msgAtrrParam2: {
+//       type: "object"
+//     }
+//   }
+// };
+
+module.exports.main = middleware.wrap(async (event, context, callback) => {
+
+  try {
+
+    let recordPromises = [];
+
+    // loop each record and send to mainFunction
+    await Promise.all(event.Records.map(async record => { // promise.all for map() function
+
+      let passOnProperties = []
+      record._izContext.logger.debug('record ReceiveMsgOutHdrSqs', record);
+
+      // validate message (and MessageAttributes)
+      await recordHandlerSharedLib.validateRecord(
+        record, // one record will send to mainFunction
+        "PaginateProcessLogicalHdrSqs", // queue name that need to retry or send to dlq
+        perRecordsValidatorSchema, // schema for record.Message
+        // messageAttributeValidatorSchema  // ----- for msgAttr default is null -> do not send this parameter if not want to validate msgAtt
+      );
+
+      // add argument (to invoke lambda) to passOnProperties[]
+      passOnProperties.push(record.body.Message.logicalResultId)
+      passOnProperties.push(record.body.Message.tableName)
+      passOnProperties.push(record.body.Message.partitionKeyFieldName)
+      passOnProperties.push(record.body.Message.sortKeyFieldName)
+      passOnProperties.push(record.body.Message.comparison)
+      passOnProperties.push(record.body.Message.startKey)
+      passOnProperties.push(record.body.Message.overwriteUniqueRequestId)
+      passOnProperties.push(record.body.Message.callingFlow)
+      record._izContext.logger.debug('passOnProperties in handler', passOnProperties);
+
+      // call recordHandlerSharedLib.recordHandler with 3 parameters and return promise(resolve)
+      let recordPromise = recordHandlerSharedLib.recordHandler(
+        record, // one record will send to mainFunction
+        paginateProcessLogical.paginateProcessLogical, // mainFunction that need to invoke.
+        "PaginateProcessLogicalHdrSqs", // queue name that need to retry or send to dlq
+        passOnProperties, // all parameters that mainFunction needed.
+      );
+      record._izContext.logger.debug('after recordPromise in handler');
+      recordPromises.push(recordPromise); // push promise to recordPromises
+    }))
+
+    Logger.debug('before Promise.all(recordPromises) in handler');
+    try {
+      // --- main await all promises
+      await Promise.all(recordPromises); // await all promises
+
+      return event.Records // return all for local testing
+
+    } catch {
+      Logger.debug('Promise.all(recordPromises) in handler threw error (at least one record did no resolve)');
+    }
+    Logger.debug('after Promise.all(recordPromises) in handler');
+
+  } catch (err) {
+    Logger.error('Unhandled Error, PaginateProcessLogicalHdrSqs: ', err);
+    throw (err);
+  }
+});

package/src/TemplateManager/src/ProcessLogicalPagination/mainFunction/data.js

@@ -0,0 +1,48 @@
+/*
+Copyright (C) 2020 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict';
+
+const path = require('path');
+const { SOURCE_PATH, FUNCTION_NAME, HANDLER } = require("../../libs/Consts");
+const { firstLetterUpperCase: upperCase } = require('../../../../MainLibs/src/Utils');
+const templatePath = path.join(__dirname, "./template.ejs");
+
+
+/**
+ * receive srcPath
+ * not create data for ProcessLogical Main template
+ *
+ * @param {Object} srcPath
+ * @return {{ templatePath, templateData, setting } }
+ */
+
+function data(_izContext, srcPath) {
+
+  return {
+    templatePath: templatePath,
+    templateData: {},
+    setting: {
+      savePath: path.join(srcPath, SOURCE_PATH.processLogical),
+      saveFileName: `${upperCase(FUNCTION_NAME.paginateProcessLogical)}_Main`,
+      fileExtension: '.js',
+      isAppend: false
+    }
+  }
+}
+
+module.exports = data;

package/src/TemplateManager/src/ProcessLogicalPagination/mainFunction/template.ejs

@@ -0,0 +1,212 @@
+/*
+Copyright (C) 2020 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict';
+
+const izaraShared = require('@izara_project/izara-shared');
+const dynamodbSharedLib = izaraShared.dynamodbSharedLib;
+const logicalResultsSharedLib = izaraShared.logicalResultsSharedLib;
+const asyncFlowSharedLib = izaraShared.asyncFlowSharedLib;
+const sqsSharedLib = izaraShared.sqsSharedLib;
+
+const sqs = require('@izara_project/izara-core-library-external-request').sqs;
+
+const NoRetryError = require('@izara_project/izara-core-library-core').NoRetryError;
+
+// set query limit
+const limit = 2;
+
+/**
+ * performs the logical operation/filter/search in the element, saves into LogicalResultsData table
+ * @param {Object} _izContext
+ * @param {string} logicalResultId
+ * @param {string} tableName
+ * @param {string} partitionKeyFieldName
+ * @param {string} sortKeyFieldName
+ * @param {string} comparison
+ * @param {Object} startKey
+ * @param {string} overwriteUniqueRequestId
+ * @param {string} callingFlow
+ *
+ * @returns {string} description of return value
+ */
+
+module.exports.paginateProcessLogical = async (
+  _izContext,
+  logicalResultId,
+  tableName,
+  partitionKeyFieldName,
+  sortKeyFieldName,
+  comparison, // optional
+  startKey, // optional
+  overwriteUniqueRequestId, // optional
+  callingFlow // optional
+) => {
+
+  try {
+
+    _izContext.logger.debug('----- PaginateProcessLogical event params -----', {
+      logicalResultId: logicalResultId,
+      tableName: tableName,
+      partitionKeyFieldName: partitionKeyFieldName,
+      sortKeyFieldName: sortKeyFieldName,
+      comparison: comparison,
+      startKey: startKey,
+      overwriteUniqueRequestId: overwriteUniqueRequestId,
+      callingFlow: callingFlow
+    });
+
+    // Check/Create logicalResultsMain
+    let [checkUniqueRequestProcessingStatus, logicalResultsMain] = await asyncFlowSharedLib.checkUniqueRequestProcessing(
+      _izContext,
+      "LogicalResultsMain",
+      {
+        logicalResultId: logicalResultId
+      },
+      "paginateProcessLogical",
+      overwriteUniqueRequestId
+    );
+    _izContext.logger.debug("After check uniqueRequestProcessingStatus:", {
+      checkUniqueRequestProcessingStatus: checkUniqueRequestProcessingStatus,
+      logicalResultsMain: logicalResultsMain
+    });
+
+    if (checkUniqueRequestProcessingStatus == "stop") {
+      return;
+    } else if (checkUniqueRequestProcessingStatus == "recordNotFound") {
+      // should always have unless another request already completed it, in which case nothing to do
+      return;
+    }
+
+    let errorsFound = [];
+    let standardErrorParams = [
+      _izContext,
+      logicalResultId,
+      logicalResultsMain.logicalElement,
+      logicalResultsMain.objType,
+      "error"
+    ];
+
+    const partitionKeyValue = logicalResultsMain.logicalElement.partitionKey;
+    let queryElements = {
+      limit: limit
+    };
+
+    //if received startKey validate it, if not received will return null
+    startKey = asyncFlowSharedLib.validateStartKeyParam(
+      _izContext,
+      startKey,
+      partitionKeyFieldName,
+      sortKeyFieldName
+    );
+
+    if (startKey === false) {
+      throw new NoRetryError("split processing received invalid startKey");
+    }
+    _izContext.logger.debug("startKey :", startKey);
+
+    if (startKey) {
+      queryElements["exclusiveStartKey"] = {
+        [partitionKeyFieldName]: startKey[partitionKeyFieldName],
+        [sortKeyFieldName]: startKey[sortKeyFieldName]
+      }
+    };
+
+
+    if (comparison) {
+      const sortKeyValue = logicalResultsMain.logicalElement.sortKey;
+
+      queryElements["sortKeyCondition"] = {
+        comparison: comparison,
+        name: sortKeyFieldName,
+        value: sortKeyValue
+      };
+    }
+
+    let queryDatas = await dynamodbSharedLib.query(
+      _izContext,
+      await dynamodbSharedLib.tableName(_izContext, tableName),
+      {
+        [partitionKeyFieldName]: partitionKeyValue
+      },
+      queryElements
+    )
+    _izContext.logger.debug("queryDatas result: ", queryDatas)
+
+    if (!startKey && queryDatas.Items.length == 0) {
+      _izContext.logger.debug(`can't find data in table ${tableName} by query`)
+      errorsFound.push(`can't find data in table ${tableName} by query`)
+      standardErrorParams.push(errorsFound)
+      await logicalResultsSharedLib.completeLogicalResultsMain(...standardErrorParams);
+      return;
+    }
+
+    for (const objInstanceBase of queryDatas.Items) {
+
+      await logicalResultsSharedLib.saveLogicalData(
+        _izContext,
+        logicalResultId,
+        logicalResultsMain.objType,
+        objInstanceBase
+      )
+    }
+
+    if (queryDatas.hasOwnProperty("LastEvaluatedKey")) {
+
+      if (!overwriteUniqueRequestId) {
+        overwriteUniqueRequestId = _izContext.uniqueRequestId
+      };
+
+      let continuePaginationMsg = {
+        MessageBody: JSON.stringify({
+          logicalResultId: logicalResultId,
+          tableName: tableName,
+          partitionKeyFieldName: partitionKeyFieldName,
+          sortKeyFieldName: sortKeyFieldName,
+          comparison: comparison,
+          overwriteUniqueRequestId: overwriteUniqueRequestId,
+          startKey: queryDatas.LastEvaluatedKey,
+          callingFlow: callingFlow
+        }),
+        QueueUrl: await sqsSharedLib.sqsQueueUrl(_izContext, "PaginateProcessLogicalHdrDsq"),
+      };
+
+      _izContext.logger.debug("continuePaginationMessage: ", continuePaginationMsg);
+      await sqs.sendMessage(_izContext, continuePaginationMsg).promise();
+
+      return "Continue PaginateProcessLogical"
+
+    } else {
+
+      await logicalResultsSharedLib.completeLogicalResultsMain(
+        _izContext,
+        logicalResultId,
+        logicalResultsMain.logicalElement,
+        logicalResultsMain.objType,
+        "complete",
+        [],
+        callingFlow
+      )
+    }
+
+    return "Function PaginateProcessLogical is Finished"
+
+  } catch (err) {
+    _izContext.logger.error('error PaginateProcessLogicalMain: ', err);
+    throw (err)
+  }
+}

package/src/TemplateManager/src/ResourceYaml/CreateSourceData.js

@@ -40,7 +40,7 @@ module.exports.createSourceDataObjectSchema = (_izContext, objectSchema, saveFil
 
 
   const dynamoDbMainYamlDataList = dynamoDbMainYaml.dynamoDbMainYamlFromObjectSchema.templateDataFunction(_izContext, objectSchema);
-  console.log("dynamoDbMainYamlDataList: ", dynamoDbMainYamlDataList);
+  // console.log("dynamoDbMainYamlDataList: ", dynamoDbMainYamlDataList);
 
   for (let dynamoDbMainYamlData of dynamoDbMainYamlDataList) {
     // create iam role of handler here
@@ -57,7 +57,7 @@ module.exports.createSourceDataObjectSchema = (_izContext, objectSchema, saveFil
     });
   }
 
-  console.log("resultsForCreateSource: ", resultsForCreateSource);
+  // console.log("resultsForCreateSource: ", resultsForCreateSource);
 
   return resultsForCreateSource;
 }

package/src/TemplateManager/src/ResourceYaml/dynamodb/awaitingMultipleStepData.js

@@ -30,7 +30,7 @@ const STORAGE_TYPES = {
   graph: "graph"
 }
 const templatePath = path.join(__dirname, './template.ejs')
-const { SOURCE_PATH } = require('../../libs/Consts');
+const { SOURCE_PATH, SAVE_FILE_NAME } = require('../../libs/Consts');
 
 /**
  * create param of crateSouce for FindData And processLogical
@@ -39,7 +39,7 @@ const { SOURCE_PATH } = require('../../libs/Consts');
  * @param {String} srcPath
  * @returns {Object[]}
  */
-module.exports.createAwaitingMultipleStepDynamoDbData = (_izContext, srcPath) => {
+const createAwaitingMultipleStepDynamoDbData = (_izContext, srcPath) => {
 
   let resultsForCreateSource = [];
 
@@ -76,8 +76,6 @@ module.exports.createAwaitingMultipleStepDynamoDbData = (_izContext, srcPath) =>
     }
   ];
 
-  // console.log("defaultlDynamoDataList: ", (awaitingStepDynamoDbDataList));
-
   for (let defaultlDynamoData of awaitingStepDynamoDbDataList) {
     resultsForCreateSource.push({
       templatePath: templatePath,
@@ -85,14 +83,13 @@ module.exports.createAwaitingMultipleStepDynamoDbData = (_izContext, srcPath) =>
       setting: {
         initialData: 'Resources:\n',
         savePath: path.join(srcPath, SOURCE_PATH.resourceYaml),
-        saveFileName: 'generated-dynamoDB-table',
+        saveFileName: SAVE_FILE_NAME.dynamoDbYaml,
         fileExtension: '.yml',
         isAppend: true
       }
     });
   }
-
-  // console.log("resultForCreateDefaultDynamoDb: ", (resultsForCreateSource));
-
   return resultsForCreateSource;
 }
+
+module.exports = createAwaitingMultipleStepDynamoDbData

package/src/TemplateManager/src/ResourceYaml/dynamodb/mainResourcePerObjectSchemaData.js

@@ -31,7 +31,7 @@ const STORAGE_TYPES = {
   graph: "graph"
 }
 
-const { SOURCE_PATH } = require('../../libs/Consts')
+const { SOURCE_PATH, SAVE_FILE_NAME } = require('../../libs/Consts')
 const templatePath = path.join(__dirname, './template.ejs')
 
 // ---------- End Per Lambda Handler IamRole Data ----------
@@ -85,7 +85,6 @@ function dataForMainDynamoDbYamlFromObjectSchema(_izContext, objectSchema, srcPa
     }
   }
 
-  // console.log("identifierDetail :", identifierDetail)
   if (!identifierDetail.hasOwnProperty("partitionKey")) {
     return []
   }
@@ -123,51 +122,30 @@ function dataForMainDynamoDbYamlFromObjectSchema(_izContext, objectSchema, srcPa
   return dynamoDbYamlTempleteData
 }
 
-
-// data for MainDynamo from RelSchema
-// function dataForMainDynamoDbYamlFromRelSchema(_izContext, relSchema) {
-//   let relTag = Object.keys(relSchema)[0];
-//   let relData = relSchema[relTag];
-
-//   for (let parents of relData.parents) {
-//     let parentObjectType = parent.objType.objectType;
-
-
-//   }
-
-//   return
-// }
-
-
-module.exports.createSourceDataDynamoDB = (_izContext, objectSchema, srcPath) => {
+const createSourceDataDynamoDB = (_izContext, objectSchema, srcPath) => {
 
   let resultsForCreateSource = [];
 
 
   const dynamoDbMainYamlDataList = dataForMainDynamoDbYamlFromObjectSchema(_izContext, objectSchema, srcPath)
-  // console.log("dynamoDbMainYamlDataList: ", dynamoDbMainYamlDataList);
 
   for (let dynamoDbMainYamlData of dynamoDbMainYamlDataList) {
-    // create iam role of handler here
     resultsForCreateSource.push({
       templatePath: templatePath,
       templateData: dynamoDbMainYamlData,
       setting: {
         initialData: 'Resources:\n',
         savePath: path.join(srcPath, SOURCE_PATH.resourceYaml),
-        saveFileName: 'generated-dynamoDB-table',
+        saveFileName: SAVE_FILE_NAME.dynamoDbYaml,
         fileExtension: '.yml',
         isAppend: true
       }
     });
   }
 
-  // console.log("resultsForCreateSource: ", resultsForCreateSource);
 
   return resultsForCreateSource;
 }
 
-// module.exports.createSourceDataRelSchema = (_izContext, relSchema, saveFilePath) => {
 
-//   return []
-// }
+module.exports = createSourceDataDynamoDB

package/src/TemplateManager/src/ResourceYaml/dynamodb/processLogicalAndFindData.js

@@ -30,7 +30,7 @@ const STORAGE_TYPES = {
   graph: "graph"
 }
 const templatePath = path.join(__dirname, './template.ejs')
-const { SOURCE_PATH } = require('../../libs/Consts');
+const { SOURCE_PATH, SAVE_FILE_NAME } = require('../../libs/Consts');
 
 /**
  * create param of crateSouce for FindData And processLogical
@@ -39,13 +39,13 @@ const { SOURCE_PATH } = require('../../libs/Consts');
  * @param {String} srcPath
  * @returns {Object[]}
  */
-module.exports.createDefaultSourceDataDynamoDB = (_izContext, srcPath) => {
+const createDefaultSourceDataDynamoDB = (_izContext, srcPath) => {
 
   let resultsForCreateSource = [];
 
   const defaultDynamoDataList = [
     {
-      "tableName": "FindData",
+      "tableName": "FindDataMain",
       "attributes": [
         {
           "keyType": "partitionKey",
@@ -74,15 +74,13 @@ module.exports.createDefaultSourceDataDynamoDB = (_izContext, srcPath) => {
         },
         {
           "keyType": "sortKey",
-          "AttributeName": "dataId",
+          "AttributeName": "identifiersId",
           "AttributeType": "S"
         }
       ]
     }
   ];
 
-  // console.log("defaultlDynamoDataList: ", (defaultDynamoDataList));
-
   for (let defaultlDynamoData of defaultDynamoDataList) {
     resultsForCreateSource.push({
       templatePath: templatePath,
@@ -90,14 +88,12 @@ module.exports.createDefaultSourceDataDynamoDB = (_izContext, srcPath) => {
       setting: {
         initialData: 'Resources:\n',
         savePath: path.join(srcPath, SOURCE_PATH.resourceYaml),
-        saveFileName: 'generated-dynamoDB-table',
+        saveFileName: SAVE_FILE_NAME.dynamoDbYaml,
         fileExtension: '.yml',
         isAppend: true
       }
     });
   }
-
-  // console.log("resultForCreateDefaultDynamoDb: ", (resultsForCreateSource));
-
   return resultsForCreateSource;
 }
+module.exports = createDefaultSourceDataDynamoDB