@izara_project/izara-market-library-service-schemas 1.0.4 → 1.0.6

package/src/TemplateManager/src/TranslateIdReq/ProcessingTranslateIds/mainFunction/template.ejs

@@ -0,0 +1,330 @@
+/*
+Copyright (C) 2020 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict';
+
+const izaraShared = require("@izara_project/izara-shared");
+const configSharedLib = izaraShared.configSharedLib;
+const dynamodbSharedLib = izaraShared.dynamodbSharedLib;
+const lambdaSharedLib = izaraShared.lambdaSharedLib;
+const graphSharedLib = izaraShared.graphSharedLib;
+
+
+
+const translateIdsSharedLib = require('@izara_project/izara-translateids/src/TranslateIdsLib');
+const NoRetryError = require('@izara_project/izara-core-library-core/src/NoRetryError');
+const lambda = require('@izara_project/izara-core-library-external-request/src/resources/Lambda');
+const TranslateIdsLibs = require('@izara_project/izara-market-library-service-schemas/src/GenerateCodeLibs/src/TranslateIdsLibs')
+const serviceSchemas = require("@izara_project/izara-core-library-service-schemas");
+const identifiersObject = require('@izara_project/izara-core-library-service-schemas/src/IdentifiersObject');
+const getObjectSchema = require("@izara_project/izara-core-library-service-schemas/src/GetObjectSchema")
+const { firstLetterUpperCase: upperCase } = require('@izara_project/izara-market-library-service-schemas/src/MainLibs/src/Utils')
+
+/**
+ * translate ids from one object to another related object
+ * @param {Object} _izContext
+ * @param {object} fromObjType - the type of object we are translating ids from
+ * @param {string} concatinateIdentifier
+ * @param {string} fromTypeId - the type of object we are translating ids from
+ * @param {string} fromTable - table name
+ * @param {string} fromPartitionKeyId - partition key value
+ * @param {string} fromPartitionKeyFieldName - partition key field/attribute name
+ * @param {string} fromDataFieldName - data field/attribute name
+ * @param {object} toObjType - the type of object we are translating ids to
+ * @param {string} toTypeId - the type of object we are translating ids to
+ * @param {Object} [additionalData={}] - additional data to send back in completed message
+ * @param {Object} [startKey={}] - first invoke will be empty
+ * @param {Nummic} invocationCount
+ * @param {string[]} passOnStatus
+ * @param {string[]} passOnErrorsFound
+ * @param {string} callingFlow
+ */
+
+module.exports.processTranslateIds = async (
+  _izContext,
+  fromObjType,
+  fromTypeId,
+  concatinateIdentifier,
+  fromTable,
+  fromPartitionKeyId,
+  fromPartitionKeyFieldName,
+  fromDataFieldName,
+  toObjType,
+  toTypeId,
+  relType,
+  translateIdsCacheId,
+  uniqueRequestId,
+  startKey = {},
+  invocationCount,
+  passOnStatus,
+  passOnErrorsFound,
+  callingFlow = "",
+) => {
+  // translateIds request to change from propertyIds to mediaLinkIds.
+  // MediaLinkProperty: filter dataId is propertyIds
+  // MediaLinkPropertyTranslation: filter dataId is propertyIds
+  // translate 2 time , propertyIds from MediaLinkProperty and   propertyIds from MediaLinkPropertyTranslation
+  try {
+    _izContext.logger.debug("=== Function:TranslateIdsRequest ===", {
+      fromObjType: fromObjType,
+      fromTypeId: fromTypeId,
+      concatinateIdentifier: concatinateIdentifier,
+      fromTable: fromTable,
+      fromPartitionKeyId: fromPartitionKeyId,
+      fromPartitionKeyFieldName: fromPartitionKeyFieldName,
+      fromDataFieldName: fromDataFieldName,
+      toObjType: toObjType,
+      toTypeId: toTypeId,
+      relType: relType,
+      translateIdsCacheId: translateIdsCacheId,
+      uniqueRequestId: uniqueRequestId,
+      startKey,
+      invocationCount,
+      passOnStatus,
+      passOnErrorsFound,
+      callingFlow: callingFlow
+    });
+
+    let errorsFound = [];
+    // Query resultDatas from table
+    /*
+     receive fromType , toType , relType check relationshipSchema return boolean
+     just validate to check for error or use for do something further
+     if false throw error if true continue translateIds+
+    */
+    // let validateObjTypeStatus = await validateFromTypeToToTypeWithRelType(_izContext, fromObjType, toObjType, relType)
+    let validateObjTypes = await TranslateIdsLibs.validateObjtypesAndRelType(_izContext, [fromObjType, toObjType], relType)
+    if (validateObjTypes == null) {
+      errorsFound = errorsFound.concat(`can't find ${fromObjType} and ${toObjType} in relationshipTag:${JSON.stringify(relType)}`)
+    }
+
+    /*
+      check fromIdentifiers if have use it
+      if not query data from table by using TranslateIdsGetResultIds
+    */
+    let resultDatas;
+    let lastEvaluatedKeyResultDatas;
+
+    if (concatinateIdentifier == null || concatinateIdentifier == undefined) {
+
+      let getFromObjectSchema = await getObjectSchema.getObjSchemaS3WithHierarchy(_izContext, fromObjType);
+      _izContext.logger.debug("getFromObjectSchema", getFromObjectSchema);
+      // create Full tableName
+      let tableName = await dynamodbSharedLib.tableName(_izContext, fromTable, upperCase(getFromObjectSchema.complexFilterServiceTag));
+      _izContext.logger.debug("tableName", tableName);
+      resultDatas = await translateIdsSharedLib.translateIdsGetResultIds(
+        _izContext,
+        tableName,
+        fromPartitionKeyId,
+        fromPartitionKeyFieldName,
+        startKey
+      )
+
+      // lastEvaluatedKeyResultDatas.LastEvaluatedKey
+      /*
+        if not have resultDatas it mean you cant query identifier from table and you didn't send identifier it will return error
+      */
+      _izContext.logger.debug("resultDatas query fromtable =>", resultDatas);
+      if (!resultDatas && concatinateIdentifier === null) {
+        errorsFound.push(`Can not get resultDatas form Table${fromTable} and not have concatinateIdentifier to TranslateIds`)
+      }
+    }
+
+    /*
+      use from concatinateIdentifier to send in processOneRecord
+      concatinateIdentifier type string
+      check if in cache and if no save to TranslateIdsCacheFromIdDataTable, then move on to next resultData{fromDataId}
+    */
+
+    async function processOneRecord(
+      _izContext,
+      translateIdsCacheId,
+      fromObjType,
+      toObjType,
+      relType,
+      concatinateIdentifier
+    ) {
+      _izContext.logger.debug("==== Function processOneRecord =====:", {
+        translateIdsCacheId: translateIdsCacheId,
+        fromObjType: fromObjType,
+        toObjType: toObjType,
+        relType: relType,
+        concatinateIdentifier: concatinateIdentifier
+      });
+      //if true return becase has cache per FromdataId ,eles false found toDataId;
+      /*
+      check cache from each dataId
+      */
+      let [cacheExist, errorsFoundPerconcatinateIdentifier, translateIdsCacheFromId] = await translateIdsSharedLib.translateIdsCheckCacheFromId(
+        _izContext,
+        fromObjType,
+        concatinateIdentifier,
+        toObjType,
+        relType,
+        translateIdsCacheId,
+        uniqueRequestId
+      );
+
+
+      _izContext.logger.debug('After check translateIdsCheckCacheFromId', {
+        cacheExist: cacheExist,
+        errorsFoundPerconcatinateIdentifier: errorsFoundPerconcatinateIdentifier,
+        translateIdsCacheFromId: translateIdsCacheFromId
+      })
+
+      if (cacheExist) {
+        if (errorsFoundPerconcatinateIdentifier.length > 0) {
+          errorsFound = errorsFound.concat(errorsFoundPerconcatinateIdentifier)
+        }
+        return;
+      };
+      // toDataIds from string to Object using function
+      let toDataIds = []
+      // transfrom Identifier string to Identifier object to use getNodeAndRelationship
+      let identifierObject = await identifiersObject.identifiersObjectFromCompositeIdentifier(
+        _izContext,
+        fromObjType,
+        concatinateIdentifier
+      )
+      _izContext.logger.debug("identifierObject", identifierObject)
+
+      let dataIdFromGetNodeAndRelationship = await graphSharedLib.getNodeAndRelationshipsV2(
+        _izContext,
+        "GraphHandler",
+        fromObjType,
+        identifierObject,
+        relType
+      )
+      _izContext.logger.debug("dataIdFromGetNodeAndRelationship: ", dataIdFromGetNodeAndRelationship);
+
+      await Promise.all(
+        dataIdFromGetNodeAndRelationship[0].relNodes.map(async (getNodeAndRelResult) => {
+          let dataIdFromIdentifiersObject = await identifiersObject.compositeIdentifierFromIdentifiersObject(
+            _izContext,
+            toObjType,
+            getNodeAndRelResult.properties
+          )
+          _izContext.logger.debug("dataIdFromIdentifiersObject", dataIdFromIdentifiersObject)
+          toDataIds.push(dataIdFromIdentifiersObject);
+        })
+      )
+
+      _izContext.logger.debug("---------toDataIds---------", toDataIds)
+      //-------------------complete TranslateIdsCacheFromIdMain Per fromdataId------------
+      let status = 'complete';
+
+      if (passOnStatus) {
+        status = passOnStatus;
+      }
+
+      if (errorsFoundPerconcatinateIdentifier.length > 0) {
+        status = 'error';
+        passOnErrorsFound = passOnErrorsFound.concat(errorsFoundPerconcatinateIdentifier);
+
+      } else {
+        _izContext.logger.debug('saveTranslateIdsCacheBothDataTables')
+        // invoke translateIdsChache lib that saves all toDataIds into both cache Data tables
+        await translateIdsSharedLib.saveTranslateIdsCacheBothDataTables(
+          _izContext,
+          toDataIds,
+          translateIdsCacheFromId,
+          translateIdsCacheId
+        );
+      }
+
+      //complete TranslateIdsCacheFromIdDataMain
+      await translateIdsSharedLib.completeTranslateIdsCacheFromIdMain(
+        _izContext,
+        translateIdsCacheFromId,
+        status,
+        errorsFoundPerconcatinateIdentifier
+      );
+    } // end processOneRecord
+
+    let resultPromises = [];
+    /*
+    check concatinateIdentifier
+    if not use data from resultDatas that query from dynamo
+    */
+    if (concatinateIdentifier == null || concatinateIdentifier == undefined) {
+
+
+      for (let resultData of resultDatas.Items) {
+
+        _izContext.logger.debug("+++++iterate resultDatas++++++", resultData);
+        resultPromises.push(
+          processOneRecord(
+            _izContext,
+            translateIdsCacheId, //fromPartitionKeyId, toType
+            fromObjType,
+            toObjType,
+            relType,
+            resultData[fromDataFieldName], //value of DataId
+          )
+        );
+
+      }
+
+
+    } else {
+
+      resultPromises.push(
+        processOneRecord(
+          _izContext,
+          translateIdsCacheId, //fromPartitionKeyId, toType
+          fromObjType,
+          toObjType,
+          relType,
+          concatinateIdentifier, //value of DataId
+        )
+      );
+
+    }
+    await Promise.all(resultPromises);
+
+    _izContext.logger.debug('============start translateIdsContinueProcessing============')
+    //complete TranslateIdsCacheFromIdMain {cache per filterMain}
+    let translateStatus = 'complete'
+    if (errorsFound.length > 0) {
+      translateStatus = 'error'
+    };
+
+
+    await translateIdsSharedLib.translateIdsContinueProcessing( //translateIdsCheckCache the rest next  frsit resultDatas fininsh
+      _izContext,
+      fromObjType,
+      concatinateIdentifier,
+      fromTable,
+      fromPartitionKeyId,
+      fromPartitionKeyFieldName,
+      fromDataFieldName,
+      toObjType,
+      relType,
+      translateIdsCacheId,
+      lastEvaluatedKeyResultDatas,
+      invocationCount,
+      translateStatus,
+      'ProcessTranslateIdsHdrDsq', // send from calling -- for pagination // e.g,TranslateIdsRequestHdrDsq
+      errorsFound,
+      callingFlow
+    );
+
+  } catch (err) {
+    _izContext.logger.error('error TranslateIdsRequest: ', err)
+    throw (err)
+  }
+}

package/src/TemplateManager/src/TranslateIdReq/ProcessingTranslateIds/sns-out/data.js

@@ -0,0 +1,47 @@
+/*
+Copyright (C) 2020 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict';
+const path = require("path");
+
+
+const { HANDLER, ACTIONS } = require("@izara_project/izara-core-library-service-schemas/src/Consts")
+const { SOURCE_PATH, SAVE_FILE_NAME, FUNCTION_NAME } = require('../../../libs/Consts')
+const { firstLetterUpperCase: uppercase, fristLetterLowerCase: lowerCase } = require('../../../../../MainLibs/src/Utils')
+const templatePath = path.join(__dirname, '../../../ResourceYaml/sns-out/template.ejs');
+
+function data(_izContext, srcPath) {
+  return [createResourceData(_izContext, srcPath)]
+}
+
+function createResourceData(_izContext, srcPath) {
+  let queueName = uppercase(FUNCTION_NAME.outTranslateIds)
+  return {
+    templatePath: templatePath,
+    templateData: {
+      queueName
+    },
+    setting: {
+      savePath: path.join(srcPath, SOURCE_PATH.resourceYaml),
+      saveFileName: SAVE_FILE_NAME.snsOutYaml,
+      fileExtension: ".yml",
+      isAppend: true
+    }
+  }
+}
+
+module.exports = data

package/src/TemplateManager/src/TranslateIdReq/ProcessingTranslateIds/sqs-sns/data.js

@@ -0,0 +1,58 @@
+/*
+Copyright (C) 2020 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict';
+
+const path = require("path");
+const { HANDLER } = require('@izara_project/izara-core-library-service-schemas/src/Consts');
+const templatePath = path.join(__dirname, './template.ejs');
+const { SOURCE_PATH, FUNCTION_NAME, SAVE_FILE_NAME } = require('../../../libs/Consts');
+const { firstLetterLowerCase: lowerCase, firstLetterUpperCase: upperCase } = require("../../../../../MainLibs/src/Utils")
+
+/**
+ * create data for dynamoDbYaml tempalte from objectSchema
+ * return array of dynamoDb data
+ * because one objectSchema can create multiple dynamoDb table
+ *
+ * @param {Object} _izContext
+ * @param {String} saveFilePath
+ * @returns {Object[]}              - data of multiple dynamoDb template
+ */
+
+function data(_izContext, srcPath) {
+  let createTranslateIdsCompleteQueueArray = [];
+  const createTranslateIdsCompleteQueues = [
+    {
+      queueName: upperCase(FUNCTION_NAME.translateIdsProcess) + upperCase(HANDLER.hdrDsq)
+    }
+  ]
+  for (let createTranslateIdsCompleteQueue of createTranslateIdsCompleteQueues) {
+    createTranslateIdsCompleteQueueArray.push({
+      templatePath: templatePath,
+      templateData: createTranslateIdsCompleteQueue,
+      setting: {
+        initialData: "Resources:\n",
+        savePath: path.join(srcPath, SOURCE_PATH.resourceYaml),
+        saveFileName: SAVE_FILE_NAME.snsInSqsYaml,
+        fileExtension: ".yml",
+        isAppend: true
+      }
+    })
+  }
+  return createTranslateIdsCompleteQueueArray
+}
+module.exports = data

package/src/TemplateManager/src/TranslateIdReq/ProcessingTranslateIds/sqs-sns/template.ejs

@@ -0,0 +1,43 @@
+#------- queue ---------
+  <%- queueName %>:
+    Type: "AWS::SQS::Queue"
+    Properties:
+      QueueName: ${self:custom.iz_resourcePrefix}<%- queueName %>
+      RedrivePolicy:
+        deadLetterTargetArn: #!GetAtt
+          Fn::GetAtt:
+            - <%- queueName %>DLQ
+            - Arn
+        maxReceiveCount: 3
+      VisibilityTimeout: 120
+
+  <%- queueName %>DLQ:
+    Type: AWS::SQS::Queue
+    Properties:
+      QueueName: ${self:custom.iz_resourcePrefix}<%- queueName %>DLQ
+
+  <%- queueName %>Policy:
+    Type: AWS::SQS::QueuePolicy
+    Properties:
+      PolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Sid: "allow-sns-messages"
+            Effect: Allow
+            Principal: "*"
+            Resource: #!GetAtt
+              Fn::GetAtt:
+                - <%- queueName %>
+                - Arn
+            Action: "SQS:SendMessage"
+      Queues:
+        - Ref: <%- queueName %>
+  #<#<%- firstLetterUpperCase(queueName) %>QueueSetting#>
+  #<#/<%- firstLetterUpperCase(queueName) %>QueueSetting#>
+
+<%_ function firstLetterUpperCase(text){
+  return text.charAt(0).toUpperCase() + text.slice(1)
+} _%>
+<%_ function firstLetterLowerCase(str) {
+  return str.charAt(0).toLowerCase() + str.slice(1)
+} _%>

package/src/TemplateManager/src/TranslateIdReq/TranslateIds/functionYaml/HdrDsq/data.js

@@ -0,0 +1,116 @@
+/*
+Copyright (C) 2020 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict';
+const path = require('path');
+const fs = require('fs');
+
+const { ACTIONS, HANDLER, STORAGE_TYPES } = require('@izara_project/izara-core-library-service-schemas/src/Consts')
+
+const { firstLetterLowerCase: lowerCase, firstLetterUpperCase: upperCase } = require("../../../../../../MainLibs/src/Utils")
+const { DEFAULT_HANDLER_PER_ACTION, createIamRole, resourceNames, RESOURCE_CLASSES, DEFAULT_LAMBDA_ROLE_PER_ACTION, SQS_RESOURCE, SOURCE_PATH, SAVE_FILE_NAME, FUNCTION_NAME, externalResourceName } = require("../../../../libs/Consts");
+
+const templatePath = path.join(__dirname, '../../../../PerActionEndpoint/FunctionYaml/HdrDsq/template.ejs');
+
+function defaultIamRolePerAction() {
+  let defaultIamRole = [];
+
+  defaultIamRole.push(
+    createIamRole(
+      {
+        [RESOURCE_CLASSES.s3]: ["GetObject", "GetObjectVersion"],
+      },
+      [
+        resourceNames(RESOURCE_CLASSES.s3, "object-schema/perServiceSchemas/*"),
+        resourceNames(RESOURCE_CLASSES.s3, "object-schema/serviceConfig/ServiceNameConfig.json"),
+        resourceNames(RESOURCE_CLASSES.s3, "object-schema/serviceConfig/GraphServerTags.json")
+      ]
+    ),
+    createIamRole(
+      {
+        [RESOURCE_CLASSES.s3]: ["ListBucket"]
+      },
+      [
+        resourceNames(RESOURCE_CLASSES.s3, "object-schema")
+      ]
+    )
+  )
+  return defaultIamRole
+}
+
+/**
+ * create data for function yaml for translateIds HdrDsq
+ */
+function data(_izContext, srcPath) {
+  const createSoureParams = createTranslateIdFunctionYamlHdrDsq(srcPath)
+  return [createSoureParams]
+};
+
+function createTranslateIdFunctionYamlHdrDsq(srcPath) {
+  const functionName = FUNCTION_NAME.translateId
+  const handlerType = HANDLER.hdrDsq;
+  let additionalResourcePermission = defaultIamRolePerAction();
+  additionalResourcePermission.push(
+    createIamRole(
+      {
+        [RESOURCE_CLASSES.sqs]: [
+          SQS_RESOURCE.deleteMessage,
+          SQS_RESOURCE.getQueueAttributes,
+          SQS_RESOURCE.receiveMessage,
+          SQS_RESOURCE.sendMessage,
+          SQS_RESOURCE.getQueueUrl
+        ],
+        [RESOURCE_CLASSES.sns]:
+          [
+            "Publish", "Subscribe"
+          ]
+      },
+      [
+        resourceNames(RESOURCE_CLASSES.sqs, upperCase(functionName) + upperCase(handlerType)),
+        resourceNames(RESOURCE_CLASSES.sqs, upperCase(functionName) + upperCase(handlerType) + "DLQ"),
+        resourceNames(RESOURCE_CLASSES.sns, "In" + upperCase(functionName)),
+        resourceNames(RESOURCE_CLASSES.sns, "Out" + upperCase(functionName)),
+      ]
+    ),
+    createIamRole(
+      {
+        [RESOURCE_CLASSES.dynamoDbTable]: ["GetItem", "UpdateItem"]
+      },
+      [
+        resourceNames(RESOURCE_CLASSES.dynamoDbTable, "TranslateIdsCacheMain")
+      ]
+    )
+  )
+
+  return {
+    templatePath: templatePath,
+    templateData: {
+      functionName,
+      handlerType,
+      additionalResourcePermission,
+      resourceLocation: SOURCE_PATH.resourceLocationTranslateId
+    },
+    setting: {
+      savePath: path.join(srcPath, SOURCE_PATH.appYaml),
+      saveFileName: upperCase(SAVE_FILE_NAME.translateId),
+      fileExtension: ".yml",
+      isAppend: true
+    }
+  }
+}
+
+module.exports = data;