@izara_project/izara-market-library-service-schemas 1.0.5 → 1.0.7

package/src/TemplateManager/src/CreateRelationship/createRelationship/sns-in-sqs/data.js

@@ -31,8 +31,6 @@ const sqsTemplatePath = path.join(__dirname, "../../../ResourceYaml/sns-in-sqs/s
 
 /**
  * create data for dynamoDbYaml tempalte from objectSchema
- * return array of dynamoDb data
- * because one objectSchema can create multiple dynamoDb table
  *
  * @param {Object} _izContext
  * @param {String} saveFilePath

package/src/TemplateManager/src/CreateRelationship/createRelationshipComplete/functionYaml/data.js

@@ -95,8 +95,8 @@ function createParamForCreateSource(srcPath) {
         ]
       },
       [
-        resourceNames(RESOURCE_CLASSES.sqs, upperCase(functionName) + upperCase(HANDLER.hdrSqs)),
-        resourceNames(RESOURCE_CLASSES.sqs, upperCase(functionName) + upperCase(HANDLER.hdrSqs) + "DLQ"),
+        resourceNames(RESOURCE_CLASSES.sqs, upperCase(functionName)),
+        resourceNames(RESOURCE_CLASSES.sqs, upperCase(functionName) + "DLQ"),
         resourceNames(RESOURCE_CLASSES.sns, "In" + upperCase(functionName)),
         resourceNames(RESOURCE_CLASSES.sns, "Out" + upperCase(functionName))
       ],

package/src/TemplateManager/src/CreateRelationship/createRelationshipComplete/handler/HdrSqs/data.js

@@ -47,7 +47,6 @@ function createParamForCreateSource(srcPath) {
     templateData: {
       functionName: upperCase(functionName),
       queueName: upperCase(functionName) + upperCase(handler),
-      isCreateMainFunction: false
     },
     setting: {
       savePath: path.join(srcPath, SOURCE_PATH.relationshipPerAction),

package/src/TemplateManager/src/CreateRelationship/createRelationshipComplete/sns-out/data.js

@@ -47,7 +47,7 @@ const templatePath = path.join(__dirname, '../../../ResourceYaml/sns-out/templat
 
 function createDefaultSnsOut(_izContext, srcPath) {
   const createSourceParams = createParamsForCreateSource(_izContext, srcPath)
-  console.log("createSourceParamsCreateRelationshipComplete", createSourceParams)
+  // console.log("createSourceParamsCreateRelationshipComplete", createSourceParams)
 
   return [createSourceParams]
 }

package/src/TemplateManager/src/FindData/FindDataYaml/data.js

@@ -0,0 +1,188 @@
+/*
+Copyright (C) 2020 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict';
+
+const path = require('path');
+
+const templatePath = path.join(__dirname, "./template.ejs")
+
+const { HANDLER, STORAGE_TYPES, ACTIONS } = require('@izara_project/izara-core-library-service-schemas/src/Consts')
+const { firstLetterLowerCase: lowerCase, firstLetterUpperCase: upperCase } = require("../../../../MainLibs/src/Utils")
+const { createIamRole, resourceNames, RESOURCE_CLASSES, DEFAULT_LAMBDA_ROLE_PER_ACTION, SQS_RESOURCE, SOURCE_PATH, SAVE_FILE_NAME, FUNCTION_NAME } = require("../../libs/Consts");
+
+const serviceSchemas = require('@izara_project/izara-core-library-service-schemas');
+const serviceConfig = serviceSchemas.serviceConfig;
+
+function defaultIamRolePerAction() {
+
+  let defaultIamRole = [];
+
+
+  defaultIamRole.push(
+    createIamRole(
+      {
+        [RESOURCE_CLASSES.s3]: ["GetObject", "GetObjectVersion"],
+      },
+      [
+        resourceNames(RESOURCE_CLASSES.s3, "object-schema/perServiceSchemas/*"),
+        resourceNames(RESOURCE_CLASSES.s3, "object-schema/serviceConfig/ServiceNameConfig.json"),
+        resourceNames(RESOURCE_CLASSES.s3, "object-schema/serviceConfig/GraphServerTags.json"),
+      ]
+    )
+  );
+
+  defaultIamRole.push(
+    createIamRole(
+      { [RESOURCE_CLASSES.s3]: ["ListBucket"] },
+      [
+        resourceNames(RESOURCE_CLASSES.s3, "object-schema"),
+      ]
+    )
+  );
+
+  defaultIamRole.push(
+    createIamRole(
+      {
+        [RESOURCE_CLASSES.sns]: ["Publish"]
+      },
+      [
+        resourceNames(RESOURCE_CLASSES.sns, "InFindData"),
+        resourceNames(RESOURCE_CLASSES.sns, "OutFindDataComplete")
+      ]
+    )
+  )
+
+
+
+  defaultIamRole.push(
+    createIamRole(
+      {
+        [RESOURCE_CLASSES.dynamoDbTable]: [
+          DEFAULT_LAMBDA_ROLE_PER_ACTION[ACTIONS.get].dynamoDbPermission,
+          DEFAULT_LAMBDA_ROLE_PER_ACTION[ACTIONS.update].dynamoDbPermission,
+          DEFAULT_LAMBDA_ROLE_PER_ACTION[ACTIONS.delete].dynamoDbPermission
+        ]
+      },
+      [
+        resourceNames(RESOURCE_CLASSES.dynamoDbTable, "FindDataMain")
+      ]
+    )
+  )
+  // console.log("defaultIamRole Api", defaultIamRole)
+  return defaultIamRole
+}
+
+/**
+ * receive objectSchema
+ * create data for FindData Yaml template
+ *
+ * @param {Object} objectSchemas
+ * @return {{ templatePath, templateData, setting } }
+ */
+async function data(_izContext, objectSchemas, srcPath) {
+
+  let additionalResourcePermission = defaultIamRolePerAction();
+  let tableForGetItem = [];
+  let graphForGetNode = [];
+  let hasGraphStorage = false;
+  let functionName = upperCase(FUNCTION_NAME.findData);
+
+  // =======================================================================================
+  for (let objectSchema of objectSchemas.records) {
+    // console.log("objectSchema", objectSchemas)
+
+    for (let storageResource of Object.values(objectSchema.storageResources)) {
+      if (storageResource.storageType === STORAGE_TYPES.dynamoDB) {
+        // add IamRole for dynamodb depend on tableName in storageResources of objectSchema
+        tableForGetItem.push(resourceNames(RESOURCE_CLASSES.dynamoDbTable, storageResource.tableName));
+
+      } else if (storageResource.storageType === STORAGE_TYPES.graph) {
+        // find graph serviceName from graphServerTag or *GetNodeHdrInv ?
+
+        const graphHandlerServiceTag = await serviceConfig.getGraphServiceTagWithCache(
+          _izContext,
+          storageResource.graphServerTag
+        );
+        // _izContext.logger.debug("graphHandlerServiceTag: ", graphHandlerServiceTag)
+
+        if (hasGraphStorage) {
+          continue;
+        } else {
+          graphForGetNode.push(resourceNames(RESOURCE_CLASSES.lambda, graphHandlerServiceTag + "TestGetNodeV2HdrInv"))
+          hasGraphStorage = true;
+        }
+      }
+    }
+  }
+  // =======================================================================================
+  if (tableForGetItem.length) {
+    additionalResourcePermission.push(
+      createIamRole(
+        { [RESOURCE_CLASSES.dynamoDbTable]: [DEFAULT_LAMBDA_ROLE_PER_ACTION[ACTIONS.get].dynamoDbPermission] },
+        tableForGetItem
+      )
+    );
+  }
+
+  if (graphForGetNode.length) {
+    additionalResourcePermission.push(
+      createIamRole(
+        { [RESOURCE_CLASSES.lambda]: ["InvokeFunction"] },
+        graphForGetNode
+      )
+    );
+  }
+
+  additionalResourcePermission.push(
+    createIamRole({
+      [RESOURCE_CLASSES.sqs]:
+        [
+          SQS_RESOURCE.deleteMessage,
+          SQS_RESOURCE.getQueueAttributes,
+          SQS_RESOURCE.receiveMessage,
+          SQS_RESOURCE.sendMessage,
+          SQS_RESOURCE.deleteMessageBatch,
+          SQS_RESOURCE.getQueueUrl
+        ]
+    }, [
+      resourceNames(RESOURCE_CLASSES.sqs, functionName + upperCase(HANDLER.hdrSqs)),
+      resourceNames(RESOURCE_CLASSES.sqs, functionName + upperCase(HANDLER.hdrSqs) + "DLQ")
+    ]
+    )
+  )
+
+  // console.log("additionalResourcePermission", additionalResourcePermission)
+
+  return {
+    templatePath: templatePath,
+    templateData: {
+      functionName: functionName,
+      resourceLocation: SOURCE_PATH.resourceLocationFindData,
+      handlerType: HANDLER.hdrSqs,
+      additionalResourcePermission
+    },
+    setting: {
+      savePath: path.join(srcPath, SOURCE_PATH.appYaml),
+      saveFileName: upperCase(SAVE_FILE_NAME.findData),
+      fileExtension: '.yml',
+      isAppend: false
+    }
+  }
+}
+
+module.exports = data;

package/src/TemplateManager/src/FindData/FindDataYaml/template.ejs

@@ -0,0 +1,27 @@
+<%_ const join = require('path').join; _%>
+<%- functionName %><%- firstLetterUpperCase(handlerType) %>:
+  handler: <%- join(resourceLocation, `${functionName}_${firstLetterUpperCase(handlerType)}.main`)%>
+  name: ${self:custom.iz_resourcePrefix}<%- functionName %><%- firstLetterUpperCase(handlerType) %>
+  events:
+    - sqs:
+        arn: arn:aws:sqs:${self:custom.iz_region}:${self:custom.iz_accountId}:${self:custom.iz_resourcePrefix}<%- functionName %><%- firstLetterUpperCase(handlerType) %>
+        batchSize: 10
+  iamRoleStatements:
+    <%_ additionalResourcePermission.forEach(resourcePermission => { _%>
+    - Effect: <%- resourcePermission.effect %>
+      Action:
+        <%_ Object.keys(resourcePermission.action).forEach(resourcePerAction => { _%>
+        <%_ resourcePermission.action[resourcePerAction].forEach(permission => { _%>
+        - <%- resourcePerAction %>:<%- permission %>
+        <%_ }) _%>
+        <%_ }) _%>
+      Resource:
+        <%_ resourcePermission.resource.forEach(resource => { _%>
+        - "<%- resource %>"
+        <%_ }) _%>
+    <%_}) _%>
+    #<#<%- functionName %><%- handlerType %>IamRole#>
+    #<#/<%- functionName %><%- handlerType %>IamRole#>
+<%_ function firstLetterUpperCase(text){
+  return text.charAt(0).toUpperCase() + text.slice(1)
+} _%>

package/src/TemplateManager/src/FindData/GetByStorage/getByDynamo.ejs

@@ -0,0 +1,53 @@
+let identifiersBase = await identifierObj.identifiersBaseFromIdentifiers(
+  _izContext,
+  objType,
+  identifiers
+);
+_izContext.logger.debug("create identifiersBase from identifiers: ", identifiersBase) // object
+
+<%- createNameTagBefore(objectType, fieldName) %>
+
+let objInstanceBase = await dynamodbSharedLib.getItem(
+  _izContext,
+  await dynamodbSharedLib.tableName(_izContext, <%- JSON.stringify(tableName) %>),
+  identifiersBase
+);
+_izContext.logger.debug("objInstanceBase: ", objInstanceBase)
+
+if (!objInstanceBase) {
+  errorsFound.push(`can't get data in dynamo by ${identifiersBase}`)
+  standardErrorParams.push(errorsFound)
+  await findDataSharedLib.completeFindDataMain(...standardErrorParams);
+  return;
+}
+
+if (!objInstanceBase.hasOwnProperty(fieldName)) {
+  errorsFound.push(`data in dynamoDB have no ${fieldName}`)
+  standardErrorParams.push(errorsFound)
+  await findDataSharedLib.completeFindDataMain(...standardErrorParams);
+  return;
+}
+
+<%- createNameTagAfter(objectType, fieldName) %>
+
+let dataValue = objInstanceBase[fieldName];
+_izContext.logger.debug("dataValue: ", dataValue)
+
+await findDataSharedLib.completeFindDataMain(
+  _izContext,
+  findDataMain.fieldName,
+  findDataMain.objType,
+  findDataMain.identifiers,
+  findDataMain.requestProperties,
+  dataValue,
+  "complete"
+);
+
+<%_ function createNameTagBefore(objectType, fieldName){
+  return `//(<beforeGetItem${objectType}${fieldName}>)
+          //(</beforeGetItem${objectType}${fieldName}>)`
+} _%>
+<%_ function createNameTagAfter(objectType, fieldName){
+  return `//(<afterGetItem${objectType}${fieldName}>)
+          //(</afterGetItem${objectType}${fieldName}>)`
+} _%>

package/src/TemplateManager/src/FindData/GetByStorage/getByGraph.ejs

@@ -0,0 +1,99 @@
+<%- createNameTagBefore(objectType, fieldName) %>
+
+const graphHandlerServiceTag = await serviceConfig.getGraphServiceTagWithCache(
+  _izContext,
+  <%- JSON.stringify(graphServerTag) %>
+);
+_izContext.logger.debug("graphHandlerServiceTag: ", graphHandlerServiceTag)
+
+let getDataByGraph = await graphSharedLib.getNodeV2(
+  _izContext,
+  graphHandlerServiceTag,
+  objType,
+  {
+    identifiers: identifiers
+  },
+          <%- JSON.stringify(versionedDataLabel) %>
+);
+_izContext.logger.debug("getDataByGraph: ", getDataByGraph)
+
+if (!getDataByGraph) {
+  errorsFound.push(`can't get data  of ${objType.objectType} in graph`)
+  standardErrorParams.push(errorsFound)
+  await findDataSharedLib.completeFindDataMain(...standardErrorParams);
+  return;
+}
+
+<%- createNameTagAfter(objectType, fieldName) %>
+
+<% if (versionedDataLabel.length > 0) { _%>
+  // The fieldName is a key in the versionedData object.
+  let versionedDataLabel = getDataByGraph.versionedDatas<%- JSON.stringify(versionedDataLabel) %>;
+
+  if (versionedDataLabel == null) {
+    errorsFound.push(`can't find versionedData of ${objType.objectType}`)
+    standardErrorParams.push(errorsFound)
+    await findDataSharedLib.completeFindDataMain(...standardErrorParams);
+    return;
+  }
+
+  let versionedProperties = versionedDataLabel.properties;
+
+  if (!versionedProperties.hasOwnProperty(fieldName)) {
+    errorsFound.push(`versionedData have no ${fieldName}`)
+    standardErrorParams.push(errorsFound)
+    await findDataSharedLib.completeFindDataMain(...standardErrorParams);
+    return;
+  }
+
+  let dataValue = versionedProperties[fieldName];
+  _izContext.logger.debug("dataValue: ", dataValue)
+<% } else { _%>
+  let objInstanceFull = getDataByGraph.objInstanceFull;
+  _izContext.logger.debug("objInstanceFull: ", objInstanceFull)
+  <% if (getInIdentifiers) { _%>
+
+    // objInstanceFull.identifiers // Object
+    if (!objInstanceFull.identifiers.hasOwnProperty(fieldName)) {
+      errorsFound.push(`nodeProperties have no this ${fieldName}`)
+      standardErrorParams.push(errorsFound)
+      await findDataSharedLib.completeFindDataMain(...standardErrorParams);
+      return;
+    }
+
+    let dataValue = objInstanceFull.identifiers[fieldName];
+    _izContext.logger.debug("this fieldName is identifier, get dataValue in identifiers: ", dataValue)
+
+  <% } else { _%>
+
+    // objInstanceFull.fields // Object
+    if (!objInstanceFull.fields.hasOwnProperty(fieldName)) {
+      errorsFound.push(`nodeProperties have no this ${fieldName}`)
+      standardErrorParams.push(errorsFound)
+      await findDataSharedLib.completeFindDataMain(...standardErrorParams);
+      return;
+    }
+
+    let dataValue = objInstanceFull.fields[fieldName];
+    _izContext.logger.debug("this fieldName is not identifier, get dataValue in fields: ", dataValue)
+
+  <% }_%>
+<% } _%>
+await findDataSharedLib.completeFindDataMain(
+  _izContext,
+  findDataMain.fieldName,
+  findDataMain.objType,
+  findDataMain.identifiers,
+  findDataMain.requestProperties,
+  dataValue,
+  "complete"
+);
+
+<%_ function createNameTagBefore(objectType, fieldName){
+  return `//(<beforeGetNode${objectType}${fieldName}>)
+          //(</beforeGetNode${objectType}${fieldName}>)`
+} _%>
+<%_ function createNameTagAfter(objectType, fieldName){
+  return `//(<afterGetNode${objectType}${fieldName}>)
+          //(</afterGetNode${objectType}${fieldName}>)`
+} _%>

package/src/TemplateManager/src/FindData/Handler/data.js

@@ -0,0 +1,48 @@
+/*
+Copyright (C) 2020 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict';
+
+const path = require('path');
+const fs = require('fs')
+const templatePath = path.join(__dirname, "./template.ejs")
+
+const { SOURCE_PATH, FUNCTION_NAME, HANDLER } = require("../../libs/Consts");
+const { firstLetterLowerCase: lowerCase, firstLetterUpperCase: upperCase } = require("../../../../MainLibs/src/Utils")
+/**
+ * receive srcPath
+ * create data for FindData Handler template
+ *
+ * @param {Object} srcPath
+ * @return {{ templatePath, templateData, setting } }
+ */
+function data(_izContext, srcPath) {
+
+  return {
+    templatePath: templatePath,
+    templateData: {
+    },
+    setting: {
+      savePath: path.join(srcPath, SOURCE_PATH.findData),
+      saveFileName: upperCase(FUNCTION_NAME.findData) + "_" + upperCase(HANDLER.hdrSqs),
+      fileExtension: '.js',
+      isAppend: false
+    }
+  }
+}
+
+module.exports = data;

package/src/TemplateManager/src/FindData/Handler/template.ejs

@@ -0,0 +1,140 @@
+/*
+Copyright (C) 2020 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict';
+
+// const AWS = require('aws-sdk');
+// const sqs = new AWS.SQS({ apiVersion: '2012-11-05' });
+
+const izara = require("@izara_project/izara-middleware");
+const middleware = izara.middlewareHandler;
+const recordHandlerSharedLib = require("@izara_project/izara-shared").recordHandlerSharedLib;
+const Logger = require('@izara_project/izara-core-library-logger');
+
+const findData = require('./FindData_Main');
+
+// validate event properties in body.Message of sqs event
+middleware.setValidatorSchema(recordHandlerSharedLib.baseValidatorSchema());
+
+// set schema for record.body.Message
+const perRecordsValidatorSchema = {
+  type: "object",
+  required: [
+    'fieldName',
+    'objType',
+    'identifiers',
+  ],
+  properties: {
+    fieldName: {
+      type: 'string'
+    },
+    objType: {
+      type: 'object',
+      required: ["serviceTag", "objectType"],
+      properties: {
+        serviceTag: {
+          type: "string",
+          enum: [process.env.iz_serviceTag]
+        },
+        objectType: izara.validatorSchema.stringNotEmpty()
+      }
+    },
+    identifiers: {
+      type: 'object'
+    },
+    requestProperties: {
+      type: 'object',
+      default: {}
+    },
+    callingFlow: {
+      type: 'string',
+      default: ''
+    }
+
+  }
+};
+
+// set schema for record.body.MessageAttributes
+// const messageAttributeValidatorSchema = {
+//   type: "object",
+//   required: ['msgAtrrParam1', 'msgAtrrParam2'],
+//   properties: {
+//     msgAtrrParam1: {
+//       type: "string"
+//     },
+//     msgAtrrParam2: {
+//       type: "object"
+//     }
+//   }
+// };
+
+module.exports.main = middleware.wrap(async (event, context, callback) => {
+
+  try {
+
+    let recordPromises = [];
+
+    // loop each record and send to mainFunction
+    await Promise.all(event.Records.map(async record => { // promise.all for map() function
+
+      let passOnProperties = [];
+      record._izContext.logger.debug('record ReceiveMsgOutHdrSqs', record);
+
+      // validate message (and MessageAttributes)
+      await recordHandlerSharedLib.validateRecord(
+        record,                             // one record will send to mainFunction
+        "FindDataHdrSqs",                   // queue name that need to retry or send to dlq
+        perRecordsValidatorSchema,          // schema for record.Message
+        // messageAttributeValidatorSchema  // ----- for msgAttr default is null -> do not send this parameter if not want to validate msgAtt
+      );
+
+      // add argument (to invoke lambda) to passOnProperties[]
+      passOnProperties.push(record.body.Message.fieldName)
+      passOnProperties.push(record.body.Message.objType)
+      passOnProperties.push(record.body.Message.identifiers)
+      passOnProperties.push(record.body.Message.requestProperties)
+      passOnProperties.push(record.body.Message.callingFlow)
+      record._izContext.logger.debug('passOnProperties in handler', passOnProperties);
+
+      // call recordHandlerSharedLib.recordHandler with 3 parameters and return promise(resolve)
+      let recordPromise = recordHandlerSharedLib.recordHandler(
+        record,             // one record will send to mainFunction
+        findData.findData,  // mainFunction that need to invoke.
+        "FindDataHdrSqs",   // queue name that need to retry or send to dlq
+        passOnProperties,   // all parameters that mainFunction needed.
+      );
+      record._izContext.logger.debug('after recordPromise in handler');
+      recordPromises.push(recordPromise); // push promise to recordPromises
+    }))
+
+    Logger.debug('before Promise.all(recordPromises) in handler');
+    try {
+      // --- main await all promises
+      await Promise.all(recordPromises); // await all promises
+
+      return event.Records // return all for local testing
+
+    } catch {
+      Logger.debug('Promise.all(recordPromises) in handler threw error (at least one record did no resolve)');
+    }
+    Logger.debug('after Promise.all(recordPromises) in handler');
+
+  } catch (err) {
+    Logger.error('Unhandled Error, FindDataHdrSqs: ', err);
+    throw (err);
+  }
+});