@izara_project/izara-market-library-service-schemas 1.0.84 → 1.0.86

package/src/Libs/Libs.js

@@ -0,0 +1,711 @@
+/*
+Copyright(C) 2021 Sven Mason < http://izara.io>
+
+This program is free software: you can redistribute it and / or modify
+it under the terms of the GNU Affero General Public License as
+  published by the Free Software Foundation, either version 3 of the
+License, or(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.If not, see < http://www.gnu.org/licenses/>.
+*/
+
+import lodash from 'lodash'
+
+// core libs
+import asyncFlowSharedLib from '@izara_project/izara-core-library-asynchronous-flow';
+import callingFlowSharedLib from '@izara_project/izara-core-library-calling-flow';
+import dynamodbSharedLib from '@izara_project/izara-core-library-dynamodb';
+import { sns } from '@izara_project/izara-core-library-external-request';
+import graphSharedLibs from '@izara_project/izara-core-library-graph-service';
+import lambdaSharedLib from '@izara_project/izara-core-library-lambda';
+import {
+  getObjectSchema,
+  dataDetailsLib
+} from '@izara_project/izara-core-library-service-schemas';
+const { createDataDetailsLib } = dataDetailsLib;
+import snsSharedLib from '@izara_project/izara-core-library-sns';
+import { objectHash as hash } from '@izara_project/izara-shared-core';
+import { validateObjType } from '@izara_project/izara-shared-service-schemas';
+import { consts, utils } from '@izara_project/izara-core-generate-service-code';
+const { TOPIC_NAME_GENERATE_CODE, TOPIC_NAME_GRAPH_HANDLER, PREFIX, STORAGE_TYPES, ADD_ON_DATA_STRUCTURE_TYPE } = consts;
+
+
+/**
+ *
+ * @param {Object} from
+ * @param {String} from.objectType
+ * @param {String} from.serviceTag
+ * @param {String} from.linkType
+ * @param {String} from.requiredOnCreate
+ * @param {Object} to
+ * @param {String} to.objectType
+ * @param {String} to.serviceTag
+ * @param {String} to.linkType
+ * @param {String} to.requiredOnCreate
+ * @param {Object} storageResources
+ * @param {Boolean} topLevelParent
+ */
+function createBasicRelationshipsProperties(
+  from,
+  to,
+  storageResources,
+  baseDirection,
+  topLevelParent = false
+) {
+  validateObjType.validateObjType({
+    objectType: from.objectType,
+    serviceTag: from.serviceTag
+  });
+  validateObjType.validateObjType({
+    objectType: to.objectType,
+    serviceTag: to.serviceTag
+  });
+
+  let otherDirection;
+  if (baseDirection === 'to') {
+    otherDirection = 'from';
+  } else if (baseDirection === 'from') {
+    otherDirection = 'to';
+  }
+
+  let relationshipProperties = {
+    fieldNames: {
+      originTimeStamp: {
+        type: 'number',
+        requiredOnCreate: true,
+        canUpdate: false,
+        validation: {
+          minimum: 1111111111111,
+          maximum: 9999999999999
+        }
+      }
+    },
+    storageResources: storageResources,
+    links: [
+      {
+        storageResourceTags: Object.keys(storageResources),
+        from: {
+          objType: {
+            objectType: from.objectType,
+            serviceTag: from.serviceTag
+          },
+          linkType: from.linkType,
+          direction: baseDirection,
+          requiredOnCreate: from.requiredOnCreate
+        },
+        to: {
+          objType: {
+            objectType: to.objectType,
+            serviceTag: to.serviceTag
+          },
+          linkType: to.linkType,
+          direction: otherDirection,
+          requiredOnCreate: to.requiredOnCreate
+        }
+      }
+    ]
+  };
+
+  if (topLevelParent === true) {
+    relationshipProperties.links.push({
+      storageResourceTags: Object.keys(storageResources),
+      from: {
+        objType: {
+          serviceTag: to.serviceTag,
+          objectType: to.objectType
+        },
+        linkType: 'one',
+        direction: 'from',
+        requiredOnCreate: false
+      },
+      to: {
+        objType: {
+          serviceTag: to.serviceTag,
+          objectType: to.objectType
+        },
+        linkType: 'many',
+        direction: 'to',
+        requiredOnCreate: false
+      }
+    });
+  }
+  return relationshipProperties;
+}
+
+function createFieldNamesFromRequestParamsByObjectSchema(
+  _izContext,
+  requestParams,
+  objectSchema
+) {
+  let createFieldNameParams = {};
+  _izContext.logger.debug(
+    'createFieldNamesFromRequestParamsByObjectSchema ::',
+    { requestParams, objectSchema }
+  );
+  for (const fieldName of Object.keys(objectSchema.fieldNames)) {
+    if (Object.keys(requestParams).includes(fieldName)) {
+      if (!createFieldNameParams.hasOwnProperty(fieldName)) {
+        Object.assign(createFieldNameParams, {
+          [fieldName]: requestParams[fieldName]
+        });
+      }
+    }
+  }
+  return createFieldNameParams;
+}
+
+async function sharedCreateEndpoint(_izContext, objType, requestParams, callingFlowConfig,
+  settings = {
+    userAgent: false,
+    directInvoke: false,
+    parentObject: {}
+  }) {
+  let objectSchemas = await getObjectSchema.getObjectSchemaCombineFieldNamesWithCache(_izContext, objType)
+  let createDataDetails = await createDataDetailsLib(_izContext, objectSchemas)
+  _izContext.logger.debug("createDataDetails is =", createDataDetails);
+  let listOfObjectForCreates = [];
+  let allAwaitingStepsId = [];
+
+  let objInstanceFull = {
+    identifiers: {},
+    fields: {}
+  };
+
+  for (let fieldName of Object.keys(requestParams.fieldNames)) {
+    if (objectSchemas.identifierFieldNames.includes(fieldName)) {
+      _izContext.logger.debug('identifiers is', fieldName);
+      Object.assign(objInstanceFull.identifiers, {
+        [fieldName]: requestParams.fieldNames[fieldName]
+      });
+    }
+  }
+
+  let objInstanceFullForDynamoDb = lodash.cloneDeep(objInstanceFull);
+  let objInstanceFullForGraph = lodash.cloneDeep(objInstanceFull);
+  let objInstanceFullForExternalTopic = lodash.cloneDeep(objInstanceFull);
+
+  for (const [storageTag, createDataDetail] of Object.entries(
+    createDataDetails
+  )) {
+    if (createDataDetail.storageType === STORAGE_TYPES.dynamoDB) {
+      _izContext.logger.debug('::::::DynamoDB::::::', {
+        storageTag,
+        objInstanceFull
+      });
+
+      if (settings.parentObject) {
+        if (
+          createDataDetail.tableName.includes(
+            utils.firstLetterUpperCase(`${settings.parentObject.objectType}Records`)
+          )
+        ) {
+          Object.assign(objInstanceFullForDynamoDb.fields, {
+            [`${settings.parentObject.objectType}HandlerServiceTag`]:
+              process.env.iz_serviceTag
+          });
+        }
+      }
+
+      Object.assign(
+        objInstanceFullForDynamoDb.fields,
+        createObjInstanceFullFieldsByStorageTag(
+          _izContext,
+          storageTag,
+          createDataDetail
+        )
+      );
+      _izContext.logger.debug(
+        'objInstanceFull before create item in dynamoDB',
+        objInstanceFullForDynamoDb
+      );
+
+      let identifiersForCreateData = await dynamoDbIdentifiersByStorageResource(
+        _izContext,
+        objectSchemas,
+        createDataDetail,
+        objInstanceFullForDynamoDb.identifiers
+      );
+      _izContext.logger.debug(
+        'identifiersForCreateData',
+        identifiersForCreateData
+      );
+
+      await dynamodbSharedLib.putItem(
+        _izContext,
+        await dynamodbSharedLib.tableName(
+          _izContext,
+          createDataDetail.tableName,
+          createDataDetail.serviceTag
+        ),
+        {
+          ...identifiersForCreateData,
+          ...objInstanceFullForDynamoDb.fields
+        }
+      );
+
+      objInstanceFullForDynamoDb.fields = {}; // prevent using fieldNames from other tableName
+      if (allAwaitingStepsId.length === 0) {
+        let messageObject = {
+          objType: objType,
+          objInstanceFull: {
+            identifiers: objInstanceFullForDynamoDb.identifiers,
+            fields: objInstanceFullForDynamoDb.fields
+          }
+        };
+
+        if (
+          callingFlowConfig[
+          callingFlowSharedLib.consts.CALLINGFLOW_PROPERTYNAME
+          ]
+        ) {
+          // send message to SNS OutCreateNodeComplete
+          _izContext.logger.debug('HAVE CallingFlow');
+          messageObject =
+            callingFlowSharedLib.addPassBackPropertiesToSnsResponseMessageObject(
+              callingFlowConfig,
+              messageObject
+            );
+
+          let messageAttributes =
+            callingFlowSharedLib.addCallingFlowToSnsResponseMessageAttributes(
+              callingFlowConfig,
+              {}
+            );
+          let outCreateNodeCompleteTopic =
+            await snsSharedLib.snsTopicArnByFlowSchema(
+              _izContext,
+              TOPIC_NAME_GENERATE_CODE.outCreateNodeComplete
+            );
+          let messageParams = {
+            Message: JSON.stringify(messageObject),
+            MessageAttributes: sns.createStringMessageAttributes(
+              _izContext,
+              messageAttributes
+            ),
+            TopicArn: outCreateNodeCompleteTopic
+          };
+
+          _izContext.logger.debug(
+            'messageParams OutCreateNode ::::::: ',
+            messageParams
+          );
+          await sns.publishAsync(_izContext, messageParams);
+        }
+        return {
+          objType: objType,
+          objInstanceFull: objInstanceFullForDynamoDb,
+          relationships: requestParams.relationships || [],
+          status: 'complete',
+          errorsFound: []
+        };
+      }
+    } else if (createDataDetail.storageType === STORAGE_TYPES.graph) {
+      _izContext.logger.debug('::::::Graph::::::', {
+        storageTag,
+        objInstanceFull
+      });
+
+      Object.assign(
+        objInstanceFullForGraph.fields,
+        createObjInstanceFullFieldsByStorageTag(
+          _izContext,
+          storageTag,
+          createDataDetail
+        )
+      );
+      _izContext.logger.debug(
+        'objInstanceFull before send to Graph',
+        objInstanceFullForGraph
+      );
+
+      // allStorageTagComplete = false;
+      allAwaitingStepsId.push(
+        await asyncFlowSharedLib.createAwaitingStepId(
+          hash({
+            objType: objType,
+            graphServerTag: storageTag,
+            identifiers: objInstanceFullForGraph.identifiers,
+            callingFlowProperties: callingFlowConfig.callingFlowProperties || {}
+          }),
+          PREFIX.CREATE_OBJECT_ASYNC
+        )
+      );
+      listOfObjectForCreates.push({
+        [storageTag]: {
+          objInstanceFull: {
+            identifiers: objInstanceFullForGraph.identifiers,
+            fields: objInstanceFullForGraph.fields
+          },
+          allStorageTagComplete: false
+        }
+      });
+
+      _izContext.logger.debug('FIRST', listOfObjectForCreates);
+    } else if (
+      createDataDetail.storageType === STORAGE_TYPES.externalTopic
+    ) {
+      _izContext.logger.debug('::::::externalTopic::::::', {
+        storageTag,
+        createDataDetail
+      });
+      Object.assign(
+        objInstanceFullForExternalTopic.fields,
+        createObjInstanceFullFieldsByStorageTag(
+          _izContext,
+          storageTag,
+          createDataDetail
+        )
+      );
+      _izContext.logger.debug(
+        'objInstanceFull before send to external topic',
+        objInstanceFullForExternalTopic
+      );
+
+      allAwaitingStepsId.push(
+        await asyncFlowSharedLib.createAwaitingStepId(
+          hash({
+            objType: objType,
+            graphServerTag: storageTag,
+            identifiers: objInstanceFullForGraph.identifiers,
+            callingFlowProperties: callingFlowConfig.callingFlowProperties || {}
+          }),
+          PREFIX.CREATE_OBJECT_EXTERNAL_TOPIC
+        )
+      );
+
+      let sendToCreateExternalTopic = {
+        Message: JSON.stringify({
+          objType,
+          objInstanceFull: objInstanceFullForExternalTopic,
+          relationships: requestParams.relationships
+        }),
+        TopicArn: `arn:aws:sns:${createDataDetail.region}:${createDataDetail.accountId}:${createDataDetail.serviceTag}_${createDataDetail.stage}_Create_In`
+      };
+      _izContext.logger.debug(
+        'Request params before send to create endpoint:',
+        sendToCreateExternalTopic
+      );
+
+      await sns.publishAsync(_izContext, sendToCreateExternalTopic);
+      return sendToCreateExternalTopic;
+    }
+  }
+  _izContext.logger.debug(
+    '[1]listOfObjectForCreates:::',
+    listOfObjectForCreates
+  );
+
+  /// step save awaitingMultipleStep of storageType graph ...........................................................
+  if (!settings.userAgent) {
+    _izContext.logger.debug('not api will create awaitingStep');
+    await asyncFlowSharedLib.createAwaitingMultipleSteps(
+      _izContext,
+      allAwaitingStepsId,
+      asyncFlowSharedLib.createPendingStepId(
+        hash({ objType: objType, identifiers: objInstanceFull.identifiers }),
+        PREFIX.CREATE_OBJECT_ASYNC_COMPLETE
+      )
+    );
+  }
+  //............................................................................................................
+
+  // process storageType : graph
+  let messageObject = {};
+  for (let objectCreate of listOfObjectForCreates) {
+    for (let [graphServiceName, objectForCreate] of Object.entries(
+      objectCreate
+    )) {
+      _izContext.logger.debug('Loop each graph:', {
+        [graphServiceName]: objectForCreate
+      });
+
+      if (objectForCreate.allStorageTagComplete === false) {
+        // needless check because in listOfObjectForCreates push case graph  only!
+        messageObject = {
+          objType: objType,
+          objInstanceFull: objectForCreate.objInstanceFull,
+          relationships: requestParams.relationships || []
+        };
+      }
+
+      _izContext.logger.debug('SENT MESSAGE EACH GRAPH::::', {
+        graphServiceName,
+        messageObject,
+        callingFlowConfig
+      });
+
+      Object.assign(messageObject, {
+        settings: { updatePropertiesOnMatch: false },
+        originTimestamp: Date.now()
+      });
+      if (settings.userAgent) {
+        _izContext.logger.debug('have userAgent will invoke GraphHandler');
+        let createNodeResult = await graphSharedLibs.createNodeV2(
+          _izContext,
+          graphServiceName,
+          objType,
+          objInstanceFullForGraph,
+          requestParams.relationships || [],
+          Date.now(),
+          {
+            updatePropertiesOnMatch: false
+          }
+        );
+        return createNodeResult;
+      } else if (settings.directInvoke) {
+        _izContext.logger.debug(
+          'Direction Invoke case will invoke GraphHandlerInv'
+        );
+        let createNodeResult = await graphSharedLibs.createNodeV2(
+          _izContext,
+          graphServiceName,
+          objType,
+          objInstanceFullForGraph,
+          requestParams.relationships,
+          Date.now(),
+          {
+            updatePropertiesOnMatch: false
+          }
+        );
+
+        return createNodeResult;
+      } else {
+        _izContext.logger.debug(
+          'not have userAgent will send Message to GraphHandler'
+        );
+        messageObject =
+          callingFlowSharedLib.addCallingFlowToSnsRequestMessageObject(
+            messageObject,
+            callingFlowSharedLib.addParentCallingFlowConfig(
+              callingFlowConfig, // receive parent callingFlowConfig.
+              callingFlowSharedLib.createCallingFlowConfig(
+                // calling flow own service
+                await lambdaSharedLib.lambdaFunctionName(
+                  _izContext,
+                  TOPIC_NAME_GENERATE_CODE.createNodeComplete
+                ),
+                {}
+              )
+            )
+          );
+
+        let messageToCreateObject = {
+          Message: JSON.stringify(messageObject),
+          TopicArn: await snsSharedLib.snsTopicArnByFlowSchema(
+            _izContext,
+            TOPIC_NAME_GRAPH_HANDLER.inCreateNode,
+            graphServiceName
+          )
+        };
+        _izContext.logger.debug(
+          'RequestParams before send to sqs messageToCreateObject ::::::: ',
+          messageToCreateObject
+        );
+        await sns.publishAsync(_izContext, messageToCreateObject);
+
+        return messageObject;
+      }
+    }
+  }
+
+  function createObjInstanceFullFieldsByStorageTag(
+    _izContext,
+    storageTag,
+    createDataDetail
+  ) {
+    let fields = {};
+    for (let fieldName of Object.keys(requestParams.fieldNames)) {
+      if (
+        createDataDetails[storageTag].fieldNames.includes(fieldName) &&
+        createDataDetail.fieldNames.includes(fieldName)
+      ) {
+        if (!objInstanceFull.identifiers.hasOwnProperty(fieldName)) {
+          if (!fields.hasOwnProperty(fieldName)) {
+            Object.assign(fields, {
+              [fieldName]: requestParams.fieldNames[fieldName]
+            });
+          }
+        }
+      }
+    }
+    return fields;
+  }
+}
+
+async function sharedCreateCompleteEndpoint(
+  _izContext,
+  returnValue,
+  status,
+  errorsFound,
+  graphServiceTag,
+  passBackProperties
+) {
+  let existsNode = null;
+  let createNodeFinished = false;
+  if (Object.keys(returnValue.queryResult.returnCreateNode.existsNode).length) {
+    existsNode = {
+      objType: {
+        objectType: returnValue.requestParams.objType.objectType,
+        serviceTag: returnValue.requestParams.objType.serviceTag
+      },
+      objInstanceFull:
+        returnValue.queryResult.returnCreateNode.existsNode.objInstanceFull
+    };
+  }
+
+  let awaitingStepId = await asyncFlowSharedLib.createAwaitingStepId(
+    hash({
+      objType: returnValue.requestParams.objType,
+      graphServerTag: graphServiceTag,
+      identifiers: returnValue.requestParams.objInstanceFull.identifiers,
+      callingFlowProperties:
+        passBackProperties.passBackProperties.parentPassBackProperties
+          .callingFlowProperties || {}
+    }),
+    PREFIX.CREATE_OBJECT_ASYNC
+  );
+  _izContext.logger.debug('awaitingStepId:', awaitingStepId);
+
+  let recordAwaitingSteps = await dynamodbSharedLib.query(
+    _izContext,
+    await dynamodbSharedLib.tableName(_izContext, 'AwaitingMultipleSteps'),
+    {
+      awaitingStepId: awaitingStepId
+    }
+  );
+  _izContext.logger.debug('record awaitingSteps::', recordAwaitingSteps);
+
+  await Promise.all(
+    recordAwaitingSteps.Items.map(async ({ pendingStepId }) => {
+      if (
+        await asyncFlowSharedLib.checkAllAwaitingStepsFinished(
+          _izContext,
+          pendingStepId,
+          awaitingStepId,
+          errorsFound
+        )
+      ) {
+        _izContext.logger.debug('finish all awaitingStep');
+
+        let messageObject = {
+          objType: {
+            objectType: returnValue.requestParams.objType.objectType,
+            serviceTag: returnValue.requestParams.objType.serviceTag
+          },
+          objInstanceFull:
+            returnValue.queryResult.returnCreateNode.objInstanceFull,
+          relationships: returnValue.queryResult.returnCreateNode.relationships,
+          existsNode: existsNode,
+          status: status,
+          errorsFound: errorsFound
+        };
+
+        messageObject =
+          callingFlowSharedLib.addParentPassBackPropertiesToSnsResponseMessageObject(
+            passBackProperties,
+            messageObject
+          );
+
+        let messageAttributes =
+          callingFlowSharedLib.addParentPassBackCallingFlowToSnsResponseMessageAttributes(
+            passBackProperties,
+            {}
+          );
+
+        let sendMessageOutCreateObjectComplete = {
+          Message: JSON.stringify(messageObject),
+          MessageAttributes: sns.createStringMessageAttributes(
+            _izContext,
+            messageAttributes
+          ),
+          TopicArn: await snsSharedLib.snsTopicArnByFlowSchema(
+            _izContext,
+            TOPIC_NAME_GENERATE_CODE.outCreateNodeComplete
+          )
+        };
+        _izContext.logger.debug(
+          'Send message to OutCreateObjectComplete :::>',
+          sendMessageOutCreateObjectComplete
+        );
+        await sns.publishAsync(_izContext, sendMessageOutCreateObjectComplete);
+        createNodeFinished = true;
+      } else {
+        createNodeFinished = false;
+      }
+
+      await asyncFlowSharedLib.removeAwaitingMultipleStep(
+        _izContext,
+        awaitingStepId,
+        pendingStepId,
+        errorsFound
+      );
+    })
+  );
+  return createNodeFinished;
+}
+
+async function validateAddOnDataStructure(_izContext, settings) {
+  let errorsFound = [];
+  let returnObjectValue = {};
+
+  const objTypeString = objType =>
+    `{ serviceTag: ${objType.serviceTag}, objectType: ${objType.objectType} }`;
+
+  const objectSchema = await getObjectSchema.getObjSchemaS3WithHierarchy(
+    _izContext,
+    settings.objType
+  );
+
+  if (!objectSchema) {
+    errorsFound.push(
+      `Invalid objType:${objTypeString(settings.objType)}, cannot generate attribute tree schema cause objectSchema is not exists`
+    );
+  }
+
+  if (
+    !objectSchema.hasOwnProperty('addOnDataStructure') ||
+    !Array.isArray(objectSchema.addOnDataStructure) ||
+    !objectSchema.addOnDataStructure.length
+  ) {
+    errorsFound.push(
+      `objType: ${objTypeString(settings.objType)}.cause addOnDataStructure is not exists or empty`
+    );
+  }
+
+  if (settings.hasOwnProperty('attributeTag')) {
+    let addOnAttributeTree = objectSchema.addOnDataStructure.find(addOn => {
+      return (
+        addOn.type === ADD_ON_DATA_STRUCTURE_TYPE.attributeTree &&
+        addOn.attributeTreeTag === settings.attributeTag
+      );
+    });
+    Object.assign(returnObjectValue, {
+      addOnAttributeTree: addOnAttributeTree
+    });
+    if (!addOnAttributeTree) {
+      errorsFound.push(
+        `objType: ${objTypeString(settings.objType)}. cannot generate attribute tree schema cause not found addOnDataStructure with attributeTreeTag: ${settings.attributeTag}`
+      );
+    }
+  }
+
+  Object.assign(returnObjectValue, {
+    objectSchema: objectSchema
+  });
+
+  return [returnObjectValue, errorsFound];
+}
+
+export default {
+  createBasicRelationshipsProperties,
+  createFieldNamesFromRequestParamsByObjectSchema,
+  sharedCreateEndpoint,
+  sharedCreateCompleteEndpoint,
+  validateAddOnDataStructure
+};