@izara_project/izara-core-library-service-schemas 1.0.106 → 1.0.107

package/index.js

@@ -25,6 +25,8 @@ import uploadUseCase from './src/libs/UploadUseCase.js';
 import relSchemaLib from './src/libs/RelSchemaLib.js';
 import s3Utils from './src/libs/s3Utils.js';
 import createNodeLib from './src/libs/CreateNodeLib.js';
+import dataDetailsLib from './src/libs/DataDetailsLib.js';
+import checkPermission from './src/libs/CheckPermission.js';
 
 import consts from './src/Consts.js';
 import getObjectSchema from './src/GetObjectSchema.js';
@@ -43,6 +45,8 @@ export {
   relSchemaLib,
   s3Utils,
   createNodeLib,
+  dataDetailsLib,
+  checkPermission,
 
   // ./src/
   consts,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@izara_project/izara-core-library-service-schemas",
-  "version": "1.0.106",
+  "version": "1.0.107",
   "description": "Schemas for the service and objects it controls",
   "main": "index.js",
   "scripts": {
@@ -30,6 +30,7 @@
     "@izara_project/izara-core-library-lambda": "^1.0.6",
     "@izara_project/izara-core-library-logger": "^1.0.8",
     "@izara_project/izara-core-library-s3": "^1.0.5",
+    "@izara_project/izara-core-library-sqs": "^1.0.5",
     "@izara_project/izara-shared-core": "^1.0.8",
     "@izara_project/izara-shared-service-schemas": "^1.0.35",
     "glob": "^13.0.0",

package/src/IdentifiersObject.js

@@ -507,6 +507,164 @@ async function explodedNestedIdentifiersConcat(
   return concatIdentifier.split(deliminator.repeat(compositeDeliminatorAmount + level));
 }
 
+/**
+ * create dynamoDb identifiers using request param and objectSchema
+ *
+ * @param {Object} _izContext
+ * @param {Object} objectSchema
+ * @param {Object} reqIdentifiers       - input identifier from calling function
+ */
+async function generateDynamoDbIdentifiers(
+  _izContext,
+  objectSchema,
+  reqIdentifiers,
+  bucketName = process.env.iz_serviceSchemaBucketName
+) {
+  // start create identifiers for dynamo
+  let dynamoIdentifiers = {};
+
+  for (let identifier of objectSchema.identifiers) {
+    if (identifier.name) {
+      let deliminator =
+        identifier.deliminator || coreConsts.DEFAULT_IDENTIFIER_DELIMINATOR;
+      let currentDeliminatorAmount = 0;
+
+      let concatenateValue = [];
+      for (let fieldName of identifier.fieldNames) {
+        concatenateValue.push(reqIdentifiers[fieldName]);
+
+        let deliminatorTreeFieldName =
+          await deliminatorTree.generateDeliminatorTreePerFieldName(
+            _izContext,
+            fieldName,
+            objectSchema.fieldNames[fieldName],
+            {},
+            bucketName
+          );
+        _izContext.logger.debug(
+          'deliminatorTreeFieldName: ',
+          deliminatorTreeFieldName
+        );
+
+        if (
+          deliminatorTreeFieldName[fieldName]?.deliminatorList?.[deliminator]
+        ) {
+          currentDeliminatorAmount = Math.max(
+            currentDeliminatorAmount,
+            deliminatorTreeFieldName[fieldName].deliminatorList[deliminator]
+          );
+        }
+      }
+
+      Object.assign(dynamoIdentifiers, {
+        [identifier.name]: concatenateValue.join(
+          deliminator.repeat(currentDeliminatorAmount + 1)
+        )
+      });
+    } else {
+      Object.assign(dynamoIdentifiers, {
+        [identifier.fieldName]: reqIdentifiers[identifier.fieldName]
+      });
+    }
+  }
+
+  _izContext.logger.debug('dynamoIdentifiers: ', dynamoIdentifiers);
+
+  return dynamoIdentifiers;
+}
+
+/**
+ * create dynamoDb identifiers using request param and objectSchema
+ *
+ * @param {Object} _izContext
+ * @param {Object} objectSchema
+ * @param {Object} reqIdentifiers       - input identifier from calling function
+ */
+const generateDynamoDbIdentifiersWithCache = inMemoryCacheLib(
+  generateDynamoDbIdentifiers, // fn
+  {
+    // setting
+    max: 20,
+    maxAge: 86400000,
+    promise: true,
+    profileName: 'generateDynamoDbIdentifiers',
+    normalizer: function (args) {
+      return hash([args[1], args[2]]);
+    }
+  }
+);
+
+
+/**
+ *
+ * @param {Object} _izContext
+ * @param {Object} objectSchema
+ * @param {Object} dynamoStorageResource
+ * @param {Object} reqIdentifiers
+ */
+async function dynamoDbIdentifiersByStorageResource(
+  _izContext,
+  objectSchema,
+  dynamoStorageResource,
+  reqIdentifiers,
+  bucketName = process.env.iz_serviceSchemaBucketName
+) {
+  let dynamoIdentifiers = await generateDynamoDbIdentifiersWithCache(
+    _izContext,
+    objectSchema,
+    reqIdentifiers,
+    bucketName
+  );
+
+  let identifiersByStorageResource = {};
+
+  if (dynamoStorageResource.groupByPartitionKeyField) {
+    let deliminatorTreeIdentifiers =
+      await deliminatorTree.generateDeliminatorTreeIdentifier(
+        _izContext,
+        objectSchema,
+        bucketName
+      );
+    _izContext.logger.debug('deliminatorTree: ', deliminatorTreeIdentifiers);
+
+    let compositeKeyDeliminator =
+      objectSchema.compositeKeyDeliminator ||
+      coreConsts.DEFAULT_IDENTIFIER_DELIMINATOR;
+
+    let currentDeliminatorAmount = 0;
+
+    for (let currentDeliminator of Object.values(deliminatorTreeIdentifiers)) {
+      if (currentDeliminator.deliminator === compositeKeyDeliminator) {
+        if (
+          currentDeliminatorAmount <
+          currentDeliminator.deliminatorList[compositeKeyDeliminator]
+        ) {
+          currentDeliminatorAmount =
+            currentDeliminator.deliminatorList[compositeKeyDeliminator];
+        }
+      }
+    }
+
+    // sort dynamoDbIdentifiers data by objectSchema
+    let sortedIdentifiersValue = [];
+    for (let identifier of objectSchema.identifiers) {
+      let identifierName = identifier.fieldName || identifier.name;
+      sortedIdentifiersValue.push(dynamoIdentifiers[identifierName]);
+    }
+
+    let newIdentifierValue = sortedIdentifiersValue.join(
+      compositeKeyDeliminator.repeat(currentDeliminatorAmount + 1)
+    );
+
+    identifiersByStorageResource = {
+      [dynamoStorageResource.groupByPartitionKeyField]: newIdentifierValue
+    };
+  } else {
+    identifiersByStorageResource = dynamoIdentifiers;
+  }
+
+  return identifiersByStorageResource;
+}
 
 
 export default {
@@ -517,5 +675,10 @@ export default {
   identifiersFromIdentifiersConcat, // identifiersObjectFromCompositeIdentifier
 
   createNestedIdentifiersConcat, // createConcatIdentifier
-  explodedNestedIdentifiersConcat // explodedConcatIdentifier
+  explodedNestedIdentifiersConcat, // explodedConcatIdentifier
+
+  generateDynamoDbIdentifiers,
+  generateDynamoDbIdentifiersWithCache,
+
+  dynamoDbIdentifiersByStorageResource
 }

package/src/Utils.js

@@ -766,6 +766,75 @@ async function getApiLinksV2(objectTypes) {
   }
 }
 
+
+function createFieldForUpdateDynamoDb(
+  _izContext,
+  objectSchema,
+  dynamoDataDetail,
+  fields
+) {
+  _izContext.logger.debug('createFieldForUpdateDynamoDb', {
+    objectSchema,
+    dynamoDataDetail,
+    fields
+  });
+
+  let fieldsForUpdateDynamo = {};
+
+  for (let [fieldName, fieldSetting] of Object.entries(
+    objectSchema.fieldNames
+  )) {
+    if (
+      fieldSetting.canUpdate == true ||
+      !fieldSetting.hasOwnProperty('canUpdate')
+    ) {
+      if (dynamoDataDetail.fieldNames.includes(fieldName)) {
+        if (fields[fieldName]) {
+          if (!fieldsForUpdateDynamo.hasOwnProperty(fields[fieldName])) {
+            _izContext.logger.debug('assign new fieldName', fields[fieldName]);
+            Object.assign(fieldsForUpdateDynamo, {
+              [fieldName]: fields[fieldName]
+            });
+          }
+        }
+      }
+    }
+  }
+  return fieldsForUpdateDynamo;
+}
+
+/**
+ * @param {object} record               - One record from sqs.
+ * @param {string} messageFailError     - The message error.
+ * @param {object} lambdaFunctionName   - Queue anme what to send to dlq./name of Lambda being invoked, used to build SQS/DLQ
+ *
+ */
+async function messageToDlq(record, messageFailError, queueUrl) {
+  let messageBody = record.body;
+
+  let params = {
+    QueueUrl: queueUrl,
+    MessageAttributes: {
+      messageFailError: {
+        DataType: 'String',
+        StringValue: messageFailError
+      }
+    },
+    MessageBody: JSON.stringify(messageBody)
+  };
+
+  record._izContext.logger.debug(
+    'messageToDlq, params before sending DLQ',
+    params
+  );
+
+  await sqs.sendMessage(record._izContext, params);
+  record._izContext.logger.debug(
+    '----- messageToDlq sendMessage success -----'
+  );
+}
+
+
 export default {
   createObjType,
   getIdentifierTypeByPriority,
@@ -792,5 +861,8 @@ export default {
 
   createLinkTypeId,
   getApiLinksV1,
-  getApiLinksV2
+  getApiLinksV2,
+
+  createFieldForUpdateDynamoDb,
+  messageToDlq
 }

package/src/ValidatorSchema.js

@@ -24,31 +24,30 @@ import { objectHash as hash } from '@izara_project/izara-shared-core';
 import Logger from '@izara_project/izara-core-library-logger';
 import uploadUseCase from './libs/UploadUseCase.js';
 
+import { validateObjType } from '@izara_project/izara-shared-service-schemas';
+import { validator as validateObject } from '@izara_project/izara-core-library-core';
+import sqsSharedLib from '@izara_project/izara-core-library-sqs';
+import explodedReqParams from './libs/ExplodedReqParams.js';
+import getObjectSchema from './GetObjectSchema.js';
 import serviceConfig from './ServiceConfig.js';
+import consts from './Consts.js';
 import utils from './Utils.js';
-import getObjectSchema from './GetObjectSchema.js';
-import consts from './Consts.js'
-import { validateObjType } from '@izara_project/izara-shared-service-schemas';
-
-// module.exports.createFieldNameUniqueRequestId = (prefix = 'cache') => {
-//   return prefix + "UniqueRequestId";
-// }
 
-// module.exports.createFieldNameStatus = (prefix = 'cache') => {
-//   return prefix + 'Status';
-// }
 
-// module.exports.createFieldNameErrorsFound = (prefix = 'cache') => {
-//   return prefix + 'ErrorsFound';
-// }
+const schemaFunctionPerAction = {
+  [consts.ACTIONS.create]: generateValidatorSchemaForCreate,
+  [consts.ACTIONS.update]: generateValidatorSchemaForUpdate,
+  [consts.ACTIONS.get]: generateValidatorSchemaForIdentifier,
+  [consts.ACTIONS.delete]: generateValidatorSchemaForIdentifier
+};
 
-// module.exports.createFieldNameCacheComplete = (prefix = 'cache') => { // timeStamp
-//   return prefix + 'CacheComplete';
-// }
+const explodeDataPerAction = {
+  [consts.ACTIONS.create]: explodedReqParams.explodedDataForCreate,
+  [consts.ACTIONS.get]: explodedReqParams.explodedDataForIdentifiers,
+  [consts.ACTIONS.update]: explodedReqParams.explodedDataForUpdate,
+  [consts.ACTIONS.delete]: explodedReqParams.explodedDataForIdentifiers
+};
 
-// module.exports.createFlowParamsFieldName = (prefix = 'cache') => {
-//   return prefix + 'FlowParams';
-// }
 
 function createSpecificFieldNameForCacheTable(prefix = 'cache', statusType) {
   return {
@@ -1375,11 +1374,112 @@ async function validateLocalSchema(_izContext, schemasPath, serviceConfigPath) {
   }
 }
 
+/**
+ * use to validate record for Lambda handler hdrSqs/hdrDsq
+ * Note!: cannot throw error when use this function outside recordHandler
+ *
+ * @param {object} record                         - one record from sqs
+ * @param {string} lambdaFunctionName             - name of Lambda used to build SQS/DLQ // QueueName
+ * @param {string} objectType                     - name of ObjectType
+ * @param {string} action                         - action type of Lambda create | update | get | delete
+ * @param {object} [setting]
+ * @param {string[]} [setting.specificFieldNames] - optional - specific fieldNames use as param in generateValidatorFunction
+ */
+async function validateSchemaPerRecord(
+  record,
+  lambdaFunctionName,
+  objType,
+  action,
+  setting = {
+    bucketName: process.env.iz_serviceSchemaBucketName,
+    specificFieldNames: []
+  }
+) {
+  try {
+    record._izContext.logger.debug(
+      'validateSchemaPerRecord: ',
+      record,
+      lambdaFunctionName,
+      objType,
+      action,
+      setting
+    );
+
+    if (!schemaFunctionPerAction[action] || !explodeDataPerAction[action]) {
+      throw new Error(
+        `Not found function for generateValidatorSchema or function for explodeData of action:${action}`
+      );
+    }
+
+    const objectSchema = await getObjectSchema.getObjSchemaS3WithHierarchy(
+      record._izContext,
+      objType
+    );
+
+    if (!objectSchema) {
+      throw new Error('not have objectSchema in service');
+    }
+
+    const generateValidatorFunction = schemaFunctionPerAction[action];
+    const explodeDataFunction = explodeDataPerAction[action];
+
+    const generatedSchema = await generateValidatorFunction(
+      record._izContext,
+      objType,
+      setting
+    );
+    record._izContext.logger.debug(
+      `generatedSchema in validateSchemaPerRecord : `,
+      generatedSchema
+    );
+
+    const explodedDataRequestParams = await explodeDataFunction(
+      record._izContext,
+      record.body.Message,
+      objectSchema,
+      setting
+    );
+
+    record._izContext.logger.debug(
+      'explodedDataRequestParams in validateSchemaPerRecord : ',
+      JSON.stringify(explodedDataRequestParams)
+    );
+
+    let validateStatus = validateObject(
+      generatedSchema,
+      explodedDataRequestParams
+    );
+    record._izContext.logger.debug('validateStatus : ', validateStatus);
+
+    // if not pass validate will sent message to dlq and throw NoRetryError
+    if (!validateStatus.pass) {
+      await utils.messageToDlq(
+        record,
+        `Invalid: ${validateStatus.error}`,
+        await sqsSharedLib.sqsQueueUrlDLQ(record._izContext, lambdaFunctionName)
+      );
+      record._izError = new Error(validateStatus.error);
+    }
+  } catch (error) {
+    record._izContext.logger.debug('validate PerSchema Record error', error);
+    await utils.messageToDlq(
+      record,
+      error.message,
+      await sqsSharedLib.sqsQueueUrlDLQ(record._izContext, lambdaFunctionName)
+    );
+    record._izError = new Error(error);
+  }
+}
+
+
+
 export default {
   generateValidatorSchemaForCreate,
   generateValidatorSchemaForUpdate,
   generateValidatorSchemaForIdentifier,
 
   generateValidatorSchemaForExplodedData,
-  validateLocalSchema
+  validateLocalSchema,
+
+  validateSchemaPerRecord,
 }

package/src/libs/CheckPermission.js

@@ -0,0 +1,122 @@
+/*
+Copyright (C) 2021 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import lambdaSharedLib from '@izara_project/izara-core-library-lambda';
+import { lambda } from '@izara_project/izara-core-library-external-request';
+import { consts } from '@izara_project/izara-core-library-core';
+
+/**
+ * Checks a user's permission by invoking another Lambda function.
+ * Adheres to Izara.io backend syntax guidelines.
+ *
+ * @param {object} _izContext - The context object containing logger, credentials, etc.
+ * @param {object} payload - The data payload for the permission check.
+ * @param {string} [payload.objectType] - The type of the object (e.g., 'INVOICE'). Must be used with `action`.
+ * @param {string} [payload.action] - The action to perform (e.g., 'READ', 'APPROVE'). Must be used with `objectType`.
+ * @param {string} [payload.flowTag] - The tag for a flow-based permission check. Used instead of `objectType`/`action`.
+ * @param {string} [payload.serviceName] - The name of the calling service (optional).
+ * @returns {Promise<{status: boolean, errorFound?: string}>}
+ * Resolves with the result of the permission check:
+ * - `status`: Whether the user has permission.
+ * - `errorFound`: Optional error message if the permission check failed.
+ *
+ * @throws {Error} If the payload is invalid or Lambda invocation fails.
+ */
+async function checkPermission(_izContext, payload) {
+  // Validate that the payload object itself is provided.
+  if (!payload || typeof payload !== 'object') {
+    const error = new Error('Payload object is required.');
+    _izContext.logger.error(error.message);
+    throw error;
+  }
+
+  const userId = _izContext.correlationIds.get(consts.BASE_USER_ID);
+  const targetId = _izContext.correlationIds.get(consts.TARGET_ID);
+
+  // Validate the required 'userId' field.
+  if (!userId || typeof userId !== 'string' || userId.trim() === '') {
+    const error = new Error(
+      'userId is required and must be a non-empty string.'
+    );
+    _izContext.logger.error(error.message);
+    throw error;
+  } else {
+    payload.userId = userId;
+  }
+
+  if (targetId) {
+    payload.targetId = targetId;
+  }
+
+  // Define validation flags for business rules.
+  const hasObjectParams = payload.objectType || payload.action;
+  const hasFlowTag = payload.flowTag;
+
+  // Rule: Cannot mix object-based and flow-based parameters.
+  if (hasObjectParams && hasFlowTag) {
+    const error = new Error(
+      'Invalid payload: Cannot provide flowTag together with objectType or action.'
+    );
+    _izContext.logger.error(error.message, { payload });
+    throw error;
+  }
+
+  // Rule: If using object-based, both objectType and action are required.
+  if (hasObjectParams && (!payload.objectType || !payload.action)) {
+    const error = new Error(
+      'Invalid payload: Both objectType and action must be provided together.'
+    );
+    _izContext.logger.error(error.message, { payload });
+    throw error;
+  }
+
+  // Rule: Must provide at least one of the two valid schemas.
+  if (!hasObjectParams && !hasFlowTag) {
+    const error = new Error(
+      'Invalid payload: Must provide either (objectType and action) or flowTag.'
+    );
+    _izContext.logger.error(error.message, { payload });
+    throw error;
+  }
+
+  // Log the payload for debugging before invoking the next service.
+  _izContext.logger.debug('Checking permission with payload:', payload);
+
+  try {
+    const lambdaName = lambdaSharedLib.lambdaFunctionName(
+      _izContext,
+      'CheckPermissionHdrInv',
+      'UserAccount'
+    );
+
+    const result = await lambda.invokeSync(_izContext, lambdaName, payload);
+
+    _izContext.logger.info('Permission check invoked successfully.');
+
+    // example result = { status: boolean , errorFound: string }
+    return result;
+  } catch (err) {
+    // Log the full error object for better traceability.
+    _izContext.logger.error(
+      'Error invoking CheckPermissionHdrInv Lambda: ',
+      err
+    );
+    throw err;
+  }
+}
+
+export default checkPermission;

package/src/libs/DataDetailsLib.js

@@ -0,0 +1,642 @@
+/*
+Copyright(C) 2021 Sven Mason < http://izara.io>
+
+This program is free software: you can redistribute it and / or modify
+it under the terms of the GNU Affero General Public License as
+  published by the Free Software Foundation, either version 3 of the
+License, or(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.If not, see < http://www.gnu.org/licenses/>.
+*/
+
+import { NoRetryError } from "@izara_project/izara-core-library-core";
+import serviceConfig from "../ServiceConfig.js";
+import utils from "../Utils.js";
+
+/**
+ * helper function for get endpoint
+ * create get data detail for each storageTag
+ *
+ * @param {Object} _izContext
+ * @param {Object} objectSchema
+ */
+async function createGetDataDetails(
+  _izContext,
+  objectSchema,
+  settings = { bucketName: process.env.iz_serviceSchemaBucketName }
+) {
+  // group versionedData per storageTag
+  // or should group versionedData per graph serviceTag
+  let versionedDataPerStorageTag = {};
+  if (objectSchema?.addOnDataStructure?.length) {
+    for (let addOn of objectSchema.addOnDataStructure) {
+      if (addOn.type !== 'versionedData') {
+        continue;
+      }
+
+      if (
+        !versionedDataPerStorageTag.hasOwnProperty(addOn.storageResourceTag)
+      ) {
+        versionedDataPerStorageTag[addOn.storageResourceTag] = [];
+      }
+
+      versionedDataPerStorageTag[addOn.storageResourceTag].push(addOn);
+    }
+  }
+
+  _izContext.logger.debug(
+    'versionedDataPerStorageTag: ',
+    versionedDataPerStorageTag
+  );
+
+  let getGraphDataDetails = {};
+  let getDynamoDbDataDetails = {};
+
+  // reference to graph serviceTag that already used
+  let graphStorageTagPerGraphServiceTag = {}; // {serviceTag:storageTag,... }
+
+  for (let [storageTag, storageData] of Object.entries(
+    objectSchema.storageResources
+  )) {
+    // collect fieldNames per storageTag
+    let storageTagFieldNames = [];
+    for (let [fieldName, fieldNameData] of Object.entries(
+      objectSchema.fieldNames
+    )) {
+      // if (fieldName.startsWith("versionedDataField")) {
+      //   continue;
+      // }
+
+      if (fieldNameData.storageResourceTags.includes(storageTag)) {
+        storageTagFieldNames.push(fieldName);
+      }
+    }
+
+    if (storageData.storageType === coreConsts.STORAGE_TYPES.graph) {
+      let useStorageTag = storageTag;
+      let graphServiceTag = await serviceConfig.getGraphServiceTagWithCache(
+        _izContext,
+        storageData.graphServerTag,
+        settings.bucketName
+      );
+
+      // check duplicate graph serviceTag
+      if (!graphStorageTagPerGraphServiceTag.hasOwnProperty(graphServiceTag)) {
+        graphStorageTagPerGraphServiceTag[graphServiceTag] = storageTag;
+      } else {
+        // overwrite useStorageTag if graphServiceTag already used
+        _izContext.logger.debug('current storageTag: ', storageTag);
+        _izContext.logger.debug('use storageTag: ', useStorageTag);
+
+        useStorageTag = graphStorageTagPerGraphServiceTag[graphServiceTag];
+      }
+
+      if (!getGraphDataDetails.hasOwnProperty(useStorageTag)) {
+        getGraphDataDetails[useStorageTag] = {
+          storageType: storageData.storageType,
+          graphServiceTag: graphServiceTag,
+          fieldNames: storageTagFieldNames
+        };
+      } else {
+        getGraphDataDetails[useStorageTag].fieldNames = [
+          ...new Set(
+            getGraphDataDetails[useStorageTag].fieldNames.concat(
+              storageTagFieldNames
+            )
+          )
+        ];
+      }
+
+      // add versionedData fieldNames
+      if (versionedDataPerStorageTag.hasOwnProperty(storageTag)) {
+        if (
+          !getGraphDataDetails[useStorageTag].hasOwnProperty('versionedDatas')
+        ) {
+          getGraphDataDetails[useStorageTag].versionedDatas = {};
+        }
+
+        for (let versionedData of versionedDataPerStorageTag[storageTag]) {
+          _izContext.logger.debug(
+            'versionedData.fieldNames',
+            versionedData.fieldNames
+          );
+          let versionedDataFieldNames = Object.keys(versionedData.fieldNames);
+          Object.assign(getGraphDataDetails[useStorageTag].versionedDatas, {
+            [versionedData.versionedDataLabel]: versionedDataFieldNames
+          });
+        }
+      }
+    } else if (storageData.storageType === coreConsts.STORAGE_TYPES.dynamoDB) {
+      let serviceTag = storageData.serviceTag || process.env.iz_serviceTag;
+      Object.assign(getDynamoDbDataDetails, {
+        [storageTag]: {
+          storageType: storageData.storageType,
+          tableName: storageData.tableName,
+          serviceTag: serviceTag,
+          fieldNames: storageTagFieldNames
+        }
+      });
+
+      if (storageData.groupByPartitionKeyField) {
+        Object.assign(getDynamoDbDataDetails[storageTag], {
+          groupByPartitionKeyField: storageData.groupByPartitionKeyField
+        });
+      }
+    } else {
+      throw new NoRetryError('Unhanled storageType');
+    }
+  }
+
+  _izContext.logger.debug('getGraphDataDetail: ', getGraphDataDetails);
+  _izContext.logger.debug('getDynamoDbData: ', getDynamoDbDataDetails);
+
+  //-------- sort descending order of fieldNames.length --------
+  let sortedGetGraphWithVersionedData = Object.entries(getGraphDataDetails)
+    .filter(getGraphDetail =>
+      getGraphDetail[1].hasOwnProperty('versionedDatas')
+    )
+    .sort(
+      (storageDataA, storageDataB) =>
+        storageDataB[1].fieldNames.length - storageDataA[1].fieldNames.length
+    );
+
+  _izContext.logger.debug(
+    'sortedGetGraphWithVersionedData: ',
+    sortedGetGraphWithVersionedData
+  );
+
+  let sortedGetGraphWithoutVersionedData = Object.entries(getGraphDataDetails)
+    .filter(
+      getGraphDetail => !getGraphDetail[1].hasOwnProperty('versionedDatas')
+    )
+    .sort(
+      (storageDataA, storageDataB) =>
+        storageDataB[1].fieldNames.length - storageDataA[1].fieldNames.length
+    );
+
+  _izContext.logger.debug(
+    'sortedGetGraphWithoutVersionedData: ',
+    sortedGetGraphWithoutVersionedData
+  );
+
+  let sortedGetDynamoDbData = Object.entries(getDynamoDbDataDetails).sort(
+    (storageDataA, storageDataB) =>
+      storageDataB[1].fieldNames.length - storageDataA[1].fieldNames.length
+  );
+
+  _izContext.logger.debug('sortedGetDynamoDbData: ', sortedGetDynamoDbData);
+
+  // after sort data then filter duplicate fieldName
+  let allSortedGetDataDetails = [
+    ...sortedGetGraphWithVersionedData,
+    ...sortedGetDynamoDbData,
+    ...sortedGetGraphWithoutVersionedData
+  ];
+  _izContext.logger.debug('allSortedGetDataDetails: ', allSortedGetDataDetails);
+
+  let usedFieldName = [];
+  for (let [storageTag, getDataDetail] of allSortedGetDataDetails) {
+    let remainsFieldNames = [];
+    while (getDataDetail.fieldNames.length) {
+      let checkedFiedName = getDataDetail.fieldNames.pop();
+      if (!usedFieldName.includes(checkedFiedName)) {
+        usedFieldName.push(checkedFiedName);
+        remainsFieldNames.push(checkedFiedName);
+      }
+    }
+
+    getDataDetail.fieldNames = remainsFieldNames;
+  }
+
+  _izContext.logger.debug('newSortedArray after: ', allSortedGetDataDetails);
+
+  let getDataDetailResults = Object.fromEntries(
+    allSortedGetDataDetails.filter(([storageTag, getDataDetail]) => {
+      return (
+        getDataDetail.fieldNames.length ||
+        getDataDetail.hasOwnProperty('versionedDatas')
+      );
+    })
+  );
+
+  _izContext.logger.debug('getDataDetailResults: ', getDataDetailResults);
+
+  for (const getDataDetail of Object.values(getDataDetailResults)) {
+    if (getDataDetail.hasOwnProperty('versionedDatas')) {
+      getDataDetail.versionedDataLabels = [];
+      for (let [versionedDataLabel, fieldNames] of Object.entries(
+        getDataDetail.versionedDatas
+      )) {
+        getDataDetail.versionedDataLabels.push(versionedDataLabel);
+        getDataDetail.fieldNames.push(...fieldNames);
+      }
+    }
+  }
+
+  return getDataDetailResults;
+}
+
+/**
+ * receive result from multiple database and create data format depend on returnFormat setting
+ *
+ * @param {object} _izContext
+ * @param {object[]} getResults
+ * @param {object} objectSchema
+ * @returns
+ */
+function collectGetData(
+  _izContext,
+  getResults,
+  objectSchema,
+  returnSystemFieldsName
+) {
+  _izContext.logger.debug('collectGetData: ', {
+    getResults,
+    objectSchema,
+    returnSystemFieldsName
+  });
+
+  let collectedData = {
+    identifiers: {},
+    fields: {}
+  };
+
+  if (returnSystemFieldsName) {
+    Object.assign(collectedData, { versionDataSystemFields: {} });
+  }
+  const identifierFieldNames = new Set(
+    utils.getUsedFieldNamesOfIdentifiers(_izContext, objectSchema.identifiers)
+  );
+
+  let addedFieldNames = new Set();
+
+  //  if isFound === true then return null from this function
+  let isFound = false;
+
+  for (const [getData, getDataDetail] of getResults) {
+    if (!getData || !Object.keys(getData).length) {
+      continue;
+    }
+
+    // this process will add result from graph and dynamo to collectedData object by add identifiers and fieldName from getData
+    if (getDataDetail.storageType === coreConsts.STORAGE_TYPES.graph) {
+      if (
+        getData.returnValue.queryResults.refactoredNodes.objInstanceFull &&
+        Object.keys(
+          getData.returnValue.queryResults.refactoredNodes.objInstanceFull
+        ).length
+      ) {
+        for (const getDetailFieldName of getDataDetail.fieldNames) {
+          if (addedFieldNames.has(getDetailFieldName)) {
+            continue;
+          }
+
+          if (
+            getData.returnValue.queryResults.refactoredNodes.objInstanceFull.identifiers.hasOwnProperty(
+              getDetailFieldName
+            )
+          ) {
+            Object.assign(collectedData.identifiers, {
+              [getDetailFieldName]:
+                getData.returnValue.queryResults.refactoredNodes.objInstanceFull
+                  .identifiers[getDetailFieldName]
+            });
+            addedFieldNames.add(getDetailFieldName);
+            isFound = true;
+          } else if (
+            getData.returnValue.queryResults.refactoredNodes.objInstanceFull.fields.hasOwnProperty(
+              getDetailFieldName
+            )
+          ) {
+            Object.assign(collectedData.fields, {
+              [getDetailFieldName]:
+                getData.returnValue.queryResults.refactoredNodes.objInstanceFull
+                  .fields[getDetailFieldName]
+            });
+            addedFieldNames.add(getDetailFieldName);
+            isFound = true;
+          }
+        }
+      }
+      if (
+        getData.returnValue.queryResults.refactoredNodes
+          .versionDataSystemFields ||
+        returnSystemFieldsName == true
+      ) {
+        Object.assign(
+          collectedData.versionDataSystemFields,
+          getData.returnValue.queryResults.refactoredNodes
+            .versionDataSystemFields
+        );
+      }
+    } else if (
+      getDataDetail.storageType === coreConsts.STORAGE_TYPES.dynamoDB
+    ) {
+      for (const getDetailFieldName of getDataDetail.fieldNames) {
+        if (addedFieldNames.has(getDetailFieldName)) {
+          continue;
+        }
+
+        if (getData.hasOwnProperty(getDetailFieldName)) {
+          if (identifierFieldNames.has(getDetailFieldName)) {
+            Object.assign(collectedData.identifiers, {
+              [getDetailFieldName]: getData[getDetailFieldName]
+            });
+            addedFieldNames.add(getDetailFieldName);
+            isFound = true;
+          } else {
+            Object.assign(collectedData.fields, {
+              [getDetailFieldName]: getData[getDetailFieldName]
+            });
+            addedFieldNames.add(getDetailFieldName);
+            isFound = true;
+          }
+        }
+      }
+    } else {
+      throw new NoRetryError('collectGetData unhandled storageType');
+    }
+  }
+
+  if (isFound) {
+    return collectedData;
+  } else {
+    return null;
+  }
+}
+
+async function createUpdateDataDetail(
+  _izContext,
+  objectSchema,
+  settings = { bucketName: process.env.iz_serviceSchemaBucketName }
+) {
+  let getGraphDataDetails = {};
+  let getDynamoDbDataDetails = {};
+  let allUpdateDataDetail = {};
+  for (let [storageTag, storageProperties] of Object.entries(
+    objectSchema.storageResources
+  )) {
+    let storageFieldNames = [];
+    for (let [fieldName, fieldNameProperties] of Object.entries(
+      objectSchema.fieldNames
+    )) {
+      if (fieldNameProperties.storageResourceTags.includes(storageTag)) {
+        let versionedDataLabel = fieldNameProperties.versionedDataLabel;
+        if (!fieldName.includes(versionedDataLabel)) {
+          // not get fieldNames VersionedData_VersionedDataLabel_VersionedDataFieldNames
+          storageFieldNames.push(fieldName);
+        }
+      }
+    }
+    if (storageProperties.storageType === coreConsts.STORAGE_TYPES.graph) {
+      let graphStorageTagPerGraphServiceTag = {};
+      let useStorageTag = storageTag;
+      let graphServiceName = await serviceConfig.getGraphServiceTagWithCache(
+        _izContext,
+        storageProperties.graphServerTag,
+        settings.bucketName
+      );
+      if (!graphStorageTagPerGraphServiceTag.hasOwnProperty(graphServiceName)) {
+        graphStorageTagPerGraphServiceTag[graphServiceName] = storageTag;
+        _izContext.logger.debug(
+          'graphStorageTagPerGraphServiceTag',
+          graphStorageTagPerGraphServiceTag
+        );
+      } else {
+        _izContext.logger.debug('current storageTag', storageTag);
+        _izContext.logger.debug('used storageTag', useStorageTag);
+        useStorageTag = graphStorageTagPerGraphServiceTag[graphServiceName];
+      }
+      if (!getGraphDataDetails.hasOwnProperty(useStorageTag)) {
+        getGraphDataDetails[graphServiceName] = {
+          storageType: storageProperties.storageType,
+          fieldNames: storageFieldNames
+        };
+      } else {
+        getGraphDataDetails[graphServiceName].fieldNames = [
+          ...new Set(
+            getGraphDataDetails[useStorageTag].fieldNames.concat(
+              storageFieldNames
+            )
+          )
+        ];
+      }
+    } else if (
+      storageProperties.storageType === coreConsts.STORAGE_TYPES.dynamoDB
+    ) {
+      let serviceTag =
+        storageProperties.serviceTag || process.env.iz_serviceTag;
+      Object.assign(getDynamoDbDataDetails, {
+        [storageTag]: {
+          storageType: storageProperties.storageType,
+          tableName: storageProperties.tableName,
+          serviceTag: serviceTag,
+          fieldNames: storageFieldNames
+        }
+      });
+
+      if (storageTag.groupByPartitionKeyField) {
+        Object.assign(getDynamoDbDataDetails[storageTag], {
+          groupByPartitionKeyField: storageProperties.groupByPartitionKeyField
+        });
+      }
+    }
+  }
+
+  _izContext.logger.debug('getGraphDataDetails', getGraphDataDetails);
+  _izContext.logger.debug('getDynamoDataDetail', getDynamoDbDataDetails);
+
+  Object.assign(
+    allUpdateDataDetail,
+    getDynamoDbDataDetails,
+    getGraphDataDetails
+  );
+  return allUpdateDataDetail;
+}
+
+async function createDataDetailsLib(
+  _izContext,
+  objectSchemas,
+  settings = { bucketName: process.env.iz_serviceSchemaBucketName }
+) {
+  _izContext.logger.debug('Lib: createDataDetailsLib:', {
+    objectSchemas: objectSchemas
+  });
+  
+  let createDataDetails = {};
+
+  let storageResources = objectSchemas.storageResources;
+  _izContext.logger.debug('storageResources:', storageResources);
+
+  for (let [keyFieldName, settingFieldName] of Object.entries(
+    objectSchemas.fieldNames
+  )) {
+    // _izContext.logger.debug("Loop fieldNamesObjectSchemas", { keyFieldName });
+
+    for (let eachStorageResourceTag of settingFieldName.storageResourceTags) {
+      // _izContext.logger.debug("Loop eachStorageResourceTags", eachStorageResourceTag);
+
+      if (!storageResources.hasOwnProperty(eachStorageResourceTag)) {
+        throw new Error("storageResources is'n exist"); // should be validate in step upload in s3
+      } else {
+        if (
+          storageResources[eachStorageResourceTag].storageType ==
+          coreConsts.STORAGE_TYPES.dynamoDB
+        ) {
+          if (createDataDetails.hasOwnProperty(eachStorageResourceTag)) {
+            // _izContext.logger.debug("SAME STG DB", eachStorageResourceTag);
+            createDataDetails[eachStorageResourceTag].fieldNames.push(
+              keyFieldName
+            );
+          } else {
+            Object.assign(createDataDetails, {
+              [eachStorageResourceTag]: {
+                storageType: coreConsts.STORAGE_TYPES.dynamoDB,
+                tableName: storageResources[eachStorageResourceTag].tableName,
+                serviceTag:
+                  storageResources[eachStorageResourceTag].serviceTag ||
+                  process.env.iz_serviceTag,
+                fieldNames: [keyFieldName]
+              }
+            });
+
+            if (
+              storageResources[eachStorageResourceTag].hasOwnProperty(
+                'groupByPartitionKeyField'
+              )
+            ) {
+              Object.assign(createDataDetails[eachStorageResourceTag], {
+                groupByPartitionKeyField:
+                  storageResources[eachStorageResourceTag]
+                    .groupByPartitionKeyField
+              });
+            }
+          }
+        } else if (
+          storageResources[eachStorageResourceTag].storageType ==
+          coreConsts.STORAGE_TYPES.graph
+        ) {
+          let checkGraphServerTags =
+            await serviceConfig.getGraphServiceTagWithCache(
+              _izContext,
+              storageResources[eachStorageResourceTag].graphServerTag,
+              settings.bucketName
+            );
+          if (checkGraphServerTags) {
+            if (createDataDetails.hasOwnProperty(checkGraphServerTags)) {
+              // _izContext.logger.debug("SAME STG", checkGraphServerTags);
+              createDataDetails[checkGraphServerTags].fieldNames.push(
+                keyFieldName
+              );
+            } else {
+              // _izContext.logger.debug("NEW STG", checkGraphServerTags);
+              Object.assign(createDataDetails, {
+                [checkGraphServerTags]: {
+                  storageType: coreConsts.STORAGE_TYPES.graph,
+                  fieldNames: [keyFieldName]
+                }
+              });
+            }
+          }
+        } else if (
+          storageResources[eachStorageResourceTag].storageType ===
+          coreConsts.STORAGE_TYPES.externalTopic
+        ) {
+          let topicName =
+            storageResources[eachStorageResourceTag].serviceTag +
+            storageResources[eachStorageResourceTag].stage +
+            storageResources[eachStorageResourceTag].topicName;
+          Object.assign(createDataDetails, {
+            [topicName]: {
+              storageType: coreConsts.STORAGE_TYPES.externalTopic,
+              fieldNames: [keyFieldName],
+              accountId: storageResources[eachStorageResourceTag].accountId,
+              region: storageResources[eachStorageResourceTag].region,
+              serviceTag: storageResources[eachStorageResourceTag].serviceTag,
+              stage: storageResources[eachStorageResourceTag].stage
+            }
+          });
+        }
+      }
+    } // end loop storageResourceTags
+  } // end loop
+
+  return createDataDetails;
+}
+
+async function createDeleteDataDetail(
+  _izContext,
+  objectSchema,
+  settings = { bucketName: process.env.iz_serviceSchemaBucketName }
+) {
+  let deleteDynamoDataDetail = {};
+  let deleteGraphDataDetail = {};
+  let allDeleteDataDetail = {};
+
+  for (let [storageTag, storageProperties] of Object.entries(
+    objectSchema.storageResources
+  )) {
+    if (storageProperties.storageType === coreConsts.STORAGE_TYPES.dynamoDB) {
+      let serviceTag =
+        storageProperties.serviceTag || process.env.iz_serviceTag;
+      Object.assign(deleteDynamoDataDetail, {
+        [storageTag]: {
+          storageType: storageProperties.storageType,
+          serviceTag: serviceTag,
+          tableName: storageProperties.tableName
+        }
+      });
+      if (storageProperties.groupByPartitionKeyField) {
+        Object.assign(deleteDynamoDataDetail[storageTag], {
+          groupByPartitionKeyField: storageProperties.groupByPartitionKeyField
+        });
+      }
+    } else if (
+      storageProperties.storageType === coreConsts.STORAGE_TYPES.graph
+    ) {
+      let dataDetailPerGraphStorage = {};
+      let useStorageTag = storageTag;
+      let graphServiceName = await serviceConfig.getGraphServiceTagWithCache(
+        _izContext,
+        storageProperties.graphServerTag,
+        settings.bucketName
+      );
+      if (!dataDetailPerGraphStorage.hasOwnProperty(graphServiceName)) {
+        dataDetailPerGraphStorage[graphServiceName] = storageTag;
+      } else {
+        useStorageTag = dataDetailPerGraphStorage[graphServiceName];
+      }
+      if (!deleteGraphDataDetail.hasOwnProperty(useStorageTag)) {
+        deleteGraphDataDetail[graphServiceName] = {
+          storageType: storageProperties.storageType
+        };
+      }
+    }
+  }
+  _izContext.logger.debug('deleteGraphDataDetail', deleteGraphDataDetail);
+  _izContext.logger.debug('deleteDynamoDataDetail', deleteDynamoDataDetail);
+  Object.assign(
+    allDeleteDataDetail,
+    deleteGraphDataDetail,
+    deleteDynamoDataDetail
+  );
+
+  _izContext.logger.debug('allDeleteDataDetail', allDeleteDataDetail);
+  return allDeleteDataDetail;
+}
+
+
+export default {
+  createUpdateDataDetail,
+  createDataDetailsLib,
+  createDeleteDataDetail,
+
+  createGetDataDetails,
+  collectGetData,
+}