@izara_project/izara-market-library-service-schemas 1.0.85 → 1.0.86

package/src/reStructure/TemplateData/processLogicalPagination/handler/sqs/template.ejs

@@ -1,143 +0,0 @@
-/*
-Copyright (C) 2020 Sven Mason <http://izara.io>
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-
-
-import { middlewareHandler as middleware } from "@izara_project/izara-middleware";
-import Logger from '@izara_project/izara-core-library-logger';
-import { recordHandlerSharedLib } from "@izara_project/izara-core-library-record-handler";
-import callingFlowSharedLib from '@izara_project/izara-core-library-calling-flow';
-
-import paginateProcessLogical from './PaginateProcessLogical_Main.js';
-
-// validate event properties in body.Message of sqs event
-middleware.setValidatorSchema(recordHandlerSharedLib.baseValidatorSchema());
-
-// set schema for record.body.Message
-const perRecordsValidatorSchema = {
-  type: "object",
-  required: [
-    'logicalResultId',
-    'tableName',
-    'partitionKeyFieldName',
-    'sortKeyFieldName'
-  ],
-  properties: {
-    logicalResultId: {
-      type: "string"
-    },
-    tableName: {
-      type: "string"
-    },
-    partitionKeyFieldName: {
-      type: "string"
-    },
-    sortKeyFieldName: {
-      type: "string"
-    },
-    comparison: {
-      type: "string",
-      default: ""
-    },
-    startKey: {
-      type: "object",
-      default: {}
-    },
-    overwriteUniqueRequestId: {
-      type: "string",
-      default: ""
-    },
-    callingFlow: {
-      type: 'string',
-      default: ""
-    }
-  }
-};
-
-// set schema for record.body.MessageAttributes
-// const messageAttributeValidatorSchema = {
-//   type: "object",
-//   required: ['msgAtrrParam1', 'msgAtrrParam2'],
-//   properties: {
-//     msgAtrrParam1: {
-//       type: "string"
-//     },
-//     msgAtrrParam2: {
-//       type: "object"
-//     }
-//   }
-// };
-
-export const main = middleware.wrap(async (event, context, callback) => {
-
-  try {
-
-    let recordPromises = [];
-
-    // loop each record and send to mainFunction
-    await Promise.all(event.Records.map(async record => { // promise.all for map() function
-
-      let passOnProperties = []
-      record._izContext.logger.debug('record ReceiveMsgOutHdrSqs', record);
-
-      // validate message (and MessageAttributes)
-      await recordHandlerSharedLib.validateRecord(
-        record, // one record will send to mainFunction
-        "PaginateProcessLogicalHdrSqs", // queue name that need to retry or send to dlq
-        perRecordsValidatorSchema, // schema for record.Message
-        // messageAttributeValidatorSchema  // ----- for msgAttr default is null -> do not send this parameter if not want to validate msgAtt
-      );
-
-      // add argument (to invoke lambda) to passOnProperties[]
-      passOnProperties.push(record.body.Message.logicalResultId)
-      passOnProperties.push(record.body.Message.tableName)
-      passOnProperties.push(record.body.Message.partitionKeyFieldName)
-      passOnProperties.push(record.body.Message.sortKeyFieldName)
-      passOnProperties.push(record.body.Message.comparison)
-      passOnProperties.push(record.body.Message.startKey)
-      passOnProperties.push(record.body.Message.overwriteUniqueRequestId)
-      passOnProperties.push(record.body.Message.callingFlow)
-      record._izContext.logger.debug('passOnProperties in handler', passOnProperties);
-
-      // call recordHandlerSharedLib.recordHandler with 3 parameters and return promise(resolve)
-      let recordPromise = recordHandlerSharedLib.recordHandler(
-        record, // one record will send to mainFunction
-        paginateProcessLogical.paginateProcessLogical, // mainFunction that need to invoke.
-        "PaginateProcessLogicalHdrSqs", // queue name that need to retry or send to dlq
-        passOnProperties, // all parameters that mainFunction needed.
-      );
-      record._izContext.logger.debug('after recordPromise in handler');
-      recordPromises.push(recordPromise); // push promise to recordPromises
-    }))
-
-    Logger.debug('before Promise.all(recordPromises) in handler');
-    try {
-      // --- main await all promises
-      await Promise.all(recordPromises); // await all promises
-
-      return event.Records // return all for local testing
-
-    } catch {
-      Logger.debug('Promise.all(recordPromises) in handler threw error (at least one record did no resolve)');
-    }
-    Logger.debug('after Promise.all(recordPromises) in handler');
-
-  } catch (err) {
-    Logger.error('Unhandled Error, PaginateProcessLogicalHdrSqs: ', err);
-    throw (err);
-  }
-});

package/src/reStructure/TemplateData/processLogicalPagination/mainFunction/data.js

@@ -1,48 +0,0 @@
-/*
-Copyright (C) 2020 Sven Mason <http://izara.io>
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-import path from 'path';
-import { fileURLToPath } from 'url';
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-import { consts, utils } from '../../../../MainLibs/index.js';
-const { SOURCE_PATH, FUNCTION_NAME } = consts;
-const { firstLetterUpperCase: upperCase } = utils;
-const templatePath = path.join(__dirname, 'template.ejs');
-
-/**
- * receive srcPath
- * not create data for ProcessLogical Main template
- *
- * @param {Object} srcPath
- * @return {{ templatePath, templateData, setting } }
- */
-
-function data(_izContext, srcPath) {
-  return {
-    templatePath: templatePath,
-    templateData: {},
-    setting: {
-      savePath: path.join(srcPath, SOURCE_PATH.processLogical),
-      saveFileName: `${upperCase(FUNCTION_NAME.paginateProcessLogical)}_Main`,
-      fileExtension: '.js',
-      isAppend: false
-    }
-  };
-}
-
-export default data;

package/src/reStructure/TemplateData/processLogicalPagination/mainFunction/template.ejs

@@ -1,210 +0,0 @@
-/*
-Copyright (C) 2020 Sven Mason <http://izara.io>
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-
-
-import sqsSharedLib from '@izara_project/izara-core-library-sqs';
-import dynamodbSharedLib from '@izara_project/izara-core-library-dynamodb';
-import asyncFlowSharedLib from '@izara_project/izara-core-library-asynchronous-flow';
-
-import { sqs } from '@izara_project/izara-core-library-external-request';
-
-import { NoRetryError } from '@izara_project/izara-core-library-core';
-
-// set query limit
-const limit = 2;
-
-/**
- * performs the logical operation/filter/search in the element, saves into LogicalResultsData table
- * @param {Object} _izContext
- * @param {string} logicalResultId
- * @param {string} tableName
- * @param {string} partitionKeyFieldName
- * @param {string} sortKeyFieldName
- * @param {string} comparison
- * @param {Object} startKey
- * @param {string} overwriteUniqueRequestId
- * @param {string} callingFlow
- *
- * @returns {string} description of return value
- */
-
-export default async function paginateProcessLogical(
-  _izContext,
-  logicalResultId,
-  tableName,
-  partitionKeyFieldName,
-  sortKeyFieldName,
-  comparison, // optional
-  startKey, // optional
-  overwriteUniqueRequestId, // optional
-  callingFlow // optional
-) {
-
-  try {
-
-    _izContext.logger.debug('----- PaginateProcessLogical event params -----', {
-      logicalResultId: logicalResultId,
-      tableName: tableName,
-      partitionKeyFieldName: partitionKeyFieldName,
-      sortKeyFieldName: sortKeyFieldName,
-      comparison: comparison,
-      startKey: startKey,
-      overwriteUniqueRequestId: overwriteUniqueRequestId,
-      callingFlow: callingFlow
-    });
-
-    // Check/Create logicalResultsMain
-    let [checkUniqueRequestProcessingStatus, logicalResultsMain] = await asyncFlowSharedLib.checkUniqueRequestProcessing(
-      _izContext,
-      "LogicalResultsMain",
-      {
-        logicalResultId: logicalResultId
-      },
-      "paginateProcessLogical",
-      overwriteUniqueRequestId
-    );
-    _izContext.logger.debug("After check uniqueRequestProcessingStatus:", {
-      checkUniqueRequestProcessingStatus: checkUniqueRequestProcessingStatus,
-      logicalResultsMain: logicalResultsMain
-    });
-
-    if (checkUniqueRequestProcessingStatus == "stop") {
-      return;
-    } else if (checkUniqueRequestProcessingStatus == "recordNotFound") {
-      // should always have unless another request already completed it, in which case nothing to do
-      return;
-    }
-
-    let errorsFound = [];
-    let standardErrorParams = [
-      _izContext,
-      logicalResultId,
-      logicalResultsMain.logicalElement,
-      logicalResultsMain.objType,
-      "error"
-    ];
-
-    const partitionKeyValue = logicalResultsMain.logicalElement.partitionKey;
-    let queryElements = {
-      limit: limit
-    };
-
-    //if received startKey validate it, if not received will return null
-    startKey = asyncFlowSharedLib.validateStartKeyParam(
-      _izContext,
-      startKey,
-      partitionKeyFieldName,
-      sortKeyFieldName
-    );
-
-    if (startKey === false) {
-      throw new NoRetryError("split processing received invalid startKey");
-    }
-    _izContext.logger.debug("startKey :", startKey);
-
-    if (startKey) {
-      queryElements["exclusiveStartKey"] = {
-        [partitionKeyFieldName]: startKey[partitionKeyFieldName],
-        [sortKeyFieldName]: startKey[sortKeyFieldName]
-      }
-    };
-
-
-    if (comparison) {
-      const sortKeyValue = logicalResultsMain.logicalElement.sortKey;
-
-      queryElements["sortKeyCondition"] = {
-        comparison: comparison,
-        name: sortKeyFieldName,
-        value: sortKeyValue
-      };
-    }
-
-    let queryDatas = await dynamodbSharedLib.query(
-      _izContext,
-      await dynamodbSharedLib.tableName(_izContext, tableName),
-      {
-        [partitionKeyFieldName]: partitionKeyValue
-      },
-      queryElements
-    )
-    _izContext.logger.debug("queryDatas result: ", queryDatas)
-
-    if (!startKey && queryDatas.Items.length == 0) {
-      _izContext.logger.debug(`can't find data in table ${tableName} by query`)
-      errorsFound.push(`can't find data in table ${tableName} by query`)
-      standardErrorParams.push(errorsFound)
-      await logicalResultsSharedLib.completeLogicalResultsMain(...standardErrorParams);
-      return;
-    }
-
-    for (const objInstanceBase of queryDatas.Items) {
-
-      await logicalResultsSharedLib.saveLogicalData(
-        _izContext,
-        logicalResultId,
-        logicalResultsMain.objType,
-        objInstanceBase
-      )
-    }
-
-    if (queryDatas.hasOwnProperty("LastEvaluatedKey")) {
-
-      if (!overwriteUniqueRequestId) {
-        overwriteUniqueRequestId = _izContext.uniqueRequestId
-      };
-
-      let continuePaginationMsg = {
-        MessageBody: JSON.stringify({
-          logicalResultId: logicalResultId,
-          tableName: tableName,
-          partitionKeyFieldName: partitionKeyFieldName,
-          sortKeyFieldName: sortKeyFieldName,
-          comparison: comparison,
-          overwriteUniqueRequestId: overwriteUniqueRequestId,
-          startKey: queryDatas.LastEvaluatedKey,
-          callingFlow: callingFlow
-        }),
-        QueueUrl: await sqsSharedLib.sqsQueueUrl(_izContext, "PaginateProcessLogicalHdrDsq"),
-      };
-
-      _izContext.logger.debug("continuePaginationMessage: ", continuePaginationMsg);
-      await sqs.sendMessage(_izContext, continuePaginationMsg);
-
-      return "Continue PaginateProcessLogical"
-
-    } else {
-
-      await logicalResultsSharedLib.completeLogicalResultsMain(
-        _izContext,
-        logicalResultId,
-        logicalResultsMain.logicalElement,
-        logicalResultsMain.objType,
-        "complete",
-        [],
-        callingFlow
-      )
-    }
-
-    return "Function PaginateProcessLogical is Finished"
-
-  } catch (err) {
-    _izContext.logger.error('error PaginateProcessLogicalMain: ', err);
-    throw (err)
-  }
-}

package/src/reStructure/TemplateData/processLogicalPagination/yaml/dsq/data.js

@@ -1,162 +0,0 @@
-/*
-Copyright (C) 2020 Sven Mason <http://izara.io>
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as
-published by the Free Software Foundation, either version 3 of the
-License, or (at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-import path from 'path';
-import { fileURLToPath } from 'url';
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-import { consts, utils } from '../../../../../MainLibs/index.js';
-
-const {
-  createIamRole,
-  resourceNames,
-  RESOURCE_CLASSES,
-  DEFAULT_LAMBDA_ROLE_PER_ACTION,
-  SQS_RESOURCE,
-  SOURCE_PATH,
-  SAVE_FILE_NAME,
-  FUNCTION_NAME,
-  defaultIamRolePerAction,
-  shortNameHandler,
-  SHORT_FUNCTION_NAME,
-  SOURCE_GENERATE_IAM_ROLE,
-  HANDLER,
-  STORAGE_TYPES,
-  ACTIONS
-} = consts;
-
-const { firstLetterUpperCase: upperCase } = utils;
-
-const templatePath = path.join(__dirname, 'template.ejs');
-
-/**
- * receive objectSchema
- * create data for FindData Main template
- *
- * @param {Object} objectSchemas
- * @return {{ templatePath, templateData, setting } }
- */
-async function data(_izContext, objectSchemas, srcPath) {
-  let additionalResourcePermission = defaultIamRolePerAction();
-  let functionName = upperCase(FUNCTION_NAME.paginateProcessLogical);
-  let tableForQuery = [];
-
-  for (let objectSchema of objectSchemas.records) {
-    // console.log("objectSchema", objectSchemas)
-
-    for (let storageResource of Object.values(objectSchema.storageResources)) {
-      if (storageResource.storageType === STORAGE_TYPES.dynamoDB) {
-        // add IamRole for dynamodb depend on tableName in storageResources of objectSchema
-        tableForQuery.push(
-          resourceNames(
-            RESOURCE_CLASSES.dynamoDbTable,
-            upperCase(storageResource.tableName)
-          )
-        );
-      }
-    }
-  }
-
-  if (tableForQuery.length) {
-    additionalResourcePermission.push(
-      createIamRole(
-        { [RESOURCE_CLASSES.dynamoDbTable]: 'Query' },
-        tableForQuery
-      )
-    );
-  }
-
-  additionalResourcePermission.push(
-    createIamRole(
-      {
-        [RESOURCE_CLASSES.sqs]: [
-          SQS_RESOURCE.deleteMessage,
-          SQS_RESOURCE.getQueueAttributes,
-          SQS_RESOURCE.receiveMessage,
-          SQS_RESOURCE.sendMessage,
-          SQS_RESOURCE.deleteMessageBatch,
-          SQS_RESOURCE.getQueueUrl
-        ]
-      },
-      [
-        resourceNames(
-          RESOURCE_CLASSES.sqs,
-          functionName + upperCase(HANDLER.hdrDsq)
-        ),
-        resourceNames(
-          RESOURCE_CLASSES.sqs,
-          functionName + upperCase(HANDLER.hdrDsq) + 'DLQ'
-        )
-      ]
-    )
-  );
-
-  additionalResourcePermission.push(
-    createIamRole(
-      {
-        [RESOURCE_CLASSES.sns]: ['Publish']
-      },
-      [resourceNames(RESOURCE_CLASSES.sns, 'ProcessLogicalComplete_Out')]
-    )
-  );
-
-  additionalResourcePermission.push(
-    createIamRole(
-      {
-        [RESOURCE_CLASSES.dynamoDbTable]: [
-          DEFAULT_LAMBDA_ROLE_PER_ACTION[ACTIONS.get].dynamoDbPermission,
-          DEFAULT_LAMBDA_ROLE_PER_ACTION[ACTIONS.update].dynamoDbPermission,
-          'Query'
-        ]
-      },
-      [resourceNames(RESOURCE_CLASSES.dynamoDbTable, 'LogicalResultsMain')]
-    )
-  );
-
-  additionalResourcePermission.push(
-    createIamRole(
-      {
-        [RESOURCE_CLASSES.dynamoDbTable]: [
-          DEFAULT_LAMBDA_ROLE_PER_ACTION[ACTIONS.create].dynamoDbPermission
-        ]
-      },
-      [resourceNames(RESOURCE_CLASSES.dynamoDbTable, 'LogicalResultsData')]
-    )
-  );
-
-  return {
-    templatePath: templatePath,
-    templateData: {
-      functionName: functionName,
-      resourceLocation: SOURCE_PATH.resourceLocationProcessLogical,
-      handlerType: HANDLER.hdrDsq,
-      additionalResourcePermission,
-      functionNameConfig:
-        upperCase(SHORT_FUNCTION_NAME(FUNCTION_NAME.paginateProcessLogical)) +
-        upperCase(shortNameHandler(HANDLER.hdrDsq)),
-      roleName: SOURCE_GENERATE_IAM_ROLE.ProcessFindDataRole
-    },
-    setting: {
-      savePath: path.join(srcPath, SOURCE_PATH.appYaml),
-      saveFileName: upperCase(SAVE_FILE_NAME.processLogicalYaml),
-      fileExtension: '.yml',
-      isAppend: true
-    }
-  };
-}
-
-export default data;

package/src/reStructure/TemplateData/processLogicalPagination/yaml/dsq/template.ejs

@@ -1,12 +0,0 @@
-<%_ const join = require('path').join; _%>
-<%- firstLetterUpperCase(functionNameConfig) %>:
-  handler: <%- join(resourceLocation, `${functionName}_${firstLetterUpperCase(handlerType)}.main`)%>
-  name: ${self:custom.iz_resourcePrefix}<%- functionName %><%- firstLetterUpperCase(handlerType) %>
-  role: <%- roleName %>Role
-  #<#<%- functionName %><%- handlerType %>FunctionSetting#>
-  #<#/<%- functionName %><%- handlerType %>FunctionSetting#>
-  events:
-    - sqs:
-        arn: arn:aws:sqs:${self:custom.iz_region}:${self:custom.iz_accountId}:${self:custom.iz_resourcePrefix}<%- functionName %><%- firstLetterUpperCase(handlerType) %>
-        batchSize: 10
-<%_ function firstLetterUpperCase(text){ return text.charAt(0).toUpperCase() + text.slice(1) } _%>