@izara_project/izara-core-library-find-data 1.0.1 → 1.0.2

package/index.js

@@ -0,0 +1,42 @@
+/*
+Copyright (C) 2021 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict';
+
+// Import the Find Data shared library functions
+const findDataSharedLib = require('./src/FindDataSharedLib');
+
+// Export all the functions
+module.exports = {
+  // Find Data ID Creation
+  createFindDataId: findDataSharedLib.createFindDataId,
+
+  // Find Data Status Management
+  checkFindDataMainStatus: findDataSharedLib.checkFindDataMainStatus,
+  completeFindDataMain: findDataSharedLib.completeFindDataMain,
+  sendFindDataCompleteMessage: findDataSharedLib.sendFindDataCompleteMessage,
+
+  // Awaiting Steps Management
+  createFindDataAwaitingStep: findDataSharedLib.createFindDataAwaitingStep,
+  findFindDataIdsAwaitingStep: findDataSharedLib.findFindDataIdsAwaitingStep,
+  removeFindDataAwaitingStep: findDataSharedLib.removeFindDataAwaitingStep,
+
+  // Sorted Requests Management
+  createFindDataSortedRequest: findDataSharedLib.createFindDataSortedRequest,
+  findFindDataSortedRequests: findDataSharedLib.findFindDataSortedRequests,
+  removeFindDataSortedRequest: findDataSharedLib.removeFindDataSortedRequest
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@izara_project/izara-core-library-find-data",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "description": "Shared find data logic",
   "main": "index.js",
   "scripts": {
@@ -20,9 +20,8 @@
     "testEnvironment": "node"
   },
   "dependencies": {
+    "@izara_project/izara-core-library-core": "^1.0.15",
     "@izara_project/izara-core-library-external-request": "^1.0.17",
-    "@izara_project/izara-core-library-service-schemas": "^1.0.38",
-    "@izara_project/izara-shared": "^1.0.121",
-    "aws-jwt-verify": "^4.0.1"
+    "object-hash": "^3.0.0"
   }
-}
+}

package/src/FindDataSharedLib.js

@@ -0,0 +1,425 @@
+/*
+Copyright (C) 2021 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+'use strict';
+
+// External library imports
+const hash = require('object-hash');
+
+// Izara core libraries
+const { NoRetryError } = require('@izara_project/izara-core-library-core/src/NoRetryError');
+const { sns } = require('@izara_project/izara-core-library-external-request');
+const asyncFlowSharedLib = require("@izara_project/izara-core-library-asynchronous-flow");
+const snsSharedLib = require('@izara_project/izara-core-library-sns');
+const storedCacheSharedLib = require("@izara_project/izara-core-library-stored-cache");
+
+// Commented out libraries that are not currently used
+// const Logger = require('@izara_project/izara-core-library-logger');
+// const dynamodbSharedLib = require('./DynamodbSharedLib');
+
+
+module.exports.createFindDataId = (
+  fieldName,
+  objType,
+  identifiers,
+  requestProperties = {}
+) => {
+  return hash({
+    fieldName: fieldName,
+    objType: objType,
+    identifiers: identifiers,
+    requestProperties: requestProperties
+  });
+}
+
+
+/**
+ * Checks if existing FindDataMain exists, if it does check expired? if yes reset to processing
+ * @param {string} fieldName
+ * @param {string} objType
+ * @param {object} identifiers
+ *
+ * @returns {string} status "process"|"processing"|"complete" ("process" means requesting function needs to continue processing)
+ * if unable to reset expired record status to processing (because other thread changing record) throw NoRetryError
+ */
+module.exports.checkFindDataMainStatus = async (
+  _izContext,
+  fieldName,
+  objType,
+  identifiers,
+  requestProperties,
+  callingFlow = "",
+  removeAttributes = []
+) => {
+  try {
+
+    const findDataId = this.createFindDataId(
+      fieldName,
+      objType,
+      identifiers,
+      requestProperties
+    );
+
+    let [cacheStatus, findDataMain] = await storedCacheSharedLib.checkStoredCacheStatus(
+      _izContext,
+      "FindDataMain", //dynamodbSharedLib.tableName('FindDataMain'), // ".. build table name from serviceName + FindDataMain",
+      { // keyValues
+        findDataId: findDataId
+      },
+      { // additionalAttributesWhenCreate
+        attributes: {
+          fieldName: fieldName,
+          objType: objType,
+          identifiers: identifiers,
+          requestProperties: requestProperties
+        }
+      },
+      'findData', // prefix
+      removeAttributes // removeAttributes
+    );
+    if (cacheStatus !== "process") {
+      // is either complete, processing by another thread, error, or some other status
+      // means this request should not continue to process the request
+
+      if (cacheStatus === "complete" || cacheStatus === "error") {
+
+        if (callingFlow) {
+          // send message to OutProcessLogical for complexFilter service or other calling flow knows this logical element is complete
+          await this.sendFindDataCompleteMessage(
+            _izContext,
+            fieldName,
+            objType,
+            identifiers,
+            requestProperties,
+            findDataMain.dataValue,
+            cacheStatus,
+            callingFlow
+          );
+        }
+      }
+    }
+
+    return [cacheStatus, findDataId, findDataMain];
+
+  } catch (e) {
+    _izContext.logger.error('ERROR checkFindDataMainStatus:', e)
+    throw (e)
+  }
+};
+
+/**
+  * send message to OutFindDataComplete for searchResult service to know this requiredData is complete, and update FindDataMain
+  * @param {string} fieldName
+  * @param {object} objType
+ * @param {object} identifiers
+ *
+ */
+module.exports.completeFindDataMain = async (
+  _izContext,
+  fieldName,
+  objType,
+  identifiers,
+  requestProperties,
+  dataValue,
+  status,
+  errorsFound = []
+) => {
+
+  //* validate status
+  if (status !== 'error' && status !== 'complete') {
+    throw new NoRetryError(`status: ${status} is not valided.`)
+  }
+
+  let timeCacheComplete = Date.now()
+
+  // send out message first for idempotence
+  await this.sendFindDataCompleteMessage(
+    _izContext,
+    fieldName,
+    objType,
+    identifiers,
+    requestProperties,
+    dataValue,
+    status,
+    "",
+    errorsFound,
+    timeCacheComplete
+  );
+
+  //* update FindDataMain record set status = complete, dataValue = dataValue, expiryTime = ..., remove uniqueRequestId if exists
+  //* status = complete|error, if errorsFound has elements add as well
+  const findDataId = this.createFindDataId(
+    fieldName,
+    objType,
+    identifiers,
+    requestProperties
+  );
+
+  await storedCacheSharedLib.completeStoredCache(
+    _izContext,
+    "FindDataMain",
+    { // keyValues
+      findDataId: findDataId
+    },
+    status,
+    { // additionalAttributes
+      attributes: {
+        dataValue: dataValue
+      }
+    },
+    'findData', // prefix
+    null, // expiryInterval
+    timeCacheComplete,
+    errorsFound
+  );
+
+  // const prefix = 'findData'
+  // let updateAttributes = [
+  //   {
+  //     attributeName: prefix + 'Status', // TOCHECK : prefix + "status" ??? same like cache
+  //     action: "set",
+  //     value: status,
+  //   },
+  //   {
+  //     attributeName: prefix + 'ExpiryTime',
+  //     action: "set",
+  //     value: Date.now() + expiryInterval,
+  //   },
+  //   {
+  //     attributeName: 'dataValue',
+  //     action: "set",
+  //     value: value,
+  //   },
+  //   {
+  //     attributeName: prefix + 'UniqueRequestId',
+  //     action: "remove",
+  //   },
+  // ]
+
+  // if (status == 'error' && errorsFound.length > 0) {
+  //   updateAttributes.push(
+  //     {
+  //       attributeName: prefix + 'ErrorsFound',
+  //       action: "set",
+  //       value: errorsFound,
+  //       listAppend: true
+  //     }
+  //   )
+  // }
+
+  // await dynamodbSharedLib.updateItem(
+  //   _izContext,
+  //   dynamodbSharedLib.tableName('FindDataMain'),
+  //   { [fieldName]: findDataId }, // keyValues
+  //   updateAttributes,
+  //   {}, // condition
+  //   { complexAttributes: true }
+  // )
+
+};
+
+/**
+  * send message to OutFindDataComplete for searchResult service to know this requiredData is complete
+  * @param {string} fieldName
+  * @param {object} objType
+ * @param {object} identifiers
+ *
+ */
+module.exports.sendFindDataCompleteMessage = async (
+  _izContext,
+  fieldName,
+  objType,
+  identifiers,
+  requestProperties,
+  dataValue,
+  status,
+  callingFlow = "",
+  errorsFound = [],
+  timeCacheComplete = 0
+) => {
+
+  //* validate status
+  if (status !== 'error' && status !== 'complete') {
+    throw new NoRetryError(`status: ${status} is not valided.`)
+  }
+
+  let msgParams = {
+    Message: JSON.stringify({
+      fieldName: fieldName,
+      objType: objType,
+      identifiers: identifiers,
+      requestProperties: requestProperties,
+      dataValue: dataValue,
+      status: status,
+      errorsFound: errorsFound,
+      timeCacheComplete: timeCacheComplete
+    }),
+    TopicArn: await snsSharedLib.snsTopicArn(_izContext, 'OutFindDataComplete')
+  };
+  if (callingFlow) {
+    msgParams.MessageAttributes = {
+      callingFlow: {
+        DataType: 'String',
+        StringValue: callingFlow
+      }
+    };
+  }
+  _izContext.logger.debug('msgParams before send to OutFindDataComplete topic...', msgParams);
+  await sns.publishAsync(_izContext, msgParams);
+
+};
+
+
+
+module.exports.createFindDataAwaitingStep = async (
+  _izContext,
+  awaitingId,
+  findDataId
+) => {
+  //* insert into FindDataAwaitingStep
+  // call shared function
+  await asyncFlowSharedLib.createAwaitingStep(
+    _izContext,
+    awaitingId,
+    findDataId,
+    'FindDataAwaitingStep'
+  )
+
+}
+
+/**
+ * Finds any FindDataAwaitingStep matching awaitingId
+ * @param {string} awaitingId
+ *
+ * @returns {string[]} array of resultIds
+ */
+module.exports.findFindDataIdsAwaitingStep = async (
+  _izContext,
+  awaitingId
+) => {
+  try {
+
+    return await asyncFlowSharedLib.findPendingStepIdsAwaitingStep(
+      _izContext,
+      awaitingId,
+      'FindDataAwaitingStep'
+    )
+
+  } catch (err) {
+    _izContext.logger.error('error findFindDataAwaitingStep:', err)
+    throw (err)
+  }
+};
+
+/**
+ * deletes one FindDataAwaitingStep record
+ * @param {string} awaitingId
+ * @param {string} findDataId
+ *
+ * @returns {string[]} array of resultIds
+ */
+module.exports.removeFindDataAwaitingStep = async (
+  _izContext,
+  awaitingId,
+  findDataId
+) => {
+  try {
+
+    // call shared fn
+    await asyncFlowSharedLib.removeAwaitingStep(
+      _izContext,
+      awaitingId,
+      findDataId,
+      'FindDataAwaitingStep'
+    )
+
+  } catch (err) {
+    _izContext.logger.error('error removeLogicalAwaitingStep:', err)
+    throw (err)
+  }
+};
+
+
+module.exports.createFindDataSortedRequest = async (
+  _izContext,
+  sortResultId,
+  findDataId,
+  aggregation,
+  excludeEmpty,
+  prefix = ""
+) => {
+
+  // NOTE (Tam 14 Dec 2021): new idea to standardized table, put to same FindDataAwaitingStep, shared table and function.
+  await asyncFlowSharedLib.createAwaitingStep(
+    _izContext,
+    asyncFlowSharedLib.createAwaitingStepId(sortResultId, prefix),
+    findDataId, // pendingStepId
+    'FindDataAwaitingStep',
+    { //additionalAttributes
+      aggregation: aggregation,
+      excludeEmpty: excludeEmpty
+    }
+  );
+
+}
+
+module.exports.findFindDataSortedRequests = async (
+  _izContext,
+  sortResultId,
+  prefix = ""
+) => {
+  try {
+
+    return await asyncFlowSharedLib.findPendingStepIdsAwaitingStep(
+      _izContext,
+      asyncFlowSharedLib.createAwaitingStepId(sortResultId, prefix),
+      'FindDataAwaitingStep'
+    )
+
+
+  } catch (err) {
+    _izContext.logger.error('ERROR findFindDataSortedRequests:', err)
+    throw (err)
+  }
+};
+
+/**
+ * deletes one FindDataSortedRequest record
+ * @param {string} findDataId
+ *
+ * @returns {string[]} array of resultIds
+ */
+module.exports.removeFindDataSortedRequest = async (
+  _izContext,
+  sortResultId,
+  findDataId,
+  prefix = ""
+) => {
+  try {
+
+    await asyncFlowSharedLib.removeAwaitingStep(
+      _izContext,
+      asyncFlowSharedLib.createAwaitingStepId(sortResultId, prefix),
+      findDataId, //pendingStepId
+      'FindDataAwaitingStep'
+    )
+
+  } catch (err) {
+    _izContext.logger.debug('ERROR removeFindDataSortedRequest:', err)
+    throw (err)
+  }
+};