@izara_project/izara-core-library-asynchronous-flow 1.0.20 → 1.0.21

package/index.js

@@ -15,10 +15,14 @@ You should have received a copy of the GNU Affero General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-'use strict';
-
 // Import the Asynchronous Flow shared library functions
-import asyncFlowSharedLib from './src/AsyncFlowSharedLib.js';
+import * as asyncFlowSharedLib from './src/asyncFlowSharedLib.js';
+import * as awaitingStep from './src/awaitingStep.js';
+import * as awaitingMultipleSteps from './src/awaitingMultipleSteps.js';
 
-// Export all the functions
-export default asyncFlowSharedLib;
+// Combine all functions from the imported modules into a single export object.
+export default {
+  ...asyncFlowSharedLib,
+  ...awaitingStep,
+  ...awaitingMultipleSteps
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@izara_project/izara-core-library-asynchronous-flow",
-  "version": "1.0.20",
+  "version": "1.0.21",
   "description": "Shared asynchronous flow logic",
   "main": "index.js",
   "scripts": {

package/src/asyncFlowSharedLib.js

@@ -0,0 +1,507 @@
+/*
+Copyright (C) 2021 Sven Mason <http://izara.io>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+// Core libraries
+import { NoRetryError } from '@izara_project/izara-core-library-core';
+import Logger from '@izara_project/izara-core-library-logger';
+
+// Data and service interaction libraries
+import dynamodbSharedLib from '@izara_project/izara-core-library-dynamodb';
+import sqsSharedLib from '@izara_project/izara-core-library-sqs';
+
+// External service interactions
+import { sqs } from '@izara_project/izara-core-library-external-request';
+
+/**
+ * Create a field name for storing the unique-request id, with optional prefix.
+ * @param {string} prefix
+ * @param {string} [uniqueRequestIdFieldName='UniqueRequestId']
+ * @returns {string}
+ */
+export function createFieldNameUniqueRequestId(
+  prefix,
+  uniqueRequestIdFieldName = 'UniqueRequestId'
+) {
+  return prefix + uniqueRequestIdFieldName;
+}
+
+/**
+ * Create a concatenated awaitingStepId from composite parts.
+ * @param {string} partitionKey
+ * @param {string|number} sortId
+ * @param {string} [prefix=""]
+ * @returns {string}
+ */
+export function createConcatenatedAwaitingStepId(
+  partitionKey,
+  sortId,
+  prefix = ''
+) {
+  return prefix + partitionKey + '_' + sortId;
+}
+
+/**
+ * Create a concatenated pendingStepId from parent/child ids.
+ * @param {string} parentId
+ * @param {string} childId
+ * @returns {string}
+ */
+export function createConcatenatedPendingStepId(parentId, childId) {
+  return parentId + '_' + childId;
+}
+
+/**
+ * Create a simple awaitingStepId (prefix + partitionKey).
+ * @param {string} partitionKey
+ * @param {string} [prefix=""]
+ * @returns {string}
+ */
+export function createAwaitingStepId(partitionKey, prefix = '') {
+  return prefix + partitionKey;
+}
+
+/**
+ * Create a simple pendingStepId (prefix + identifierId).
+ * @param {string} identifierId
+ * @param {string} [prefix=""]
+ * @returns {string}
+ */
+export function createPendingStepId(identifierId, prefix = '') {
+  return prefix + identifierId;
+}
+
+/**
+ * Remove the known prefix from a pendingStepId and validate it is not equal to the prefix.
+ * If prefix is empty, the original ID is returned.
+ * @param {string} pendingStepId
+ * @param {string} [prefix=""]
+ * @returns {string}
+ * @throws {NoRetryError} If the stripped value equals the prefix (invalid)
+ */
+export function explodePendingStepId(pendingStepId, prefix = '') {
+  if (prefix === '') {
+    return pendingStepId;
+  } else {
+    let identifierId = pendingStepId.slice(prefix.length);
+    // ** validate identifierId == prefix
+    if (identifierId == prefix) {
+      throw new NoRetryError('IdentifierId should not be like prefix.');
+    }
+
+    Logger.debug('return explode:', identifierId);
+    return identifierId;
+  }
+}
+
+// copy from izara-core-library-trigger-cache
+/**
+ * Build the field name that stores "cache complete" marker.
+ * @param {string} [prefix='cache']
+ * @returns {string}
+ */
+export function createFieldNameCacheComplete(prefix = 'cache') {
+  return prefix + 'CacheComplete';
+}
+
+/**
+ * Build the field name that stores a cache "status".
+ * @param {string} [prefix='cache']
+ * @returns {string}
+ */
+export function createFieldNameStatus(prefix = 'cache') {
+  return prefix + 'Status';
+}
+
+/**
+ * Checks if record has uniqueRequestId set, if not set to this requests uniqueRequestId and continue to process
+ * if already set check if matches this request uniqueRequestId, if yes continue to process, if not then stop processing
+ * status: "process"|"stop"|"recordNotFound"
+
+ * if unable to set uniqueRequestId (because other thread updated it) throw NoRetryError
+ * conditional update existingRecord set uniqueRequestId = _izContext.uniqueRequestId
+ * conditional check uniqueRequestId still not exist in case another thread added
+ * if conditional not pass "continue" and try again
+
+ * @returns [string] [returnStatus, existingRecord]
+ */
+
+/**
+ * Coordinate unique-request processing over a record:
+ * - If no record: returns ["recordNotFound", {}]
+ * - If record exists but no uniqueRequestId: tries to set it via conditional update
+ * - If record has a different uniqueRequestId: returns ["stop", existingRecord]
+ * - If record has the same uniqueRequestId: returns ["process", existingRecord]
+ *
+ * Will loop up to 2 attempts to avoid races; throws on 3rd iteration.
+ *
+ * @async
+ * @param {IzContext} _izContext
+ * @param {string} tableName - Logical table name known by dynamodbSharedLib.tableName
+ * @param {DynamoKey} primaryKey - Primary key for the target record
+ * @param {string} [prefix=''] - Optional field-name prefix
+ * @param {string|null} [overwriteUniqueRequestId=null] - If provided, use this as the "current" unique request id
+ * @param {string} [uniqueRequestIdFieldName="UniqueRequestId"] - Base field name before prefix
+ * @returns {Promise<[UniqueRequestStatus, Attrs]>}
+ * @throws {NoRetryError} If unable to set uniqueRequestId after 2 tries
+ */
+export async function checkUniqueRequestProcessing(
+  _izContext,
+  tableName,
+  primaryKey,
+  prefix = '',
+  overwriteUniqueRequestId = null,
+  uniqueRequestIdFieldName = 'UniqueRequestId' // if set will check/add this fieldname with _izContext.uniqueRequestId value
+) {
+  try {
+    _izContext.logger.debug(
+      'current uniqueRequestId:',
+      _izContext.uniqueRequestId
+    );
+
+    let uniqueRequestId = _izContext.uniqueRequestId;
+    if (overwriteUniqueRequestId) {
+      uniqueRequestId = overwriteUniqueRequestId;
+    }
+
+    // add prefix
+    uniqueRequestIdFieldName = createFieldNameUniqueRequestId(
+      prefix,
+      uniqueRequestIdFieldName
+    );
+
+    //loop 3 times incase status changes, on 3rd iteration throw error (so checks 2 times)
+    for (let i = 1; i <= 3; i++) {
+      if (i == 3) {
+        throw new NoRetryError(
+          `unable to set uniqueRequestId in table ${tableName}, record`,
+          primaryKey
+        );
+      }
+      //* get record from dynamo
+      let existingRecord = await dynamodbSharedLib.getItem(
+        _izContext,
+        await dynamodbSharedLib.tableName(_izContext, tableName),
+        primaryKey
+      );
+      _izContext.logger.debug('existingRecord:', existingRecord);
+
+      let returnStatus = 'process';
+
+      if (!existingRecord) {
+        // return ["recordNotFound", null]; // cannot return null, js cannot found object and throw error and dont know where it is?
+        return ['recordNotFound', {}];
+      } else if (!existingRecord[uniqueRequestIdFieldName]) {
+        // uniqueRequestId not yet exist
+
+        let updateAttributes = [
+          {
+            attributeName: uniqueRequestIdFieldName,
+            action: 'set',
+            value: _izContext.uniqueRequestId
+          }
+        ];
+        //* conditional check uniqueRequestId still not exist in case another thread added
+        const queryElementsWhenUpdate = {
+          logicalElements: [
+            {
+              type: 'function',
+              function: 'attribute_not_exists',
+              attribute: uniqueRequestIdFieldName
+            }
+          ],
+          returnValues: 'ALL_NEW'
+        };
+        // // ----- main query ---------
+        let updateRecord = null;
+        try {
+          updateRecord = await dynamodbSharedLib.updateItem(
+            _izContext,
+            await dynamodbSharedLib.tableName(_izContext, tableName),
+            primaryKey,
+            updateAttributes,
+            queryElementsWhenUpdate,
+            { complexAttributes: true } // force to complex
+          );
+
+          return [returnStatus, updateRecord];
+        } catch (err) {
+          // catch error when put and handle errors.
+          _izContext.logger.debug('updateItem storedCache expired err', err);
+          if (err.name == 'ConditionalCheckFailedException') {
+            continue;
+          } else {
+            // e.g. ProvisionedThroughputExceededException
+            // throw new Error('Unhandled Error when putItem', err) // TT, if throw will stop
+            throw err; // TT, if throw will stop
+          }
+        } //end catch err
+      } else if (existingRecord[uniqueRequestIdFieldName] !== uniqueRequestId) {
+        // when another request/uniqueRequestId need to stop process
+        return ['stop', existingRecord];
+      } else if (existingRecord[uniqueRequestIdFieldName] == uniqueRequestId) {
+        // if existingRecord[uniqueRequestIdFieldName] == uniqueRequestId then return "process
+        return [returnStatus, existingRecord];
+      }
+    } // end loop 2 times
+    //should never get here
+    throw new NoRetryError(
+      'unexpected logic flow in checkUniqueRequestProcessing'
+    );
+  } catch (err) {
+    _izContext.logger.error('ERROR checkUniqueRequestProcessing:', err);
+    throw err;
+  }
+}
+
+// ----- Helper funtion, common use case in triggerFlow ---------
+/**
+ * Convenience wrapper around checkAndGetTimeCacheComplete.
+ * Returns [isSameOrUnset, uniqueRequestIdWhenComplete].
+ * @async
+ * @param {IzContext} _izContext
+ * @param {string} fullMainTableName - Fully resolved table name (not logical)
+ * @param {DynamoKey} keyValues
+ * @param {number|string} timeCacheComplete - Caller’s expected cache mark
+ * @param {string} prefix - Field prefix used in cache fields
+ * @returns {Promise<[boolean, string|undefined]>}
+ */
+export async function checkTimeCacheComplete(
+  _izContext,
+  fullMainTableName,
+  keyValues,
+  timeCacheComplete,
+  prefix
+) {
+  let [returnValue, newTimeCacheComplete, uniqueRequestId] =
+    await checkAndGetTimeCacheComplete(
+      _izContext,
+      fullMainTableName,
+      keyValues,
+      timeCacheComplete,
+      prefix
+    );
+  return [returnValue, uniqueRequestId];
+}
+
+/**
+ * Read cache fields from a record and compare cacheComplete value to caller’s `timeCacheComplete`.
+ * If stored differs (and is present), returns [false, storedValue].
+ * Otherwise returns [true, storedValue, uniqueRequestId].
+ * @async
+ * @param {IzContext} _izContext
+ * @param {string} fullMainTableName - Fully resolved table name (not logical)
+ * @param {DynamoKey} keyValues
+ * @param {number|string} timeCacheComplete
+ * @param {string} prefix
+ * @returns {Promise<[boolean, any, string|undefined]>}
+ * @throws {NoRetryError} If record cannot be fetched
+ */
+export async function checkAndGetTimeCacheComplete(
+  _izContext,
+  fullMainTableName,
+  keyValues,
+  timeCacheComplete,
+  prefix
+) {
+  _izContext.logger.debug('event checkTimeCacheComplete', {
+    fullMainTableName: fullMainTableName,
+    keyValues: keyValues,
+    timeCacheComplete: timeCacheComplete,
+    prefix: prefix
+  });
+
+  let uniqueRequestIdFieldName = createFieldNameUniqueRequestId('cache');
+  let cacheCompleteFieldName = createFieldNameCacheComplete(prefix);
+  let statusFieldName = createFieldNameStatus(prefix);
+
+  let getTimeCacheComplete = await dynamodbSharedLib.getItem(
+    _izContext,
+    fullMainTableName,
+    keyValues
+  );
+  _izContext.logger.debug(
+    `getTimeCacheComplete from ${fullMainTableName}: `,
+    getTimeCacheComplete
+  );
+
+  if (!getTimeCacheComplete) {
+    throw new NoRetryError(
+      `cannot getTimeCacheComplete from ${fullMainTableName}`
+    );
+  }
+
+  // check if exist and match with initail create not changing in flow.
+  // if (!getTimeCacheComplete[uniqueRequestIdFieldName]
+  //   // || !getTimeCacheComplete[cacheCompleteFieldName]
+  //   || (getTimeCacheComplete[cacheCompleteFieldName]
+  //     && getTimeCacheComplete[cacheCompleteFieldName] !== timeCacheComplete
+  //     && getTimeCacheComplete[statusFieldName] != "complete")
+  // ) {
+  //   return [false, getTimeCacheComplete[cacheCompleteFieldName]];
+  // } else {
+  //   return [true, getTimeCacheComplete[cacheCompleteFieldName], getTimeCacheComplete[uniqueRequestIdFieldName]];
+  // }
+
+  if (
+    getTimeCacheComplete[cacheCompleteFieldName] &&
+    getTimeCacheComplete[cacheCompleteFieldName] !== timeCacheComplete
+  ) {
+    return [false, getTimeCacheComplete[cacheCompleteFieldName]];
+  } else {
+    return [
+      true,
+      getTimeCacheComplete[cacheCompleteFieldName],
+      getTimeCacheComplete[uniqueRequestIdFieldName]
+    ];
+  }
+}
+
+// ----- shared both stored cache and triggered cache, check uniqueRequestId changed ---------
+/**
+ * Ensure the cache object’s "uniqueRequestIdCompleteFieldName" equals the given checkUniqueRequestId.
+ * @async
+ * @param {IzContext} _izContext
+ * @param {string} fullMainTableName
+ * @param {DynamoKey} keyValues
+ * @param {string} checkUniqueRequestId
+ * @param {string} uniqueRequestIdCompleteFieldName
+ * @returns {Promise<boolean>}
+ */
+export async function checkCacheUniqueRequestId(
+  _izContext,
+  fullMainTableName,
+  keyValues,
+  checkUniqueRequestId,
+  uniqueRequestIdCompleteFieldName
+) {
+  let cacheObject = await dynamodbSharedLib.getItem(
+    _izContext,
+    fullMainTableName,
+    keyValues
+  );
+  _izContext.logger.debug('cacheObject', cacheObject);
+
+  if (
+    !cacheObject[uniqueRequestIdCompleteFieldName] ||
+    cacheObject[uniqueRequestIdCompleteFieldName] !== checkUniqueRequestId
+  ) {
+    return false;
+  } else {
+    return true;
+  }
+}
+
+//====================================== end Multiple Lambda Invocations (Logic pagination of handling results)
+
+/**
+ * Validate and normalize a DynamoDB pagination startKey object.
+ * Ensures partitionKeyFieldName/sortKeyFieldName are alphanumeric/underscore/hyphen.
+ * If startKey is empty object, returns `null` for easier processing downstream.
+ * @param {IzContext} _izContext
+ * @param {Attrs|null|undefined} startKey
+ * @param {string} partitionKeyFieldName
+ * @param {string} sortKeyFieldName
+ * @returns {Attrs|null}
+ * @throws {NoRetryError} On invalid field names or malformed startKey
+ */
+export function validateStartKeyParam(
+  _izContext,
+  startKey,
+  partitionKeyFieldName,
+  sortKeyFieldName
+) {
+  const stringNotEmptyRegex = /^[A-Za-z0-9_-]+$/;
+
+  if (
+    !partitionKeyFieldName ||
+    !sortKeyFieldName ||
+    !stringNotEmptyRegex.test(partitionKeyFieldName) ||
+    !stringNotEmptyRegex.test(sortKeyFieldName)
+  ) {
+    throw new NoRetryError(
+      'validateStartKeyParam: Invalid partitionKeyFieldName or sortKeyFieldName'
+    );
+  }
+
+  if (startKey && Object.keys(startKey).length != 0) {
+    if (!startKey[partitionKeyFieldName] || !startKey[sortKeyFieldName]) {
+      throw new NoRetryError(
+        'validateStartKeyParam: Invalid startKey, missing partitionKeyFieldName or sortKeyFieldName'
+      );
+    }
+  } else {
+    startKey = null; // if empty set to null so lib functions process correctly
+  }
+
+  return startKey;
+}
+
+/**
+ * For paginated multi-invocation flows: validates the current invocation count,
+ * attaches the (optional) next startKey to the message, increments the count,
+ * and re-enqueues the message to the specified SQS queue.
+ * @async
+ * @param {IzContext} _izContext
+ * @param {Attrs} messageProperty - The message body object to re-dispatch
+ * @param {Attrs} [passOnStartKey={}] - Optional startKey to pass along
+ * @param {number} numberInvocation - Current invocation count
+ * @param {string} queueName - Logical queue name resolvable via sqsSharedLib.sqsQueueUrl
+ * @returns {Promise<void>}
+ * @throws {NoRetryError} If `numberInvocation` exceeds internal safety limit
+ */
+export async function validateMultipleInvocations(
+  _izContext,
+  messageProperty,
+  passOnStartKey = {},
+  numberInvocation,
+  queueName
+) {
+  _izContext.logger.debug('Lib:validateMultipleInvocations', {
+    messageProperty: messageProperty,
+    passOnStartKey: passOnStartKey,
+    numberInvocation: numberInvocation,
+    queueName: queueName
+  });
+  try {
+    const maxProcessInvocationCountImport = 20;
+    if (numberInvocation >= maxProcessInvocationCountImport) {
+      _izContext.logger.error(
+        'numberInvocation exceeds maxProcessInvocationCountImport limit'
+      );
+      throw new NoRetryError(
+        'numberInvocation exceeds maxProcessInvocationCountImport limit'
+      );
+    }
+
+    if (passOnStartKey) {
+      messageProperty['startKey'] = passOnStartKey;
+    }
+    messageProperty['numberInvocation'] = numberInvocation;
+
+    let messageReInvokeFunction = {
+      MessageBody: JSON.stringify(messageProperty),
+      QueueUrl: await sqsSharedLib.sqsQueueUrl(_izContext, queueName)
+    };
+    _izContext.logger.debug(
+      `Send mesage to Dsq:${queueName} `,
+      messageReInvokeFunction
+    );
+    await sqs.sendMessage(_izContext, messageReInvokeFunction);
+  } catch (err) {
+    throw err;
+  }
+}