@hotmeshio/hotmesh 0.0.56 → 0.0.58

package/README.md

@@ -1,7 +1,7 @@
 # HotMesh
 ![alpha release](https://img.shields.io/badge/release-alpha-yellow)
 
-HotMesh transforms Redis into a distributed orchestration engine.
+HotMesh transforms Redis into an Orchestration Engine.
 
 *Write functions in your own preferred style, and let Redis govern their execution, reliably and durably.*
 
@@ -295,28 +295,28 @@ const hotMesh = await HotMesh.init({
 ```
 
 ### Observability
-Workflows and activities are run according to the rules you define, offering [Graph-Oriented](https://github.com/hotmeshio/sdk-typescript/tree/main/docs/system_lifecycle.md#telemetry) telemetry insights into your legacy function executions.
+Workflows and activities are run according to the rules you define, offering [Graph-Oriented](https://github.com/hotmeshio/sdk-typescript/blob/main/docs/system_lifecycle.md#telemetry) telemetry insights into your legacy function executions.
 
 ## FAQ
-Refer to the [FAQ](https://github.com/hotmeshio/sdk-typescript/tree/main/docs/faq.md) for terminology, definitions, and an exploration of how HotMesh facilitates orchestration use cases.
+Refer to the [FAQ](https://github.com/hotmeshio/sdk-typescript/blob/main/docs/faq.md) for terminology, definitions, and an exploration of how HotMesh facilitates orchestration use cases.
 
 ## Quick Start
-Refer to the [Quick Start](https://github.com/hotmeshio/sdk-typescript/tree/main/docs/quickstart.md) for sample flows you can easily copy, paste, and modify to get started.
+Refer to the [Quick Start](https://github.com/hotmeshio/sdk-typescript/blob/main/docs/quickstart.md) for sample YAML workflows you can copy, paste, and modify to get started.
 
 ## Developer Guide
-For more details on the complete development process, including information about schemas, APIs, and deployment, consult the [Developer Guide](https://github.com/hotmeshio/sdk-typescript/tree/main/docs/developer_guide.md).
+For more details on the complete development process, including information about schemas, APIs, and deployment, consult the [Developer Guide](https://github.com/hotmeshio/sdk-typescript/blob/main/docs/developer_guide.md).
 
 ## Model Driven Development
-[Model Driven Development](https://github.com/hotmeshio/sdk-typescript/tree/main/docs/model_driven_development.md) is an established strategy for managing process-oriented tasks. Check out this guide to understand its foundational principles.
+[Model Driven Development](https://github.com/hotmeshio/sdk-typescript/blob/main/docs/model_driven_development.md) is an established strategy for managing process-oriented tasks. Check out this guide to understand its foundational principles.
 
 ## Data Mapping
-Exchanging data between activities is central to HotMesh. For detailed information on supported functions and the functional mapping syntax (@pipes), see the [Data Mapping Overview](https://github.com/hotmeshio/sdk-typescript/tree/main/docs/data_mapping.md).
+Exchanging data between activities is central to HotMesh. For detailed information on supported functions and the functional mapping syntax (@pipes), see the [Data Mapping Overview](https://github.com/hotmeshio/sdk-typescript/blob/main/docs/data_mapping.md).
 
 ## Composition
-While the simplest graphs are linear, detailing a consistent sequence of non-cyclical activities, graphs can be layered to represent intricate business scenarios. Some can even be designed to accommodate long-lasting workflows that span months. For more details, check out the [Composable Workflow Guide](https://github.com/hotmeshio/sdk-typescript/tree/main/docs/composable_workflow.md).
+While the simplest graphs are linear, detailing a consistent sequence of non-cyclical activities, graphs can be layered to represent intricate business scenarios. Some can even be designed to accommodate long-lasting workflows that span months. For more details, check out the [Composable Workflow Guide](https://github.com/hotmeshio/sdk-typescript/blob/main/docs/composable_workflow.md).
 
 ## Distributed Orchestration
-HotMesh is a distributed orchestration engine. Refer to the [Distributed Orchestration Guide](https://github.com/hotmeshio/sdk-typescript/tree/main/docs/distributed_orchestration.md) for a detailed breakdown of the approach.
+HotMesh is a distributed orchestration engine. Refer to the [Distributed Orchestration Guide](https://github.com/hotmeshio/sdk-typescript/blob/main/docs/distributed_orchestration.md) for a detailed breakdown of the approach.
 
 ## System Lifecycle
-Gain insight into HotMesh's monitoring, exception handling, and alarm configurations via the [System Lifecycle Guide](https://github.com/hotmeshio/sdk-typescript/tree/main/docs/system_lifecycle.md).
+Gain insight into HotMesh's monitoring, exception handling, and alarm configurations via the [System Lifecycle Guide](https://github.com/hotmeshio/sdk-typescript/blob/main/docs/system_lifecycle.md).

package/build/modules/enums.js

@@ -1,7 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.HMSH_SCOUT_INTERVAL_SECONDS = exports.HMSH_FIDELITY_SECONDS = exports.HMSH_EXPIRE_DURATION = exports.HMSH_XPENDING_COUNT = exports.HMSH_XCLAIM_COUNT = exports.HMSH_XCLAIM_DELAY_MS = exports.HMSH_BLOCK_TIME_MS = exports.HMSH_DURABLE_EXP_BACKOFF = exports.HMSH_DURABLE_MAX_INTERVAL = exports.HMSH_DURABLE_MAX_ATTEMPTS = exports.HMSH_GRADUATED_INTERVAL_MS = exports.HMSH_MAX_TIMEOUT_MS = exports.HMSH_MAX_RETRIES = exports.HMSH_EXPIRE_JOB_SECONDS = exports.HMSH_OTT_WAIT_TIME = exports.HMSH_ACTIVATION_MAX_RETRY = exports.HMSH_QUORUM_DELAY_MS = exports.HMSH_QUORUM_ROLLCALL_CYCLES = exports.HMSH_STATUS_UNKNOWN = exports.HMSH_CODE_DURABLE_RETRYABLE = exports.HMSH_CODE_DURABLE_FATAL = exports.HMSH_CODE_DURABLE_MAXED = exports.HMSH_CODE_DURABLE_TIMEOUT = exports.HMSH_CODE_DURABLE_WAIT = exports.HMSH_CODE_DURABLE_PROXY = exports.HMSH_CODE_DURABLE_CHILD = exports.HMSH_CODE_DURABLE_ALL = exports.HMSH_CODE_DURABLE_SLEEP = exports.HMSH_CODE_UNACKED = exports.HMSH_CODE_TIMEOUT = exports.HMSH_CODE_UNKNOWN = exports.HMSH_CODE_INTERRUPT = exports.HMSH_CODE_NOTFOUND = exports.HMSH_CODE_PENDING = exports.HMSH_CODE_SUCCESS = exports.HMSH_LOGLEVEL = void 0;
+// HOTMESH SYSTEM
 exports.HMSH_LOGLEVEL = process.env.HMSH_LOGLEVEL || 'info';
+// HOTMESH STATUS CODES
 exports.HMSH_CODE_SUCCESS = 200;
 exports.HMSH_CODE_PENDING = 202;
 exports.HMSH_CODE_NOTFOUND = 404;
@@ -9,6 +11,7 @@ exports.HMSH_CODE_INTERRUPT = 410;
 exports.HMSH_CODE_UNKNOWN = 500;
 exports.HMSH_CODE_TIMEOUT = 504;
 exports.HMSH_CODE_UNACKED = 999;
+// DURABLE STATUS CODES
 exports.HMSH_CODE_DURABLE_SLEEP = 588;
 exports.HMSH_CODE_DURABLE_ALL = 589;
 exports.HMSH_CODE_DURABLE_CHILD = 590;
@@ -18,15 +21,20 @@ exports.HMSH_CODE_DURABLE_TIMEOUT = 596;
 exports.HMSH_CODE_DURABLE_MAXED = 597;
 exports.HMSH_CODE_DURABLE_FATAL = 598;
 exports.HMSH_CODE_DURABLE_RETRYABLE = 599;
+// HOTMESH MESSAGES
 exports.HMSH_STATUS_UNKNOWN = 'unknown';
-exports.HMSH_QUORUM_ROLLCALL_CYCLES = 12;
+// QUORUM
+exports.HMSH_QUORUM_ROLLCALL_CYCLES = 12; //max iterations
 exports.HMSH_QUORUM_DELAY_MS = 250;
 exports.HMSH_ACTIVATION_MAX_RETRY = 3;
+// ENGINE
 exports.HMSH_OTT_WAIT_TIME = parseInt(process.env.HMSH_OTT_WAIT_TIME, 10) || 1000;
 exports.HMSH_EXPIRE_JOB_SECONDS = parseInt(process.env.HMSH_EXPIRE_JOB_SECONDS, 10) || 1;
+// STREAM ROUTER
 exports.HMSH_MAX_RETRIES = parseInt(process.env.HMSH_MAX_RETRIES, 10) || 3;
 exports.HMSH_MAX_TIMEOUT_MS = parseInt(process.env.HMSH_MAX_TIMEOUT_MS, 10) || 60000;
 exports.HMSH_GRADUATED_INTERVAL_MS = parseInt(process.env.HMSH_GRADUATED_INTERVAL_MS, 10) || 5000;
+// DURABLE
 exports.HMSH_DURABLE_MAX_ATTEMPTS = 3;
 exports.HMSH_DURABLE_MAX_INTERVAL = '120s';
 exports.HMSH_DURABLE_EXP_BACKOFF = 10;
@@ -36,6 +44,7 @@ exports.HMSH_BLOCK_TIME_MS = process.env.HMSH_BLOCK_TIME_MS ? parseInt(process.e
 exports.HMSH_XCLAIM_DELAY_MS = parseInt(process.env.HMSH_XCLAIM_DELAY_MS, 10) || 1000 * 60;
 exports.HMSH_XCLAIM_COUNT = parseInt(process.env.HMSH_XCLAIM_COUNT, 10) || 3;
 exports.HMSH_XPENDING_COUNT = parseInt(process.env.HMSH_XPENDING_COUNT, 10) || 10;
+// TASK WORKER
 exports.HMSH_EXPIRE_DURATION = parseInt(process.env.HMSH_EXPIRE_DURATION, 10) || 1;
 const BASE_FIDELITY_SECONDS = 5;
 const TEST_FIDELITY_SECONDS = 1;

package/build/modules/key.d.ts

@@ -1,10 +1,48 @@
 import { KeyStoreParams, KeyType } from '../types/hotmesh';
+/**
+ * Keys
+ *
+ * hmsh ->                                            {hash}    hotmesh config {version: "0.0.1", namespace: "hmsh"}
+ * hmsh:a:<appid> ->                                  {hash}    app profile { "id": "appid", "version": "2", "versions/1": "GMT", "versions/2": "GMT"}
+ * hmsh:<appid>:e:<engineId> ->                       {string}  setnx to ensure only one engine of given id
+ * hmsh:<appid>:w: ->                                 {zset}    work items/tasks an engine must do like garbage collect or hook a set of matching records (hookAll)
+ * hmsh:<appid>:t: ->                                 {zset}    an ordered set of list (work lists) ids
+ * hmsh:<appid>:t:<timeValue?> ->                     {list}    a worklist of `jobId+activityId` items that should be awakened
+ * hmsh:<appid>:q: ->                                 {hash}    quorum-wide messages
+ * hmsh:<appid>:q:<ngnid> ->                          {hash}    engine-targeted messages (targeted quorum-oriented message)
+ * hmsh:<appid>:j:<jobid> ->                          {hash}    job data
+ * hmsh:<appid>:j:<jobid>:<activityid>  ->            {hash}    job activity data (a1)
+ * hmsh:<appid>:s:<jobkey>:<dateTime> ->              {hash}    job stats (general)
+ * hmsh:<appid>:s:<jobkey>:<dateTime>:mdn:<field/path>:<fieldvalue> ->      {zset}    job stats (median)
+ * hmsh:<appid>:s:<jobkey>:<dateTime>:index:<field/path>:<fieldvalue> ->    {list}    job stats (index of jobid[])
+ * hmsh:<appid>:v:<version>:activities ->             {hash}    schemas [cache]
+ * hmsh:<appid>:v:<version>:transitions ->            {hash}    transitions [cache]
+ * hmsh:<appid>:v:<version>:subscriptions ->          {hash}    subscriptions [cache]
+ * hmsh:<appid>:x: ->                                 {xstream} when an engine is sent or reads a buffered task (engines read from their custom topic)
+ * hmsh:<appid>:x:<topic> ->                          {xstream} when a worker is sent or reads a buffered task (workers read from their custom topic)
+ * hmsh:<appid>:hooks ->                              {hash}    hook patterns/rules; set at compile time
+ * hmsh:<appid>:signals ->                            {hash}    dynamic hook signals (hget/hdel) when resolving (always self-clean); added/removed at runtime
+ * hmsh:<appid>:sym:keys: ->                          {hash}    list of symbol ranges and :cursor assigned at version deploy time for job keys
+ * hmsh:<appid>:sym:keys:<activityid|$subscribes> ->  {hash}    list of symbols based upon schema enums (initially) and adaptively optimized (later) during runtime; if '$subscribes' is used as the activityid, it is a top-level `job` symbol set (for job keys)
+ * hmsh:<appid>:sym:vals: ->                          {hash}    list of symbols for job values across all app versions
+ */
 declare const HMNS = "hmsh";
 declare const KEYSEP = ":";
 declare const VALSEP = "::";
 declare const WEBSEP = "::";
 declare const TYPSEP = "::";
 declare class KeyService {
+    /**
+     * returns a key that can be used to access a value in the key/value store
+     * appropriate for the given key type; the keys have an implicit hierarchy
+     * and are used to organize data in the store in a tree-like structure
+     * via the use of colons as separators. The top-level entity is the hmsh manifest.
+     * This file will reveal the full scope of what is on the server (apps, versions, etc)
+     * @param namespace
+     * @param keyType
+     * @param params
+     * @returns {string}
+     */
     static mintKey(namespace: string, keyType: KeyType, params: KeyStoreParams): string;
 }
 export { KeyService, KeyType, KeyStoreParams, HMNS, KEYSEP, TYPSEP, WEBSEP, VALSEP };

package/build/modules/key.js

@@ -3,17 +3,55 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.VALSEP = exports.WEBSEP = exports.TYPSEP = exports.KEYSEP = exports.HMNS = exports.KeyType = exports.KeyService = void 0;
 const hotmesh_1 = require("../types/hotmesh");
 Object.defineProperty(exports, "KeyType", { enumerable: true, get: function () { return hotmesh_1.KeyType; } });
+/**
+ * Keys
+ *
+ * hmsh ->                                            {hash}    hotmesh config {version: "0.0.1", namespace: "hmsh"}
+ * hmsh:a:<appid> ->                                  {hash}    app profile { "id": "appid", "version": "2", "versions/1": "GMT", "versions/2": "GMT"}
+ * hmsh:<appid>:e:<engineId> ->                       {string}  setnx to ensure only one engine of given id
+ * hmsh:<appid>:w: ->                                 {zset}    work items/tasks an engine must do like garbage collect or hook a set of matching records (hookAll)
+ * hmsh:<appid>:t: ->                                 {zset}    an ordered set of list (work lists) ids
+ * hmsh:<appid>:t:<timeValue?> ->                     {list}    a worklist of `jobId+activityId` items that should be awakened
+ * hmsh:<appid>:q: ->                                 {hash}    quorum-wide messages
+ * hmsh:<appid>:q:<ngnid> ->                          {hash}    engine-targeted messages (targeted quorum-oriented message)
+ * hmsh:<appid>:j:<jobid> ->                          {hash}    job data
+ * hmsh:<appid>:j:<jobid>:<activityid>  ->            {hash}    job activity data (a1)
+ * hmsh:<appid>:s:<jobkey>:<dateTime> ->              {hash}    job stats (general)
+ * hmsh:<appid>:s:<jobkey>:<dateTime>:mdn:<field/path>:<fieldvalue> ->      {zset}    job stats (median)
+ * hmsh:<appid>:s:<jobkey>:<dateTime>:index:<field/path>:<fieldvalue> ->    {list}    job stats (index of jobid[])
+ * hmsh:<appid>:v:<version>:activities ->             {hash}    schemas [cache]
+ * hmsh:<appid>:v:<version>:transitions ->            {hash}    transitions [cache]
+ * hmsh:<appid>:v:<version>:subscriptions ->          {hash}    subscriptions [cache]
+ * hmsh:<appid>:x: ->                                 {xstream} when an engine is sent or reads a buffered task (engines read from their custom topic)
+ * hmsh:<appid>:x:<topic> ->                          {xstream} when a worker is sent or reads a buffered task (workers read from their custom topic)
+ * hmsh:<appid>:hooks ->                              {hash}    hook patterns/rules; set at compile time
+ * hmsh:<appid>:signals ->                            {hash}    dynamic hook signals (hget/hdel) when resolving (always self-clean); added/removed at runtime
+ * hmsh:<appid>:sym:keys: ->                          {hash}    list of symbol ranges and :cursor assigned at version deploy time for job keys
+ * hmsh:<appid>:sym:keys:<activityid|$subscribes> ->  {hash}    list of symbols based upon schema enums (initially) and adaptively optimized (later) during runtime; if '$subscribes' is used as the activityid, it is a top-level `job` symbol set (for job keys)
+ * hmsh:<appid>:sym:vals: ->                          {hash}    list of symbols for job values across all app versions
+ */
 const HMNS = "hmsh";
 exports.HMNS = HMNS;
-const KEYSEP = ':';
+const KEYSEP = ':'; //default delimiter for keys 
 exports.KEYSEP = KEYSEP;
-const VALSEP = '::';
+const VALSEP = '::'; //default delimiter for vals
 exports.VALSEP = VALSEP;
-const WEBSEP = '::';
+const WEBSEP = '::'; //default delimiter for webhook vals
 exports.WEBSEP = WEBSEP;
-const TYPSEP = '::';
+const TYPSEP = '::'; //delimiter for ZSET task typing (how should a list be used?)
 exports.TYPSEP = TYPSEP;
 class KeyService {
+    /**
+     * returns a key that can be used to access a value in the key/value store
+     * appropriate for the given key type; the keys have an implicit hierarchy
+     * and are used to organize data in the store in a tree-like structure
+     * via the use of colons as separators. The top-level entity is the hmsh manifest.
+     * This file will reveal the full scope of what is on the server (apps, versions, etc)
+     * @param namespace
+     * @param keyType
+     * @param params
+     * @returns {string}
+     */
     static mintKey(namespace, keyType, params) {
         switch (keyType) {
             case hotmesh_1.KeyType.HOTMESH:
@@ -45,12 +83,16 @@ class KeyService {
             case hotmesh_1.KeyType.SUBSCRIPTION_PATTERNS:
                 return `${namespace}:${params.appId}:v:${params.appVersion}:transitions`;
             case hotmesh_1.KeyType.HOOKS:
+                //`hooks` provide the pattern to resolve a value
                 return `${namespace}:${params.appId}:hooks`;
             case hotmesh_1.KeyType.SIGNALS:
+                //`signals` provide the registry of resolved values that link back to paused jobs
                 return `${namespace}:${params.appId}:signals`;
             case hotmesh_1.KeyType.SYMKEYS:
+                //`symbol keys` provide the registry of replacement values for job keys
                 return `${namespace}:${params.appId}:sym:keys:${params.activityId || ''}`;
             case hotmesh_1.KeyType.SYMVALS:
+                //`symbol vals` provide the registry of replacement values for job vals
                 return `${namespace}:${params.appId}:sym:vals:`;
             case hotmesh_1.KeyType.STREAMS:
                 return `${namespace}:${params.appId || ''}:x:${params.topic || ''}`;

package/build/modules/utils.d.ts

@@ -7,6 +7,7 @@ import { StreamCode, StreamStatus } from "../types/stream";
 import { SystemHealth } from '../types/quorum';
 export declare function getSystemHealth(): Promise<SystemHealth>;
 export declare function sleepFor(ms: number): Promise<unknown>;
+export declare function sleepImmediate(): Promise<void>;
 export declare function guid(): string;
 export declare function deterministicRandom(seed: number): number;
 export declare function identifyRedisType(redisInstance: any): 'redis' | 'ioredis' | null;
@@ -22,7 +23,15 @@ export declare function XSleepFor(ms: number): {
 };
 export declare function findTopKey(obj: AppTransitions, input: string): string | null;
 export declare function findSubscriptionForTrigger(obj: AppSubscriptions, value: string): string | null;
+/**
+ * Get the subscription topic for the flow to which @activityId belongs.
+ * TODO: resolve this value in the compiler...do not call this at runtime
+ */
 export declare function getSubscriptionTopic(activityId: string, store: StoreService<RedisClient, RedisMulti>, appVID: AppVID): Promise<string | undefined>;
+/**
+ * returns the 12-digit format of the iso timestamp (e.g, 202101010000); returns
+ * an empty string if overridden by the user to not segment by time (infinity).
+ */
 export declare function getTimeSeries(granularity: string): string;
 export declare function formatISODate(input: Date | string): string;
 export declare function getSymKey(number: number): string;

package/build/modules/utils.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.restoreHierarchy = exports.getValueByPath = exports.getIndexedHash = exports.getSymVal = exports.getSymKey = exports.formatISODate = exports.getTimeSeries = exports.getSubscriptionTopic = exports.findSubscriptionForTrigger = exports.findTopKey = exports.XSleepFor = exports.matchesStatus = exports.matchesStatusCode = exports.identifyRedisTypeFromClass = exports.polyfill = exports.identifyRedisType = exports.deterministicRandom = exports.guid = exports.sleepFor = exports.getSystemHealth = void 0;
+exports.restoreHierarchy = exports.getValueByPath = exports.getIndexedHash = exports.getSymVal = exports.getSymKey = exports.formatISODate = exports.getTimeSeries = exports.getSubscriptionTopic = exports.findSubscriptionForTrigger = exports.findTopKey = exports.XSleepFor = exports.matchesStatus = exports.matchesStatusCode = exports.identifyRedisTypeFromClass = exports.polyfill = exports.identifyRedisType = exports.deterministicRandom = exports.guid = exports.sleepImmediate = exports.sleepFor = exports.getSystemHealth = void 0;
 const os_1 = __importDefault(require("os"));
 const systeminformation_1 = __importDefault(require("systeminformation"));
 const nanoid_1 = require("nanoid");
@@ -21,13 +21,16 @@ async function getSystemHealth() {
     const freeMemory = os_1.default.freemem();
     const usedMemory = totalMemory - freeMemory;
     const cpus = os_1.default.cpus();
+    // CPU load calculation remains unchanged
     const cpuLoad = cpus.map((cpu, i) => {
         const total = Object.values(cpu.times).reduce((acc, tv) => acc + tv, 0);
         const idle = cpu.times.idle;
         const usage = ((total - idle) / total) * 100;
         return { [`CPU ${i} Usage`]: `${usage.toFixed(2)}%` };
     });
+    // Wrap each systeminformation call with safeExecute
     const networkStats = await safeExecute(systeminformation_1.default.networkStats(), []);
+    // Construct the system health object with error handling in mind
     const systemHealth = {
         TotalMemoryGB: `${(totalMemory / 1024 / 1024 / 1024).toFixed(2)} GB`,
         FreeMemoryGB: `${(freeMemory / 1024 / 1024 / 1024).toFixed(2)} GB`,
@@ -42,6 +45,10 @@ async function sleepFor(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
 }
 exports.sleepFor = sleepFor;
+function sleepImmediate() {
+    return new Promise((resolve) => setImmediate(resolve));
+}
+exports.sleepImmediate = sleepImmediate;
 function guid() {
     return (0, nanoid_1.nanoid)().replace(/[_-]/g, '0');
 }
@@ -74,6 +81,7 @@ function identifyRedisType(redisInstance) {
     return null;
 }
 exports.identifyRedisType = identifyRedisType;
+//todo: the polyfill methods will all be deleted in the `beta` release.
 exports.polyfill = {
     resolveActivityType(activityType) {
         if (activityType === 'activity') {
@@ -94,6 +102,7 @@ function identifyRedisTypeFromClass(redisClass) {
 exports.identifyRedisTypeFromClass = identifyRedisTypeFromClass;
 function matchesStatusCode(code, pattern) {
     if (typeof pattern === 'string') {
+        // Convert '*' wildcard to its regex equivalent (\d)
         const regexPattern = `^${pattern.replace(/\*/g, "\\d")}$`;
         return new RegExp(regexPattern).test(code.toString());
     }
@@ -105,6 +114,7 @@ function matchesStatus(status, targetStatus) {
 }
 exports.matchesStatus = matchesStatus;
 function XSleepFor(ms) {
+    //can be interrupted with `clearTimeout`
     let timerId;
     let promise = new Promise((resolve) => {
         timerId = setTimeout(resolve, ms);
@@ -131,6 +141,10 @@ function findSubscriptionForTrigger(obj, value) {
     return null;
 }
 exports.findSubscriptionForTrigger = findSubscriptionForTrigger;
+/**
+ * Get the subscription topic for the flow to which @activityId belongs.
+ * TODO: resolve this value in the compiler...do not call this at runtime
+ */
 async function getSubscriptionTopic(activityId, store, appVID) {
     const appTransitions = await store.getTransitions(appVID);
     const appSubscriptions = await store.getSubscriptions(appVID);
@@ -139,6 +153,10 @@ async function getSubscriptionTopic(activityId, store, appVID) {
     return topic;
 }
 exports.getSubscriptionTopic = getSubscriptionTopic;
+/**
+ * returns the 12-digit format of the iso timestamp (e.g, 202101010000); returns
+ * an empty string if overridden by the user to not segment by time (infinity).
+ */
 function getTimeSeries(granularity) {
     if (granularity.toString() === 'infinity') {
         return '0';

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.0.56",
+    "version": "0.0.58",
     "description": "Unbreakable Workflows",
     "main": "./build/index.js",
     "types": "./build/index.d.ts",

package/build/services/activities/activity.d.ts

@@ -7,6 +7,9 @@ import { JobState, JobStatus } from '../../types/job';
 import { MultiResponseFlags, RedisClient, RedisMulti } from '../../types/redis';
 import { StringAnyType } from '../../types/serializer';
 import { StreamCode, StreamData, StreamStatus } from '../../types/stream';
+/**
+ * The base class for all activities
+ */
 declare class Activity {
     config: ActivityType;
     data: ActivityData;
@@ -23,7 +26,15 @@ declare class Activity {
     adjacentIndex: number;
     constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);
     setLeg(leg: ActivityLeg): void;
+    /**
+     * Upon entering leg 1 of a duplexed activty, verify
+     * all aspects of the entry including job and activty state
+     */
     verifyEntry(): Promise<void>;
+    /**
+     * Upon entering leg 2 of a duplexed activty, verify
+     * all aspects of the re-entry including job and activty state
+     */
     verifyReentry(): Promise<number>;
     processEvent(status?: StreamStatus, code?: StreamCode, type?: 'hook' | 'output'): Promise<void>;
     processPending(telemetry: TelemetryService, type: 'hook' | 'output'): Promise<MultiResponseFlags>;
@@ -35,7 +46,14 @@ declare class Activity {
     mapInputData(): void;
     mapOutputData(): void;
     registerTimeout(): Promise<void>;
+    /**
+     * Any StreamMessage with a status of ERROR is bound to the activity
+     */
     bindActivityError(data: Record<string, unknown>): void;
+    /**
+     * unhandled activity errors (activities that return an ERROR StreamMessage
+     * status and have no adjacent children to transition to) are bound to the job
+     */
     bindJobError(data: Record<string, unknown>): void;
     getTriggerConfig(): Promise<ActivityType>;
     getJobStatus(): null | number;
@@ -50,6 +68,12 @@ declare class Activity {
     bindJobMetadataPaths(): string[];
     bindActivityMetadataPaths(): string[];
     getState(): Promise<void>;
+    /**
+     * if the job is created/deleted/created with the same key,
+     * the 'gid' ensures no stale messages (such as sleep delays)
+     * enter the workstream. Any message with a mismatched gid
+     * belongs to a prior job and can safely be ignored/dropped.
+     */
     assertGenerationalId(jobGID: string, msgGID?: string): void;
     initDimensionalAddress(dad: string): void;
     initSelf(context: StringAnyType): JobState;
@@ -59,6 +83,10 @@ declare class Activity {
     resolveAdjacentDad(): string;
     filterAdjacent(): Promise<StreamData[]>;
     transition(adjacencyList: StreamData[], jobStatus: JobStatus): Promise<string[]>;
+    /**
+     * A job with a vale < -100_000_000 is considered interrupted,
+     * as the interruption event decrements the job status by 1billion.
+     */
     jobWasInterrupted(jobStatus: JobStatus): boolean;
 }
 export { Activity, ActivityType };

package/build/services/activities/activity.js

@@ -10,6 +10,9 @@ const pipe_1 = require("../pipe");
 const serializer_1 = require("../serializer");
 const telemetry_1 = require("../telemetry");
 const stream_1 = require("../../types/stream");
+/**
+ * The base class for all activities
+ */
 class Activity {
     constructor(config, data, metadata, hook, engine, context) {
         this.status = stream_1.StreamStatus.SUCCESS;
@@ -27,12 +30,20 @@ class Activity {
     setLeg(leg) {
         this.leg = leg;
     }
+    /**
+     * Upon entering leg 1 of a duplexed activty, verify
+     * all aspects of the entry including job and activty state
+     */
     async verifyEntry() {
         this.setLeg(1);
         await this.getState();
         collator_1.CollatorService.assertJobActive(this.context.metadata.js, this.context.metadata.jid, this.metadata.aid);
         await collator_1.CollatorService.notarizeEntry(this);
     }
+    /**
+     * Upon entering leg 2 of a duplexed activty, verify
+     * all aspects of the re-entry including job and activty state
+     */
     async verifyReentry() {
         const guid = this.context.metadata.guid;
         this.setLeg(2);
@@ -40,6 +51,7 @@ class Activity {
         collator_1.CollatorService.assertJobActive(this.context.metadata.js, this.context.metadata.jid, this.metadata.aid);
         return await collator_1.CollatorService.notarizeReentry(this, guid);
     }
+    //********  DUPLEX RE-ENTRY POINT  ********//
     async processEvent(status = stream_1.StreamStatus.SUCCESS, code = 200, type = 'output') {
         this.setLeg(2);
         const jid = this.context.metadata.jid;
@@ -132,6 +144,7 @@ class Activity {
         telemetry.mapActivityAttributes();
         const jobStatus = this.resolveStatus(multiResponse);
         const attrs = { 'app.job.jss': jobStatus };
+        //adjacencyList membership has already been set at this point (according to activity status)
         const messageIds = await this.transition(this.adjacencyList, jobStatus);
         if (messageIds?.length) {
             attrs['app.activity.mids'] = messageIds.join(',');
@@ -154,6 +167,10 @@ class Activity {
             if (output) {
                 for (const key in output) {
                     const f1 = key.indexOf('[');
+                    //keys with array notation suffix `somekey[]` represent
+                    //dynamically-keyed mappings whose `value` must be moved to the output.
+                    //The `value` must be an object with keys appropriate to the
+                    //notation type: `somekey[0] (array)`, `somekey[-] (mark)`, OR `somekey[_] (search)`
                     if (f1 > -1) {
                         const amount = key.substring(f1 + 1).split(']')[0];
                         if (!isNaN(Number(amount))) {
@@ -180,6 +197,7 @@ class Activity {
         }
     }
     mapOutputData() {
+        //activity YAML may include output map data that produces/extends activity output data.
         if (this.config.output?.maps) {
             const mapper = new mapper_1.MapperService(this.config.output.maps, this.context);
             const actOutData = mapper.mapRules();
@@ -189,12 +207,21 @@ class Activity {
         }
     }
     async registerTimeout() {
+        //set timeout in support of hook and/or duplex
     }
+    /**
+     * Any StreamMessage with a status of ERROR is bound to the activity
+     */
     bindActivityError(data) {
         const md = this.context[this.metadata.aid].output.metadata;
         md.err = JSON.stringify(this.data);
+        //(temporary...useful for mapping error parts in the app.yaml)
         md.$error = { ...data, is_stream_error: true };
     }
+    /**
+     * unhandled activity errors (activities that return an ERROR StreamMessage
+     * status and have no adjacent children to transition to) are bound to the job
+     */
     bindJobError(data) {
         this.context.metadata.err = JSON.stringify({ ...data, is_stream_error: true });
     }
@@ -209,6 +236,7 @@ class Activity {
         await this.store.setStatus(amount, this.context.metadata.jid, appId, multi);
     }
     authorizeEntry(state) {
+        //pre-authorize activity state to allow entry for adjacent activities
         return this.adjacencyList?.map((streamData) => {
             const { metadata: { aid } } = streamData;
             state[`${aid}/output/metadata/as`] = collator_1.CollatorService.getSeed();
@@ -228,6 +256,7 @@ class Activity {
         const presets = this.authorizeEntry(state);
         this.bindDimensionalAddress(state);
         this.bindActivityState(state);
+        //symbolNames holds symkeys
         const symbolNames = [
             `$${this.config.subscribes}`,
             this.metadata.aid,
@@ -237,6 +266,7 @@ class Activity {
         return await this.store.setState(state, this.getJobStatus(), jobId, symbolNames, dIds, multi);
     }
     bindJobMetadata() {
+        //both legs of the most recently run activity (1 and 2) modify ju (job_updated)
         this.context.metadata.ju = (0, utils_1.formatISODate)(new Date());
     }
     bindActivityMetadata() {
@@ -323,6 +353,7 @@ class Activity {
         telemetry_1.TelemetryService.addTargetTelemetryPaths(consumes, this.config, this.metadata, this.leg);
         let { dad, jid } = this.context.metadata;
         const dIds = collator_1.CollatorService.getDimensionsById([...this.config.ancestors, this.metadata.aid], dad || '');
+        //`state` is a unidimensional hash; context is a tree
         const [state, _status] = await this.store.getState(jid, consumes, dIds);
         this.context = (0, utils_1.restoreHierarchy)(state);
         this.assertGenerationalId(this.context?.metadata?.gid, gid);
@@ -330,6 +361,12 @@ class Activity {
         this.initSelf(this.context);
         this.initPolicies(this.context);
     }
+    /**
+     * if the job is created/deleted/created with the same key,
+     * the 'gid' ensures no stale messages (such as sleep delays)
+     * enter the workstream. Any message with a mismatched gid
+     * belongs to a prior job and can safely be ignored/dropped.
+     */
     assertGenerationalId(jobGID, msgGID) {
         if (msgGID !== jobGID) {
             throw new errors_1.GenerationalError(jobGID, msgGID, this.context?.metadata?.jid ?? '', this.context?.metadata?.aid ?? '', this.context?.metadata?.dad ?? '');
@@ -356,9 +393,10 @@ class Activity {
         if (!self.output.metadata) {
             self.output.metadata = {};
         }
+        //prebind the updated timestamp (mappings need the time)
         self.output.metadata.au = (0, utils_1.formatISODate)(new Date());
         context['$self'] = self;
-        context['$job'] = context;
+        context['$job'] = context; //NEVER call STRINGIFY! (now circular)
         return context;
     }
     initPolicies(context) {
@@ -371,11 +409,13 @@ class Activity {
     resolveDad() {
         let dad = this.metadata.dad;
         if (this.adjacentIndex > 0) {
+            //if adjacent index > 0 the activity is cycling; replace last index with cycle index
             dad = `${dad.substring(0, dad.lastIndexOf(','))},${this.adjacentIndex}`;
         }
         return dad;
     }
     resolveAdjacentDad() {
+        //concat self and child dimension (all children (leg 1) begin life at 0)
         return `${this.resolveDad()}${collator_1.CollatorService.getDimensionalSeed(0)}`;
     }
     ;
@@ -383,6 +423,7 @@ class Activity {
         const adjacencyList = [];
         const transitions = await this.store.getTransitions(await this.engine.getVID());
         const transition = transitions[`.${this.metadata.aid}`];
+        //resolve the dimensional address for adjacent children
         const adjacentDad = this.resolveAdjacentDad();
         if (transition) {
             for (const toActivityId in transition) {
@@ -427,6 +468,10 @@ class Activity {
         }
         return mIds;
     }
+    /**
+     * A job with a vale < -100_000_000 is considered interrupted,
+     * as the interruption event decrements the job status by 1billion.
+     */
     jobWasInterrupted(jobStatus) {
         return jobStatus < -100000000;
     }

package/build/services/activities/await.js

@@ -12,6 +12,7 @@ class Await extends activity_1.Activity {
     constructor(config, data, metadata, hook, engine, context) {
         super(config, data, metadata, hook, engine, context);
     }
+    //********  INITIAL ENTRY POINT (A)  ********//
     async process() {
         this.logger.debug('await-process', { jid: this.context.metadata.jid, gid: this.context.metadata.gid, aid: this.metadata.aid });
         let telemetry;
@@ -20,12 +21,15 @@ class Await extends activity_1.Activity {
             telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
             telemetry.startActivitySpan(this.leg);
             this.mapInputData();
+            //save state and authorize reentry
             const multi = this.store.getMulti();
+            //todo: await this.registerTimeout();
             const messageId = await this.execActivity(multi);
             await collator_1.CollatorService.authorizeReentry(this, multi);
             await this.setState(multi);
             await this.setStatus(0, multi);
             const multiResponse = await multi.exec();
+            //telemetry
             telemetry.mapActivityAttributes();
             const jobStatus = this.resolveStatus(multiResponse);
             telemetry.setActivityAttributes({

package/build/services/activities/cycle.d.ts

@@ -7,6 +7,13 @@ declare class Cycle extends Activity {
     config: CycleActivity;
     constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);
     process(): Promise<string>;
+    /**
+     * Trigger the target ancestor to execute in a cycle,
+     * without violating the constraints of the DAG. Immutable
+     * `individual activity state` will execute in a new dimensional
+     * thread while `shared job state` can change. This
+     * pattern allows for retries without violating the DAG.
+     */
     cycleAncestorActivity(multi: RedisMulti): Promise<string>;
 }
 export { Cycle };