@hotmeshio/hotmesh 0.0.57 → 0.0.58

package/build/services/quorum/index.js

@@ -26,9 +26,12 @@ class QuorumService {
             instance.guid = guid;
             instance.logger = logger;
             instance.engine = engine;
+            //note: `quorum` shares/re-uses the engine's `store`/`sub` Redis clients
             await instance.initStoreChannel(config.engine.store);
             await instance.initSubChannel(config.engine.sub);
+            //general quorum subscription
             await instance.subscribe.subscribe(hotmesh_1.KeyType.QUORUM, instance.subscriptionHandler(), appId);
+            //app-specific quorum subscription (used for pubsub one-time request/response)
             await instance.subscribe.subscribe(hotmesh_1.KeyType.QUORUM, instance.subscriptionHandler(), appId, instance.guid);
             instance.engine.processWebHooks();
             instance.engine.processTimeHooks();
@@ -90,6 +93,7 @@ class QuorumService {
             else if (message.type === 'rollcall') {
                 self.doRollCall(message);
             }
+            //if there are any callbacks, call them
             if (self.callbacks.length > 0) {
                 self.callbacks.forEach(cb => cb(topic, message));
             }
@@ -131,6 +135,10 @@ class QuorumService {
         await (0, utils_1.sleepFor)(delay);
         return quorum;
     }
+    /**
+     * A quorum-wide command to broadcaset system details.
+     *
+     */
     async doRollCall(message) {
         let iteration = 0;
         let max = !isNaN(message.max) ? message.max : enums_1.HMSH_QUORUM_ROLLCALL_CYCLES;
@@ -157,15 +165,22 @@ class QuorumService {
     stop() {
         this.cancelRollCall();
     }
+    // ************* PUB/SUB METHODS *************
+    //publish a message to the quorum
     async pub(quorumMessage) {
         return await this.store.publish(hotmesh_1.KeyType.QUORUM, quorumMessage, this.appId, quorumMessage.topic || quorumMessage.guid);
     }
+    //subscribe user to quorum messages
     async sub(callback) {
+        //the quorum is always subscribed to the `quorum` topic; just register the fn
         this.callbacks.push(callback);
     }
+    //unsubscribe user from quorum messages
     async unsub(callback) {
+        //the quorum is always subscribed to the `quorum` topic; just unregister the fn
         this.callbacks = this.callbacks.filter(cb => cb !== callback);
     }
+    // ************* COMPILER METHODS *************
     async rollCall(delay = enums_1.HMSH_QUORUM_DELAY_MS) {
         await this.requestQuorum(delay, true);
         const targetStreams = [];
@@ -187,10 +202,14 @@ class QuorumService {
         });
         return this.profiles;
     }
+    /**
+     * request a quorum; if successful activate the app version
+     */
     async activate(version, delay = enums_1.HMSH_QUORUM_DELAY_MS, count = 0) {
         version = version.toString();
         const canActivate = await this.store.reserveScoutRole('activate', Math.ceil(delay * 6 / 1000) + 1);
         if (!canActivate) {
+            //another engine is already activating the app version
             this.logger.debug('quorum-activation-awaiting', { version });
             await (0, utils_1.sleepFor)(delay * 6);
             const app = await this.store.getApp(this.appId, true);
@@ -206,6 +225,7 @@ class QuorumService {
             this.store.publish(hotmesh_1.KeyType.QUORUM, { type: 'activate', cache_mode: 'nocache', until_version: version }, this.appId);
             await new Promise(resolve => setTimeout(resolve, delay));
             await this.store.releaseScoutRole('activate');
+            //confirm we received the activation message
             if (this.engine.untilVersion === version) {
                 this.logger.info('quorum-activation-succeeded', { version });
                 const { id } = config;
@@ -221,6 +241,7 @@ class QuorumService {
             this.logger.warn('quorum-rollcall-error', { q1, q2, q3, count });
             this.store.releaseScoutRole('activate');
             if (count < enums_1.HMSH_ACTIVATION_MAX_RETRY) {
+                //increase the delay (give the quorum time to respond) and try again
                 return await this.activate(version, delay * 2, count + 1);
             }
             throw new Error(`Quorum not reached. Version ${version} not activated.`);

package/build/services/reporter/index.d.ts

@@ -28,6 +28,11 @@ declare class ReporterService {
     getTargetForTime(key: string): string;
     getWorkItems(options: GetStatsOptions, facets: string[]): Promise<string[]>;
     private buildWorkerLists;
+    /**
+     * called by `trigger` activity to generate the stats that should
+     * be saved to the database. doesn't actually save the stats, but
+     * just generates the info that should be saved
+     */
     resolveTriggerStatistics({ stats: statsConfig }: TriggerActivity, context: JobState): StatsType;
     isGeneralMetric(metric: string): boolean;
     isMedianMetric(metric: string): boolean;

package/build/services/reporter/index.js

@@ -27,9 +27,11 @@ class ReporterService {
     }
     generateDateTimeSets(granularity, range, end, start) {
         if (granularity === 'infinity') {
+            //if granularity is infinity, it means a date/time sequence/slice is not used to further segment the statistics
             return ['0'];
         }
         if (!range) {
+            //pluck just a single value when no range provided
             range = '0m';
         }
         const granularitiesInMinutes = {
@@ -47,6 +49,7 @@ class ReporterService {
         if (rangeMinutes === null) {
             throw new Error('Invalid range value.');
         }
+        // If start is provided, use it. Otherwise, calculate it from the end time and range.
         let startTime;
         let endTime;
         if (start) {
@@ -57,6 +60,7 @@ class ReporterService {
             endTime = end === 'NOW' ? new Date() : new Date(end);
             startTime = new Date(endTime.getTime() - rangeMinutes * 60 * 1000);
         }
+        // Round the start time to the nearest granularity unit
         startTime.setUTCMinutes(Math.floor(startTime.getUTCMinutes() / granularityMinutes) * granularityMinutes);
         const dateTimeSets = [];
         for (let time = startTime; time <= endTime; time.setUTCMinutes(time.getUTCMinutes() + granularityMinutes)) {
@@ -266,6 +270,11 @@ class ReporterService {
         }
         return workerLists;
     }
+    /**
+     * called by `trigger` activity to generate the stats that should
+     * be saved to the database. doesn't actually save the stats, but
+     * just generates the info that should be saved
+     */
     resolveTriggerStatistics({ stats: statsConfig }, context) {
         const stats = {
             general: [],

package/build/services/router/index.d.ts

@@ -30,6 +30,15 @@ declare class Router {
     private resetThrottleState;
     createGroup(stream: string, group: string): Promise<void>;
     publishMessage(topic: string, streamData: StreamData | StreamDataResponse, multi?: RedisMulti): Promise<string | RedisMulti>;
+    /**
+     * An adjustable throttle that will interrupt a sleeping
+     * router if the throttle is reduced and the sleep time
+     * has elapsed. If the throttle is increased, or if
+     * the sleep time has not elapsed, the router will continue
+     * to sleep until the new termination point. This
+     * allows for dynamic, elastic throttling with smooth
+     * acceleration and deceleration.
+     */
     customSleep(): Promise<void>;
     consumeMessages(stream: string, group: string, consumer: string, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<void>;
     isStreamMessage(result: any): boolean;

package/build/services/router/index.js

@@ -47,13 +47,22 @@ class Router {
         const stream = this.store.mintKey(key_1.KeyType.STREAMS, { appId: this.store.appId, topic });
         return await this.store.xadd(stream, '*', 'message', JSON.stringify(streamData), multi);
     }
+    /**
+     * An adjustable throttle that will interrupt a sleeping
+     * router if the throttle is reduced and the sleep time
+     * has elapsed. If the throttle is increased, or if
+     * the sleep time has not elapsed, the router will continue
+     * to sleep until the new termination point. This
+     * allows for dynamic, elastic throttling with smooth
+     * acceleration and deceleration.
+     */
     async customSleep() {
         if (this.throttle === 0)
             return;
         if (this.isSleeping)
             return;
         this.isSleeping = true;
-        let startTime = Date.now();
+        let startTime = Date.now(); //anchor the origin
         await new Promise(async (outerResolve) => {
             this.sleepPromiseResolve = outerResolve;
             let elapsedTime = Date.now() - startTime;
@@ -81,6 +90,7 @@ class Router {
                 return;
             }
             try {
+                //randomizer that asymptotes at 150% of `HMSH_BLOCK_TIME_MS`
                 const streamDuration = enums_1.HMSH_BLOCK_TIME_MS + Math.round((enums_1.HMSH_BLOCK_TIME_MS * Math.random()));
                 const result = await this.stream.xreadgroup('GROUP', group, consumer, 'BLOCK', streamDuration, 'STREAMS', stream, '>');
                 if (this.isStreamMessage(result)) {
@@ -89,6 +99,7 @@ class Router {
                         await this.consumeOne(stream, group, id, message, callback);
                     }
                 }
+                // Check for pending messages (note: Redis 6.2 simplifies)
                 const now = Date.now();
                 if (now - lastCheckedPendingMessagesAt > this.reclaimDelay) {
                     lastCheckedPendingMessagesAt = now;
@@ -162,6 +173,7 @@ class Router {
                     await (0, utils_1.sleepFor)(timeout);
                     return await this.publishMessage(input.metadata.topic, {
                         data: input.data,
+                        //note: retain guid (this is a retry attempt)
                         metadata: { ...input.metadata, try: (input.metadata.try || 0) + 1 },
                         policies: input.policies,
                     });
@@ -181,12 +193,16 @@ class Router {
         }
     }
     shouldRetry(input, output) {
+        //const isUnhandledEngineError = output.code === 500;
         const policies = input.policies?.retry;
         const errorCode = output.code.toString();
         const policy = policies?.[errorCode];
         const maxRetries = policy?.[0];
         const tryCount = Math.min(input.metadata.try || 0, enums_1.HMSH_MAX_RETRIES);
+        //only possible values for maxRetries are 1, 2, 3
+        //only possible values for tryCount are 0, 1, 2
         if (maxRetries > tryCount) {
+            // 10ms, 100ms, or 1000ms delays between system retries
             return [true, Math.pow(10, tryCount + 1)];
         }
         return [false, 0];
@@ -222,6 +238,7 @@ class Router {
             code,
             data,
         };
+        //send unacknowleded errors to the engine (it has no topic)
         delete output.metadata.topic;
         return output;
     }
@@ -265,6 +282,7 @@ class Router {
         }
         const wasDecreased = delayInMillis < this.throttle;
         this.throttle = delayInMillis;
+        // If the throttle was decreased, and we're in the middle of a sleep cycle, adjust immediately
         if (wasDecreased) {
             if (this.sleepTimout) {
                 clearTimeout(this.sleepTimout);
@@ -276,7 +294,7 @@ class Router {
     }
     async claimUnacknowledged(stream, group, consumer, idleTimeMs = this.reclaimDelay, limit = enums_1.HMSH_XPENDING_COUNT) {
         let pendingMessages = [];
-        const pendingMessagesInfo = await this.stream.xpending(stream, group, '-', '+', limit);
+        const pendingMessagesInfo = await this.stream.xpending(stream, group, '-', '+', limit); //[[ '1688768134881-0', 'testConsumer1', 1017, 1 ]]
         for (const pendingMessageInfo of pendingMessagesInfo) {
             if (Array.isArray(pendingMessageInfo)) {
                 const [id, , elapsedTimeMs, deliveryCount] = pendingMessageInfo;
@@ -296,8 +314,16 @@ class Router {
         return pendingMessages;
     }
     async expireUnacknowledged(reclaimedMessage, stream, group, consumer, id, count) {
+        //The stream activity was not processed within established limits. Possibilities Include:
+        // 1) user error: the workers were not properly configured and are timing out
+        // 2a) system error: JSON is corrupt
+        //     i) unwitting actor
+        //     ii) corrupt hardware/network/transport/etc
+        // 3b) system error: Redis unable to accept `xadd` request
+        // 4c) system error: Redis unable to accept `xdel`/`xack` request
         this.logger.error('stream-message-max-delivery-count-exceeded', { id, stream, group, consumer, code: enums_1.HMSH_CODE_UNACKED, count });
         const streamData = reclaimedMessage[0]?.[1]?.[1];
+        //fatal risk point 1 of 3): json is corrupt
         const [err, input] = this.parseStreamData(streamData);
         if (err) {
             return this.logger.error('expire-unacknowledged-parse-fatal-error', { id, err });
@@ -311,9 +337,11 @@ class Router {
             telemetry = new telemetry_1.TelemetryService(this.appId);
             telemetry.startStreamSpan(input, stream_1.StreamRole.SYSTEM);
             telemetry.setStreamError(`Stream Message Max Delivery Count Exceeded`);
+            //fatal risk point 2 of 3): unable to publish error message (to notify the parent job)
             const output = this.structureUnacknowledgedError(input);
             messageId = await this.publishResponse(input, output);
             telemetry.setStreamAttributes({ 'app.worker.mid': messageId });
+            //fatal risk point 3 of 3): unable to ack and delete stream message
             await this.ackAndDelete(stream, group, id);
         }
         catch (err) {

package/build/services/serializer/index.js

@@ -49,6 +49,7 @@ class SerializerService {
                 return this.dIds[activityId];
             }
             else if ('$ADJACENT' in this.dIds) {
+                //else=> pre-authorizing adjacent activity entry
                 return this.dIds['$ADJACENT'];
             }
             return ',0';
@@ -117,6 +118,7 @@ class SerializerService {
                     result[shortDimensionalKey] = source[key];
                 }
                 else if (!(key in result) && this.isLiteralKeyType(key)) {
+                    //mark (-) and search (_)
                     result[key] = source[key];
                 }
             }
@@ -140,6 +142,7 @@ class SerializerService {
         const inflateWithMap = (abbreviationMap, id) => {
             const reversedAbbreviationMap = this.getReverseKeyMap(abbreviationMap, id);
             for (let key in result) {
+                //strip dimensional index from key
                 const shortKey = key.split(',')[0];
                 let longKey = reversedAbbreviationMap.get(shortKey);
                 if (longKey) {
@@ -156,22 +159,36 @@ class SerializerService {
         }
         return result;
     }
+    //stringify: convert a multi-dimensional document to a 2-d hash
     stringify(document) {
         let result = {};
         for (let key in document) {
             let value = SerializerService.toString(document[key]);
             if (value) {
+                // if (/^:*[a-zA-Z]{2}$/.test(value)) {
+                //   value = ':' + value;
+                // } else if (this.symValReverseMaps.has(value)) {
+                //   value = this.symValReverseMaps.get(value);
+                // }
                 result[key] = value;
             }
         }
         return result;
     }
+    //parse: convert a 2-d hash to a multi-dimensional document
     parse(document) {
         let result = {};
         for (let [key, value] of Object.entries(document)) {
             if (value === undefined || value === null)
                 continue;
+            // if (/^:+[a-zA-Z]{2}$/.test(value)) {
+            //   result[key] = value.slice(1);
+            // } else {
+            //   if (value?.length === 2 && this.symValMaps.has(value)) {
+            //     value = this.symValMaps.get(value);
+            //   }
             result[key] = SerializerService.fromString(value);
+            // }
         }
         return result;
     }
@@ -204,20 +221,20 @@ class SerializerService {
         const prefix = value.slice(0, 2);
         const rest = value.slice(2);
         switch (prefix) {
-            case '/t':
+            case '/t': // boolean true
                 return true;
-            case '/f':
+            case '/f': // boolean false
                 return false;
-            case '/d':
+            case '/d': // number
                 return Number(rest);
-            case '/n':
+            case '/n': // null
                 return null;
-            case '/s':
+            case '/s': // object (JSON string)
                 if (dateReg.exec(rest)) {
                     return new Date(JSON.parse(rest));
                 }
                 return JSON.parse(rest);
-            default:
+            default: // string
                 return value;
         }
     }

package/build/services/store/cache.d.ts

@@ -1,3 +1,10 @@
+/**
+ * The Cache is a key/value store and used to store commonly accessed Redis metadata
+ * (mainly the execution rules for the app) to save time accessing them as they
+ * are immutable per verison. Rules are only ejected when a new version
+ * (a new distributed executable) is deployed to the quorum
+ * and the cache is invalidated/cleared of the prior version.
+ */
 import { ActivityType } from "../../types/activity";
 import { HookRule } from "../../types/hook";
 import { HotMeshApp, HotMeshSettings } from "../../types/hotmesh";
@@ -14,7 +21,19 @@ declare class Cache {
     transitions: Record<string, Record<string, unknown>>;
     hookRules: Record<string, Record<string, HookRule[]>>;
     workItems: Record<string, string>;
+    /**
+     * The cache is ALWAYS initialized with HotMeshSettings. The other parameters are optional.
+     * @param settings
+     * @param apps
+     * @param schemas
+     * @param subscriptions
+     * @param transitions
+     * @param hookRules
+     */
     constructor(appId: string, settings: HotMeshSettings, apps?: Record<string, HotMeshApp>, schemas?: Record<string, ActivityType>, subscriptions?: Record<string, Record<string, string>>, symbols?: Record<string, Symbols>, symvals?: Record<string, Symbols>, transitions?: Record<string, Record<string, unknown>>, hookRules?: Record<string, Record<string, HookRule[]>>, workItems?: Record<string, string>);
+    /**
+     * invalidate the cache; settings are not invalidated!
+     */
     invalidate(): void;
     getSettings(): HotMeshSettings;
     setSettings(settings: HotMeshSettings): void;

package/build/services/store/cache.js

@@ -1,7 +1,23 @@
 "use strict";
+/**
+ * The Cache is a key/value store and used to store commonly accessed Redis metadata
+ * (mainly the execution rules for the app) to save time accessing them as they
+ * are immutable per verison. Rules are only ejected when a new version
+ * (a new distributed executable) is deployed to the quorum
+ * and the cache is invalidated/cleared of the prior version.
+ */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Cache = void 0;
 class Cache {
+    /**
+     * The cache is ALWAYS initialized with HotMeshSettings. The other parameters are optional.
+     * @param settings
+     * @param apps
+     * @param schemas
+     * @param subscriptions
+     * @param transitions
+     * @param hookRules
+     */
     constructor(appId, settings, apps = {}, schemas = {}, subscriptions = {}, symbols = {}, symvals = {}, transitions = {}, hookRules = {}, workItems = {}) {
         this.appId = appId;
         this.settings = settings;
@@ -14,6 +30,9 @@ class Cache {
         this.hookRules = hookRules;
         this.workItems = workItems;
     }
+    /**
+     * invalidate the cache; settings are not invalidated!
+     */
     invalidate() {
         this.apps = {};
         this.schemas = {};

package/build/services/store/clients/ioredis.js

@@ -24,6 +24,7 @@ class IORedisStoreService extends index_1.StoreService {
         return response;
     }
     hGetAllResult(result) {
+        //ioredis response signature is [null, {}] or [null, null]
         return result[1];
     }
     async addTaskQueues(keys) {

package/build/services/store/index.d.ts

@@ -41,6 +41,11 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
     zRangeByScore(key: string, score: number | string, value: string | number): Promise<string | null>;
     mintKey(type: KeyType, params: KeyStoreParams): string;
     invalidateCache(): void;
+    /**
+     * At any given time only a single engine will
+     * check for and process work items in the
+     * time and signal task queues.
+     */
     reserveScoutRole(scoutType: 'time' | 'signal' | 'activate', delay?: number): Promise<boolean>;
     releaseScoutRole(scoutType: 'time' | 'signal' | 'activate'): Promise<boolean>;
     getSettings(bCreate?: boolean): Promise<HotMeshSettings>;
@@ -59,7 +64,16 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
     setApp(id: string, version: string): Promise<HotMeshApp>;
     activateAppVersion(id: string, version: string): Promise<boolean>;
     registerAppVersion(appId: string, version: string): Promise<any>;
+    /**
+     * Registers the job, `jobId`, with `originJobId`. In the future,
+     * when `originJobId` is interrupted/expired, the items in the
+     * list (added via RPUSH) will be interrupted/expired (removed via LPOPed).
+     */
     registerJobDependency(depType: WorkListTaskType, originJobId: string, topic: string, jobId: string, gId: string, pd?: string, multi?: U): Promise<any>;
+    /**
+     * Ensures a `hook signal` is delisted when its parent activity/job
+     * is interrupted/expired.
+     */
     registerSignalDependency(jobId: string, signalKey: string, dad: string, multi?: U): Promise<any>;
     setStats(jobKey: string, jobId: string, dateTime: string, stats: StatsType, appVersion: AppVID, multi?: U): Promise<any>;
     hGetAllResult(result: any): any;
@@ -68,10 +82,24 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
     setStatus(collationKeyStatus: number, jobId: string, appId: string, multi?: U): Promise<any>;
     getStatus(jobId: string, appId: string): Promise<number>;
     setState({ ...state }: StringAnyType, status: number | null, jobId: string, symbolNames: string[], dIds: StringStringType, multi?: U): Promise<string>;
+    /**
+     * Returns custom search fields and values.
+     * NOTE: The `fields` param should NOT prefix items with an underscore.
+     */
     getQueryState(jobId: string, fields: string[]): Promise<StringAnyType>;
     getState(jobId: string, consumes: Consumes, dIds: StringStringType): Promise<[StringAnyType, number] | undefined>;
     getRaw(jobId: string): Promise<StringStringType>;
+    /**
+     * collate is a generic method for incrementing a value in a hash
+     * in order to track their progress during processing.
+     */
     collate(jobId: string, activityId: string, amount: number, dIds: StringStringType, multi?: U): Promise<number>;
+    /**
+     * synthentic collation affects those activities in the graph
+     * that represent the synthetic DAG that was materialized during compilation;
+     * Synthetic targeting ensures that re-entry due to failure can be distinguished from
+     * purposeful re-entry.
+     */
     collateSynthetic(jobId: string, guid: string, amount: number, multi?: U): Promise<number>;
     setStateNX(jobId: string, appId: string): Promise<boolean>;
     getSchema(activityId: string, appVersion: AppVID): Promise<ActivityType>;
@@ -92,11 +120,38 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
     deleteProcessedTaskQueue(workItemKey: string, key: string, processedKey: string, scrub?: boolean): Promise<void>;
     processTaskQueue(sourceKey: string, destinationKey: string): Promise<any>;
     expireJob(jobId: string, inSeconds: number): Promise<void>;
+    /**
+     * register the descendants of an expired origin flow to be
+     * expired at a future date; options indicate whether this
+     * is a standard `expire` or an `interrupt`
+     */
     registerDependenciesForCleanup(jobId: string, deletionTime: number, options: JobCompletionOptions): Promise<void>;
     getDependencies(jobId: string): Promise<string[]>;
+    /**
+     * registers a hook activity to be awakened (uses ZSET to
+     * store the 'sleep group' and LIST to store the events
+     * for the given sleep group. Sleep groups are
+     * organized into 'n'-second blocks (LISTS))
+     */
     registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, deletionTime: number, dad: string, multi?: U): Promise<void>;
     getNextTask(listKey?: string): Promise<[listKey: string, jobId: string, gId: string, activityId: string, type: WorkListTaskType] | boolean>;
+    /**
+     * when processing time jobs, the target LIST ID returned
+     * from the ZSET query can be prefixed to denote what to
+     * do with the work list. (not everything is known in advance,
+     * so the ZSET key defines HOW to approach the work in the
+     * generic LIST (lists typically contain target job ids)
+     * @param {string} listKey - composite key
+     */
     resolveTaskKeyContext(listKey: string): [WorkListTaskType, string];
+    /**
+     * Interrupts a job and sets sets a job error (410), if 'throw'!=false.
+     * This method is called by the engine and not by an activity and is
+     * followed by a call to execute job completion/cleanup tasks
+     * associated with a job completion event.
+     *
+     * Todo: move most of this logic to the engine (too much logic for the store)
+     */
     interrupt(topic: string, jobId: string, options?: JobInterruptOptions): Promise<void>;
     scrub(jobId: string): Promise<void>;
     findJobs(queryString?: string, limit?: number, batchSize?: number, cursor?: string): Promise<[string, string[]]>;