@hotmeshio/hotmesh 0.0.52 → 0.0.54

package/build/services/pipe/index.js

@@ -6,13 +6,17 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.Pipe = void 0;
 const functions_1 = __importDefault(require("./functions"));
 class Pipe {
-    constructor(rules, jobData) {
+    constructor(rules, jobData, context) {
         this.rules = rules;
         this.jobData = jobData;
+        this.context = context;
     }
     isPipeType(currentRow) {
         return !Array.isArray(currentRow) && '@pipe' in currentRow;
     }
+    isreduceType(currentRow) {
+        return !Array.isArray(currentRow) && '@reduce' in currentRow;
+    }
     static isPipeObject(obj) {
         return typeof obj === 'object' && obj !== null && !Array.isArray(obj) && '@pipe' in obj;
     }
@@ -30,36 +34,81 @@ class Pipe {
      * loop through each PipeItem row in this Pipe, resolving and transforming line by line
      * @returns {any} the result of the pipe
      */
-    process() {
-        let resolved = this.processCells(this.rules[0]);
+    process(resolved = null) {
+        let index = 0;
+        if (!(resolved || this.isPipeType(this.rules[0]) || this.isreduceType(this.rules[0]))) {
+            resolved = this.processCells(this.rules[0]); // Add type assertion
+            index = 1;
+        }
         const len = this.rules.length;
-        for (let i = 1; i < len; i++) {
-            resolved = this.processRow(this.rules[i], resolved, []);
+        const subPipeQueue = [];
+        for (let i = index; i < len; i++) {
+            resolved = this.processRow(this.rules[i], resolved, subPipeQueue);
         }
         return resolved[0];
     }
+    /**
+     * Transforms iterable `input` into a single value. Vars $output, $item, $key
+     * and $input are available. The final statement in the iterator (the reduction)
+     * is assumed to be the return value. A default $output object may be provided
+     * to the iterator by placing the the second cell of the preceding row. Otherwise,
+     * construct the object during first run and ensure it is the first cell of the
+     * last row of the iterator, so it is returned as the $output for the next cycle
+     * @param {unknown[]} input
+     * @returns {unknown}
+     * @private
+     */
+    reduce(input) {
+        let resolved = input[1] ?? null;
+        if (Array.isArray(input[0])) {
+            for (let index = 0; index < input[0].length; index++) {
+                this.context = { $input: input[0], $output: resolved, $item: input[0][index], $key: index.toString(), $index: index };
+                resolved = this.process([resolved]);
+            }
+        }
+        else {
+            let index = -1;
+            for (let $key in input[0]) {
+                index++;
+                this.context = { $input: input[0], $output: resolved, $item: input[0][$key], $key, $index: index };
+                resolved = this.process([resolved]);
+            }
+        }
+        return [resolved];
+    }
     processRow(currentRow, resolvedPriorRow, subPipeQueue) {
-        if (this.isPipeType(currentRow)) {
-            //currentRow is a recursive subPipe
-            const subPipe = new Pipe(currentRow['@pipe'], this.jobData);
+        if (resolvedPriorRow && this.isreduceType(currentRow)) {
+            //reduce the resolvedPriorRow and return the output from the reducer
+            const subPipe = new Pipe(currentRow['@reduce'], this.jobData);
+            const reduced = subPipe.reduce(resolvedPriorRow);
+            return reduced;
+        }
+        else if (this.isPipeType(currentRow)) {
+            //process subPipe and push to subPipeQueue; echo resolvedPriorRow
+            const subPipe = new Pipe(currentRow['@pipe'], this.jobData, this.context);
             subPipeQueue.push(subPipe.process());
-            //return prior row as if nothing happened
             return resolvedPriorRow;
         }
         else {
+            //pivot the subPipeQueue into the arguments array (resolvedPriorRow)
             if (subPipeQueue.length > 0) {
-                //if items in subPipeQueue, flush and use as resolvedPriorRow
                 resolvedPriorRow = [...subPipeQueue];
                 subPipeQueue.length = 0;
             }
-            else if (!resolvedPriorRow) {
+            if (!resolvedPriorRow) {
                 //if no prior row, use current row as prior row
-                return [].concat(this.processCells([...currentRow]));
+                return [].concat(this.processCells(Array.isArray(currentRow) ? [...currentRow] : []));
             }
             else {
-                const [functionName, ...params] = currentRow;
+                const [functionName, ...params] = currentRow; // Add type assertion
                 //use resolved values from prior row (n - 1) as input params to cell 1 function
-                const resolvedValue = Pipe.resolveFunction(functionName)(...resolvedPriorRow);
+                let resolvedValue;
+                if (this.isContextVariable(functionName)) {
+                    resolvedValue = this.resolveContextValue(functionName);
+                }
+                else {
+                    resolvedValue = Pipe.resolveFunction(functionName)(...resolvedPriorRow);
+                }
                 //resolve remaining cells in row and return concatenated with resolvedValue
                 return [resolvedValue].concat(this.processCells([...params]));
             }
@@ -80,14 +129,21 @@ class Pipe {
     }
     processCells(cells) {
         const resolved = [];
-        for (const currentCell of cells) {
-            resolved.push(this.resolveCellValue(currentCell));
+        if (Array.isArray(cells)) {
+            for (const currentCell of cells) {
+                resolved.push(this.resolveCellValue(currentCell));
+            }
         }
         return resolved;
     }
     isFunction(currentCell) {
         return typeof currentCell === 'string' && currentCell.startsWith('{@') && currentCell.endsWith('}');
     }
+    isContextVariable(currentCell) {
+        if (typeof currentCell === 'string' && currentCell.endsWith('}')) {
+            return currentCell.startsWith('{$item') || currentCell.startsWith('{$key') || currentCell.startsWith('{$index') || currentCell.startsWith('{$input') || currentCell.startsWith('{$output');
+        }
+    }
     isMappable(currentCell) {
         return typeof currentCell === 'string' && currentCell.startsWith('{') && currentCell.endsWith('}');
     }
@@ -96,6 +152,9 @@ class Pipe {
             const fn = Pipe.resolveFunction(currentCell);
             return fn.call();
         }
+        else if (this.isContextVariable(currentCell)) {
+            return this.resolveContextValue(currentCell);
+        }
         else if (this.isMappable(currentCell)) {
             return this.resolveMappableValue(currentCell);
         }
@@ -118,6 +177,13 @@ class Pipe {
         const term = this.resolveMapTerm(currentCell);
         return this.getNestedProperty(this.jobData, term);
     }
+    resolveContextValue(currentCell) {
+        const term = this.resolveContextTerm(currentCell);
+        return this.getNestedProperty(this.context, term);
+    }
+    resolveContextTerm(currentCell) {
+        return currentCell.substring(1, currentCell.length - 1);
+    }
     resolveFunctionTerm(currentCell) {
         return currentCell.substring(2, currentCell.length - 1);
     }

package/build/services/router/index.js

@@ -154,7 +154,7 @@ class Router {
             output = await callback(input);
         }
         catch (error) {
-            this.logger.error(`stream-call-function-error`, { error });
+            this.logger.error(`stream-call-function-error`, { ...error, input: input, stack: error.stack, message: error.message, name: error.name, stream, id });
             output = this.structureUnhandledError(input, error);
         }
         return output;
@@ -193,10 +193,19 @@ class Router {
         }
     }
     shouldRetry(input, output) {
+        //const isUnhandledEngineError = output.code === 500;
         const policies = input.policies?.retry;
         const errorCode = output.code.toString();
         const policy = policies?.[errorCode];
         const maxRetries = policy?.[0];
+        // if (isUnhandledEngineError && !policy) {
+        //   //if main goes down, replicas take over within 5s
+        //   //if this is not system/platform related, the exponential
+        //   //backoff will be applied and eventually slow to a crawl while
+        //   //the root cause is identified
+        //   input.policies = { retry: { [errorCode]: [10] } };
+        //   return [true, 0];
+        // }
         const tryCount = Math.min(input.metadata.try || 0, enums_1.HMSH_MAX_RETRIES);
         //only possible values for maxRetries are 1, 2, 3
         //only possible values for tryCount are 0, 1, 2
@@ -245,13 +254,15 @@ class Router {
         const message = output.data?.message ? output.data?.message.toString() : enums_1.HMSH_STATUS_UNKNOWN;
         const statusCode = output.code || output.data?.code;
         const code = isNaN(statusCode) ? enums_1.HMSH_CODE_UNKNOWN : parseInt(statusCode.toString());
-        const data = { message, code };
+        const stack = output.data?.stack ? output.data?.stack.toString() : undefined;
+        const data = { message, code, stack };
         if (typeof output.data?.error === 'object') {
             data.error = { ...output.data.error };
         }
         return {
             status: stream_1.StreamStatus.ERROR,
             code,
+            stack,
             metadata: { ...input.metadata, guid: (0, utils_1.guid)() },
             data
         };
@@ -314,7 +325,7 @@ class Router {
         //The stream activity was not processed within established limits. Possibilities Include:
         // 1) user error: the workers were not properly configured and are timing out
         // 2a) system error: JSON is corrupt
-        //     i) bad/unwitting actor
+        //     i) unwitting actor
         //     ii) corrupt hardware/network/transport/etc
         // 3b) system error: Redis unable to accept `xadd` request
         // 4c) system error: Redis unable to accept `xdel`/`xack` request

package/build/services/serializer/index.d.ts

@@ -30,11 +30,12 @@ export declare class SerializerService {
     getReverseValueMap(valueMap: SymbolMap): SymbolMap;
     static filterSymVals(startIndex: number, maxIndex: number, existingSymbolValues: Symbols, proposedValues: Set<string>): Symbols;
     compress(document: StringStringType, ids: string[]): StringStringType;
+    isLiteralKeyType(key: string): boolean;
     decompress(document: StringStringType, ids: string[]): StringStringType;
     stringify(document: Record<string, any>): StringStringType;
     parse(document: StringStringType): any;
-    toString(value: any): string | undefined;
-    fromString(value: string | undefined): any;
+    static toString(value: any): string | undefined;
+    static fromString(value: string | undefined): any;
     package(multiDimensionalDocument: StringAnyType, ids: string[]): StringStringType;
     unpackage(document: StringStringType, ids: string[]): StringAnyType;
     export(): SymbolSets;

package/build/services/serializer/index.js

@@ -117,6 +117,10 @@ class SerializerService {
                     const shortDimensionalKey = `${shortKey}${dimensionalIndex}`;
                     result[shortDimensionalKey] = source[key];
                 }
+                else if (!(key in result) && this.isLiteralKeyType(key)) {
+                    //mark (-) and search (_)
+                    result[key] = source[key];
+                }
             }
         };
         for (let id of ids) {
@@ -127,6 +131,9 @@ class SerializerService {
         }
         return result;
     }
+    isLiteralKeyType(key) {
+        return key.startsWith('-') || key.startsWith('_');
+    }
     decompress(document, ids) {
         if (this.symKeys.size === 0) {
             return document;
@@ -156,7 +163,7 @@ class SerializerService {
     stringify(document) {
         let result = {};
         for (let key in document) {
-            let value = this.toString(document[key]);
+            let value = SerializerService.toString(document[key]);
             if (value) {
                 if (/^:*[a-zA-Z]{2}$/.test(value)) {
                     value = ':' + value;
@@ -182,12 +189,12 @@ class SerializerService {
                 if (value?.length === 2 && this.symValMaps.has(value)) {
                     value = this.symValMaps.get(value);
                 }
-                result[key] = this.fromString(value);
+                result[key] = SerializerService.fromString(value);
             }
         }
         return result;
     }
-    toString(value) {
+    static toString(value) {
         switch (typeof value) {
             case 'string':
                 break;
@@ -210,7 +217,7 @@ class SerializerService {
         }
         return value;
     }
-    fromString(value) {
+    static fromString(value) {
         if (typeof value !== 'string')
             return undefined;
         const prefix = value.slice(0, 2);

package/build/services/store/clients/ioredis.js

@@ -65,7 +65,7 @@ class IORedisStoreService extends index_1.StoreService {
             return await (multi || this.redisClient).xadd(key, id, messageId, messageValue);
         }
         catch (error) {
-            this.logger.error(`Error publishing 'xadd'; key: ${key}`, { error });
+            this.logger.error(`Error publishing 'xadd'; key: ${key}`, { ...error });
             throw error;
         }
     }
@@ -74,7 +74,7 @@ class IORedisStoreService extends index_1.StoreService {
             return await this.redisClient.xpending(key, group, start, end, count, consumer);
         }
         catch (error) {
-            this.logger.error(`Error in retrieving pending messages for [stream ${key}], [group ${group}]`, { error });
+            this.logger.error(`Error in retrieving pending messages for [stream ${key}], [group ${group}]`, { ...error });
             throw error;
         }
     }
@@ -83,7 +83,7 @@ class IORedisStoreService extends index_1.StoreService {
             return await this.redisClient.xclaim(key, group, consumer, minIdleTime, id, ...args);
         }
         catch (error) {
-            this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, { error });
+            this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, { ...error });
             throw error;
         }
     }
@@ -92,7 +92,7 @@ class IORedisStoreService extends index_1.StoreService {
             return await (multi || this.redisClient).xack(key, group, id);
         }
         catch (error) {
-            this.logger.error(`Error in acknowledging messages in group: ${group} for key: ${key}`, { error });
+            this.logger.error(`Error in acknowledging messages in group: ${group} for key: ${key}`, { ...error });
             throw error;
         }
     }
@@ -101,7 +101,7 @@ class IORedisStoreService extends index_1.StoreService {
             return await (multi || this.redisClient).xdel(key, id);
         }
         catch (error) {
-            this.logger.error(`Error in deleting messages with id: ${id} for key: ${key}`, { error });
+            this.logger.error(`Error in deleting messages with id: ${id} for key: ${key}`, { ...error });
             throw error;
         }
     }
@@ -110,7 +110,7 @@ class IORedisStoreService extends index_1.StoreService {
             return await (multi || this.redisClient).xlen(key);
         }
         catch (error) {
-            this.logger.error(`Error getting stream depth: ${key}`, { error });
+            this.logger.error(`Error getting stream depth: ${key}`, { ...error });
             throw error;
         }
     }

package/build/services/store/clients/redis.js

@@ -71,7 +71,7 @@ class RedisStoreService extends index_1.StoreService {
         }
         catch (error) {
             const streamType = mkStream === 'MKSTREAM' ? 'with MKSTREAM' : 'without MKSTREAM';
-            this.logger.debug(`x-group-error ${streamType} for key: ${key} and group: ${groupName}`, { error });
+            this.logger.debug(`x-group-error ${streamType} for key: ${key} and group: ${groupName}`, { ...error });
             throw error;
         }
     }
@@ -84,7 +84,7 @@ class RedisStoreService extends index_1.StoreService {
             return await (multi || this.redisClient).XADD(key, id, { [args[0]]: args[1] });
         }
         catch (error) {
-            this.logger.error(`Error publishing 'xadd'; key: ${key}`, { error });
+            this.logger.error(`Error publishing 'xadd'; key: ${key}`, { ...error });
             throw error;
         }
     }
@@ -102,7 +102,7 @@ class RedisStoreService extends index_1.StoreService {
             return await this.redisClient.sendCommand(['XPENDING', ...args]);
         }
         catch (error) {
-            this.logger.error(`Error in retrieving pending messages for group: ${group} in key: ${key}`, { error });
+            this.logger.error(`Error in retrieving pending messages for group: ${group} in key: ${key}`, { ...error });
             throw error;
         }
     }
@@ -111,7 +111,7 @@ class RedisStoreService extends index_1.StoreService {
             return await this.redisClient.sendCommand(['XCLAIM', key, group, consumer, minIdleTime.toString(), id, ...args]);
         }
         catch (error) {
-            this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, { error });
+            this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, { ...error });
             throw error;
         }
     }
@@ -126,7 +126,7 @@ class RedisStoreService extends index_1.StoreService {
             }
         }
         catch (error) {
-            this.logger.error(`Error in acknowledging messages in group: ${group} for key: ${key}`, { error });
+            this.logger.error(`Error in acknowledging messages in group: ${group} for key: ${key}`, { ...error });
             throw error;
         }
     }
@@ -141,7 +141,7 @@ class RedisStoreService extends index_1.StoreService {
             }
         }
         catch (error) {
-            this.logger.error(`Error in deleting messages with ids: ${id} for key: ${key}`, { error });
+            this.logger.error(`Error in deleting messages with ids: ${id} for key: ${key}`, { ...error });
             throw error;
         }
     }
@@ -156,7 +156,7 @@ class RedisStoreService extends index_1.StoreService {
             }
         }
         catch (error) {
-            this.logger.error(`Error getting stream depth: ${key}`, { error });
+            this.logger.error(`Error getting stream depth: ${key}`, { ...error });
             throw error;
         }
     }

package/build/services/store/index.d.ts

@@ -149,6 +149,8 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
      * This method is called by the engine and not by an activity and is
      * followed by a call to execute job completion/cleanup tasks
      * associated with a job completion event.
+     *
+     * Todo: move most of this logic to the engine (too much logic for the store)
      */
     interrupt(topic: string, jobId: string, options?: JobInterruptOptions): Promise<void>;
     scrub(jobId: string): Promise<void>;

package/build/services/store/index.js

@@ -850,6 +850,8 @@ class StoreService {
      * This method is called by the engine and not by an activity and is
      * followed by a call to execute job completion/cleanup tasks
      * associated with a job completion event.
+     *
+     * Todo: move most of this logic to the engine (too much logic for the store)
      */
     async interrupt(topic, jobId, options = {}) {
         try {
@@ -876,8 +878,9 @@ class StoreService {
                 this.serializer.resetSymbols(symKeys, symVals, {});
                 //persists the standard 410 error (job is `gone`)
                 const err = JSON.stringify({
-                    code: enums_1.HMSH_CODE_INTERRUPT,
+                    code: options.code ?? enums_1.HMSH_CODE_INTERRUPT,
                     message: options.reason ?? `job [${jobId}] interrupted`,
+                    stack: options.stack ?? '',
                     job_id: jobId
                 });
                 const payload = { [errKey]: amount.toString() };

package/build/services/stream/clients/ioredis.js

@@ -45,7 +45,7 @@ class IORedisStreamService extends index_1.StreamService {
             return await (multi || this.redisClient).xadd(key, id, messageId, messageValue);
         }
         catch (error) {
-            this.logger.error(`Error publishing 'xadd'; key: ${key}`, { error });
+            this.logger.error(`Error publishing 'xadd'; key: ${key}`, { ...error });
             throw error;
         }
     }
@@ -57,7 +57,7 @@ class IORedisStreamService extends index_1.StreamService {
             blockOption, blockTime, streamsOption, streamName, id);
         }
         catch (error) {
-            this.logger.error(`Error reading stream data [Stream ${streamName}] [Group ${groupName}]`, { error });
+            this.logger.error(`Error reading stream data [Stream ${streamName}] [Group ${groupName}]`, { ...error });
             throw error;
         }
     }
@@ -76,11 +76,11 @@ class IORedisStreamService extends index_1.StreamService {
                 return await this.redisClient.call('XPENDING', ...args);
             }
             catch (error) {
-                this.logger.error('err, args', { error }, args);
+                this.logger.error('err, args', { ...error }, args);
             }
         }
         catch (error) {
-            this.logger.error(`Error in retrieving pending messages for [stream ${key}], [group ${group}]`, { error });
+            this.logger.error(`Error in retrieving pending messages for [stream ${key}], [group ${group}]`, { ...error });
             throw error;
         }
     }
@@ -89,7 +89,7 @@ class IORedisStreamService extends index_1.StreamService {
             return await this.redisClient.xclaim(key, group, consumer, minIdleTime, id, ...args);
         }
         catch (error) {
-            this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, { error });
+            this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, { ...error });
             throw error;
         }
     }
@@ -98,7 +98,7 @@ class IORedisStreamService extends index_1.StreamService {
             return await (multi || this.redisClient).xack(key, group, id);
         }
         catch (error) {
-            this.logger.error(`Error in acknowledging messages in group: ${group} for key: ${key}`, { error });
+            this.logger.error(`Error in acknowledging messages in group: ${group} for key: ${key}`, { ...error });
             throw error;
         }
     }
@@ -107,7 +107,7 @@ class IORedisStreamService extends index_1.StreamService {
             return await (multi || this.redisClient).xdel(key, id);
         }
         catch (error) {
-            this.logger.error(`Error in deleting messages with id: ${id} for key: ${key}`, { error });
+            this.logger.error(`Error in deleting messages with id: ${id} for key: ${key}`, { ...error });
             throw error;
         }
     }
@@ -116,7 +116,7 @@ class IORedisStreamService extends index_1.StreamService {
             return await (multi || this.redisClient).xlen(key);
         }
         catch (error) {
-            this.logger.error(`Error getting stream depth: ${key}`, { error });
+            this.logger.error(`Error getting stream depth: ${key}`, { ...error });
             throw error;
         }
     }

package/build/services/stream/clients/redis.js

@@ -126,7 +126,7 @@ class RedisStreamService extends index_1.StreamService {
             }
         }
         catch (error) {
-            this.logger.error(`Error getting stream depth: ${key}`, { error });
+            this.logger.error(`Error getting stream depth: ${key}`, { ...error });
             throw error;
         }
     }

package/build/types/activity.d.ts

@@ -102,29 +102,84 @@ interface SignalActivity extends BaseActivity {
 }
 interface InterruptActivity extends BaseActivity {
     type: 'interrupt';
-    /** Optional Reason; will be used as the error `message` when thrown; NOTE: 410 is the error `code` */
+    /**
+     * Optional Reason; will be used as the error `message` when thrown
+     * @default 'Job Interrupted'
+     */
     reason?: string;
-    /** default is `true` (throw JobInterrupted error upon interrupting) */
+    /**
+     * throw JobInterrupted error upon interrupting
+     * @default true
+     */
     throw?: boolean;
-    /** TODO: // default is `false` (do not interrupt child jobs) */
+    /**
+     * Interrupt child/descendant jobs
+     * @default false
+     */
     descend?: boolean;
-    /** target job id (if not present the current job will be targeted) */
+    /**
+     * Target job id (if not present the current job will be targeted)
+     */
     target?: string;
-    /** topic to publish the interrupt message (if not present the current job topic will be used) */
+    /**
+     * Optional topic to publish the interrupt message (if not present the current job topic will be used
+     */
     topic?: string;
+    /**
+     * Optional Error Code; will be used as the error `code` when thrown
+     * @default 410
+     */
+    code?: number;
+    /**
+     * Optional stack trace
+     */
+    stack?: string;
 }
 type ActivityType = BaseActivity | TriggerActivity | AwaitActivity | WorkerActivity | InterruptActivity | HookActivity | SignalActivity | CycleActivity;
 type ActivityData = Record<string, any>;
+/**
+ * Type definition for activity metadata.
+ */
 type ActivityMetadata = {
+    /**
+     * Unique identifier for the activity.
+     */
     aid: string;
+    /**
+     * Type of the activity.
+     */
     atp: string;
+    /**
+     * Subtype of the activity.
+     */
     stp: string;
+    /**
+     * Timestamp when the activity was created, in GMT.
+     */
     ac: string;
+    /**
+     * Timestamp when the activity was last updated, in GMT.
+     */
     au: string;
+    /**
+     * Optional stringified JSON containing error details such as message, code, and an optional error object.
+     */
     err?: string;
+    /**
+     * OpenTelemetry span context for the first leg of the activity.
+     */
     l1s?: string;
+    /**
+     * OpenTelemetry span context for the second leg of the activity.
+     */
     l2s?: string;
+    /**
+     * Dimensional address, used for additional metadata.
+     */
     dad?: string;
+    /**
+     * Status of the activity, could be codes representing different states.
+     */
     as?: string;
 };
 type ActivityContext = {