@hotmeshio/hotmesh 0.0.34 → 0.0.36

package/build/services/signaler/stream.js

@@ -1,22 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.StreamSignaler = void 0;
+const enums_1 = require("../../modules/enums");
 const key_1 = require("../../modules/key");
 const utils_1 = require("../../modules/utils");
 const telemetry_1 = require("../telemetry");
 const stream_1 = require("../../types/stream");
-const MAX_RETRIES = 3; //local retry; 10, 100, 1000ms
-const MAX_TIMEOUT_MS = 60000;
-const GRADUATED_INTERVAL_MS = 5000;
-const BLOCK_DURATION = 15000; //Set to `15` so SIGINT/SIGTERM can interrupt; set to `0` to BLOCK indefinitely
-const TEST_BLOCK_DURATION = 1000; //Set to `1000` so tests can interrupt quickly
-const BLOCK_TIME_MS = process.env.NODE_ENV === 'test' ? TEST_BLOCK_DURATION : BLOCK_DURATION;
-const SYSTEM_STATUS_CODE = 999;
-const UNKNOWN_STATUS_CODE = 500;
-const UNKNOWN_STATUS_MESSAGE = 'unknown';
-const XCLAIM_DELAY_MS = 1000 * 60; //max time a message can be unacked before it is claimed by another
-const XCLAIM_COUNT = 3; //max number of times a message can be claimed by another before it is dead-lettered
-const XPENDING_COUNT = 10;
 class StreamSignaler {
     constructor(config, stream, store, logger) {
         this.throttle = 0;
@@ -28,8 +17,8 @@ class StreamSignaler {
         this.topic = config.topic;
         this.stream = stream;
         this.store = store;
-        this.reclaimDelay = config.reclaimDelay || XCLAIM_DELAY_MS;
-        this.reclaimCount = config.reclaimCount || XCLAIM_COUNT;
+        this.reclaimDelay = config.reclaimDelay || enums_1.XCLAIM_DELAY_MS;
+        this.reclaimCount = config.reclaimCount || enums_1.XCLAIM_COUNT;
         this.logger = logger;
     }
     async createGroup(stream, group) {
@@ -59,7 +48,7 @@ class StreamSignaler {
                 return;
             }
             try {
-                const result = await this.stream.xreadgroup('GROUP', group, consumer, 'BLOCK', BLOCK_TIME_MS, 'STREAMS', stream, '>');
+                const result = await this.stream.xreadgroup('GROUP', group, consumer, 'BLOCK', enums_1.BLOCK_TIME_MS, 'STREAMS', stream, '>');
                 if (this.isStreamMessage(result)) {
                     const [[, messages]] = result;
                     for (const [id, message] of messages) {
@@ -81,7 +70,7 @@ class StreamSignaler {
                 if (this.shouldConsume && process.env.NODE_ENV !== 'test') {
                     this.logger.error(`stream-consume-message-error`, { err, stream, group, consumer });
                     this.errorCount++;
-                    const timeout = Math.min(GRADUATED_INTERVAL_MS * (2 ** this.errorCount), MAX_TIMEOUT_MS);
+                    const timeout = Math.min(enums_1.GRADUATED_INTERVAL_MS * (2 ** this.errorCount), enums_1.MAX_TIMEOUT_MS);
                     setTimeout(consume.bind(this), timeout);
                 }
             }
@@ -101,7 +90,7 @@ class StreamSignaler {
             telemetry.startStreamSpan(input, this.role);
             output = await this.execStreamLeg(input, stream, id, callback.bind(this));
             if (output?.status === stream_1.StreamStatus.ERROR) {
-                telemetry.setStreamError(`Function Status Code ${output.code || UNKNOWN_STATUS_CODE}`);
+                telemetry.setStreamError(`Function Status Code ${output.code || enums_1.STATUS_CODE_UNKNOWN}`);
             }
             this.errorCount = 0;
         }
@@ -140,6 +129,7 @@ class StreamSignaler {
                     await (0, utils_1.sleepFor)(timeout);
                     return await this.publishMessage(input.metadata.topic, {
                         data: input.data,
+                        //note: retain guid (this is a retry attempt)
                         metadata: { ...input.metadata, try: (input.metadata.try || 0) + 1 },
                         policies: input.policies,
                     });
@@ -148,6 +138,12 @@ class StreamSignaler {
                     output = this.structureError(input, output);
                 }
             }
+            else if (typeof output.metadata !== 'object') {
+                output.metadata = { ...input.metadata, guid: (0, utils_1.guid)() };
+            }
+            else {
+                output.metadata.guid = (0, utils_1.guid)();
+            }
             output.type = stream_1.StreamDataType.RESPONSE;
             return await this.publishMessage(null, output);
         }
@@ -157,7 +153,7 @@ class StreamSignaler {
         const errorCode = output.code.toString();
         const policy = policies?.[errorCode];
         const maxRetries = policy?.[0];
-        const tryCount = Math.min(input.metadata.try || 0, MAX_RETRIES);
+        const tryCount = Math.min(input.metadata.try || 0, enums_1.MAX_RETRIES);
         //only possible values for maxRetries are 1, 2, 3
         //only possible values for tryCount are 0, 1, 2
         if (maxRetries > tryCount) {
@@ -172,7 +168,7 @@ class StreamSignaler {
             error.message = err.message;
         }
         else {
-            error.message = UNKNOWN_STATUS_MESSAGE;
+            error.message = enums_1.STATUS_MESSAGE_UNKNOWN;
         }
         if (typeof err.stack === 'string') {
             error.stack = err.stack;
@@ -182,17 +178,17 @@ class StreamSignaler {
         }
         return {
             status: 'error',
-            code: UNKNOWN_STATUS_CODE,
-            metadata: { ...input.metadata },
+            code: enums_1.STATUS_CODE_UNKNOWN,
+            metadata: { ...input.metadata, guid: (0, utils_1.guid)() },
             data: error
         };
     }
     structureUnacknowledgedError(input) {
         const message = 'stream message max delivery count exceeded';
-        const code = SYSTEM_STATUS_CODE;
+        const code = enums_1.STATUS_CODE_UNACKED;
         const data = { message, code };
         const output = {
-            metadata: { ...input.metadata },
+            metadata: { ...input.metadata, guid: (0, utils_1.guid)() },
             status: stream_1.StreamStatus.ERROR,
             code,
             data,
@@ -202,9 +198,9 @@ class StreamSignaler {
         return output;
     }
     structureError(input, output) {
-        const message = output.data?.message ? output.data?.message.toString() : UNKNOWN_STATUS_MESSAGE;
+        const message = output.data?.message ? output.data?.message.toString() : enums_1.STATUS_MESSAGE_UNKNOWN;
         const statusCode = output.code || output.data?.code;
-        const code = isNaN(statusCode) ? UNKNOWN_STATUS_CODE : parseInt(statusCode.toString());
+        const code = isNaN(statusCode) ? enums_1.STATUS_CODE_UNKNOWN : parseInt(statusCode.toString());
         const data = { message, code };
         if (typeof output.data?.error === 'object') {
             data.error = { ...output.data.error };
@@ -212,7 +208,7 @@ class StreamSignaler {
         return {
             status: stream_1.StreamStatus.ERROR,
             code,
-            metadata: { ...input.metadata },
+            metadata: { ...input.metadata, guid: (0, utils_1.guid)() },
             data
         };
     }
@@ -220,7 +216,7 @@ class StreamSignaler {
         for (const instance of [...StreamSignaler.signalers]) {
             instance.stopConsuming();
         }
-        await (0, utils_1.sleepFor)(BLOCK_TIME_MS);
+        await (0, utils_1.sleepFor)(enums_1.BLOCK_TIME_MS);
     }
     async stopConsuming() {
         this.shouldConsume = false;
@@ -240,7 +236,7 @@ class StreamSignaler {
         this.throttle = delayInMillis;
         this.logger.info(`stream-throttle-reset`, { delay: this.throttle, topic: this.topic });
     }
-    async claimUnacknowledged(stream, group, consumer, idleTimeMs = this.reclaimDelay, limit = XPENDING_COUNT) {
+    async claimUnacknowledged(stream, group, consumer, idleTimeMs = this.reclaimDelay, limit = enums_1.XPENDING_COUNT) {
         let pendingMessages = [];
         const pendingMessagesInfo = await this.stream.xpending(stream, group, '-', '+', limit); //[[ '1688768134881-0', 'testConsumer1', 1017, 1 ]]
         for (const pendingMessageInfo of pendingMessagesInfo) {
@@ -269,7 +265,7 @@ class StreamSignaler {
         //     ii) corrupt hardware/network/transport/etc
         // 3b) system error: Redis unable to accept `xadd` request
         // 4c) system error: Redis unable to accept `xdel`/`xack` request
-        this.logger.error('stream-message-max-delivery-count-exceeded', { id, stream, group, consumer, code: SYSTEM_STATUS_CODE, count });
+        this.logger.error('stream-message-max-delivery-count-exceeded', { id, stream, group, consumer, code: enums_1.STATUS_CODE_UNACKED, count });
         const streamData = reclaimedMessage[0]?.[1]?.[1];
         //fatal risk point 1 of 3): json is corrupt
         const [err, input] = this.parseStreamData(streamData);

package/build/services/store/clients/ioredis.js

@@ -21,6 +21,7 @@ class IORedisStoreService extends index_1.StoreService {
             }
             return response;
         }
+        return response;
     }
     hGetAllResult(result) {
         //ioredis response signature is [null, {}] or [null, null]

package/build/services/store/index.d.ts

@@ -10,6 +10,7 @@ import { SymbolSets, StringStringType, StringAnyType, Symbols } from '../../type
 import { IdsData, JobStatsRange, StatsType } from '../../types/stats';
 import { Transitions } from '../../types/transition';
 import { ReclaimedMessageType } from '../../types/stream';
+import { JobCompletionOptions, JobInterruptOptions } from '../../types/job';
 interface AbstractRedisClient {
     exec(): any;
 }
@@ -54,6 +55,13 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
     setApp(id: string, version: string): Promise<HotMeshApp>;
     activateAppVersion(id: string, version: string): Promise<boolean>;
     registerAppVersion(appId: string, version: string): Promise<any>;
+    /**
+     * Registers jobId with the originJobId that spawned it. In the future,
+     * when originJobId is interrupted or expired, the items in the
+     * list (added via RPUSH) are LPOPed. If origin was expired, then
+     * LPOPed items from the list are likewise expired;
+     */
+    setDependency(originJobId: string, topic: string, jobId: string, multi?: U): Promise<any>;
     setStats(jobKey: string, jobId: string, dateTime: string, stats: StatsType, appVersion: AppVID, multi?: U): Promise<any>;
     hGetAllResult(result: any): any;
     getJobStats(jobKeys: string[]): Promise<JobStatsRange>;
@@ -62,8 +70,8 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
     getStatus(jobId: string, appId: string): Promise<number>;
     setState({ ...state }: StringAnyType, status: number | null, jobId: string, symbolNames: string[], dIds: StringStringType, multi?: U): Promise<string>;
     /**
-     * returns custom search fields and values. The fields param
-     * should not prefix items with an underscore.
+     * Returns custom search fields and values.
+     * NOTE: The `fields` param should NOT prefix items with an underscore.
      */
     getQueryState(jobId: string, fields: string[]): Promise<StringAnyType>;
     getState(jobId: string, consumes: Consumes, dIds: StringStringType): Promise<[StringAnyType, number] | undefined>;
@@ -87,8 +95,36 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
     deleteProcessedTaskQueue(workItemKey: string, key: string, processedKey: string, scrub?: boolean): Promise<void>;
     processTaskQueue(sourceKey: string, destinationKey: string): Promise<any>;
     expireJob(jobId: string, inSeconds: number): Promise<void>;
-    registerTimeHook(jobId: string, activityId: string, type: 'sleep' | 'expire' | 'cron', deletionTime: number, multi?: U): Promise<void>;
-    getNextTimeJob(listKey?: string): Promise<[listKey: string, jobId: string, activityId: string] | void>;
+    /**
+     * register the descendants of an expired origin flow to be
+     * expired at a future date; options indicate whether this
+     * is a standard `expire` or an `interrupt`
+     */
+    registerExpireJob(jobId: string, deletionTime: number, options: JobCompletionOptions): Promise<void>;
+    /**
+     * registers a hook activity to be awakened (uses ZSET to
+     * store the 'sleep group' and LIST to store the events
+     * for the given sleep group. Sleep groups are
+     * organized into 'n'-second blocks (LISTS))
+     */
+    registerTimeHook(jobId: string, activityId: string, type: 'sleep' | 'expire' | 'interrupt', deletionTime: number, multi?: U): Promise<void>;
+    getNextTimeJob(listKey?: string): Promise<[listKey: string, jobId: string, activityId: string, type: 'sleep' | 'expire' | 'interrupt'] | void>;
+    /**
+     * when processing time jobs, the target LIST ID returned
+     * from the ZSET query can be prefixed to denote what to
+     * do with the work list. (not everything is known in advance,
+     * so the ZSET key defines HOW to approach the work in the
+     * generic LIST (lists typically contain target job ids)
+     * @param {string} listKey - for example `::INTERRUPT::job123` or `job123`
+     */
+    resolveKeyContext(listKey: string): [('sleep' | 'expire' | 'interrupt'), string];
+    /**
+     * Interrupts a job and sets sets a job error (410), if 'throw'!=false.
+     * This method is called by the engine and not by an activity and is
+     * followed by a call to execute job completion/cleanup tasks
+     * associated with a job completion event.
+     */
+    interrupt(topic: string, jobId: string, options?: JobInterruptOptions): Promise<void>;
     scrub(jobId: string): Promise<void>;
 }
 export { StoreService };

package/build/services/store/index.js

@@ -28,6 +28,8 @@ const key_1 = require("../../modules/key");
 const serializer_1 = require("../serializer");
 const cache_1 = require("./cache");
 const utils_1 = require("../../modules/utils");
+const enums_1 = require("../../modules/enums");
+const errors_1 = require("../../modules/errors");
 class StoreService {
     constructor(redisClient) {
         this.commands = {
@@ -296,6 +298,21 @@ class StoreService {
         };
         return await this.redisClient[this.commands.hset](key, payload);
     }
+    /**
+     * Registers jobId with the originJobId that spawned it. In the future,
+     * when originJobId is interrupted or expired, the items in the
+     * list (added via RPUSH) are LPOPed. If origin was expired, then
+     * LPOPed items from the list are likewise expired;
+     */
+    async setDependency(originJobId, topic, jobId, multi) {
+        const privateMulti = multi || this.getMulti();
+        const depParams = { appId: this.appId, jobId: originJobId };
+        const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, depParams);
+        privateMulti[this.commands.rpush](depKey, `expire::${topic}::${jobId}`);
+        if (!multi) {
+            return await privateMulti.exec();
+        }
+    }
     async setStats(jobKey, jobId, dateTime, stats, appVersion, multi) {
         const params = { appId: appVersion.id, jobId, jobKey, dateTime };
         const privateMulti = multi || this.getMulti();
@@ -395,8 +412,8 @@ class StoreService {
         return jobId;
     }
     /**
-     * returns custom search fields and values. The fields param
-     * should not prefix items with an underscore.
+     * Returns custom search fields and values.
+     * NOTE: The `fields` param should NOT prefix items with an underscore.
      */
     async getQueryState(jobId, fields) {
         const key = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
@@ -437,7 +454,7 @@ class StoreService {
             return [state, status];
         }
         else {
-            throw new Error(`Job ${jobId} not found`);
+            throw new errors_1.GetStateError(jobId);
         }
     }
     async collate(jobId, activityId, amount, dIds, multi) {
@@ -646,6 +663,25 @@ class StoreService {
             await this.redisClient[this.commands.expire](jobKey, inSeconds);
         }
     }
+    /**
+     * register the descendants of an expired origin flow to be
+     * expired at a future date; options indicate whether this
+     * is a standard `expire` or an `interrupt`
+     */
+    async registerExpireJob(jobId, deletionTime, options) {
+        const depParams = { appId: this.appId, jobId };
+        const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, depParams);
+        const context = options.interrupt ? 'INTERRUPT' : 'EXPIRE';
+        const depKeyContext = `::${context}::${depKey}`;
+        const zsetKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId });
+        await this.zAdd(zsetKey, deletionTime.toString(), depKeyContext);
+    }
+    /**
+     * registers a hook activity to be awakened (uses ZSET to
+     * store the 'sleep group' and LIST to store the events
+     * for the given sleep group. Sleep groups are
+     * organized into 'n'-second blocks (LISTS))
+     */
     async registerTimeHook(jobId, activityId, type, deletionTime, multi) {
         const listKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId, timeValue: deletionTime });
         const timeEvent = `${type}::${activityId}::${jobId}`;
@@ -657,18 +693,87 @@ class StoreService {
     }
     async getNextTimeJob(listKey) {
         const zsetKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId });
-        const now = Date.now();
-        listKey = listKey || await this.zRangeByScore(zsetKey, 0, now);
+        listKey = listKey || await this.zRangeByScore(zsetKey, 0, Date.now());
         if (listKey) {
-            const timeEvent = await this.redisClient[this.commands.lpop](listKey);
+            const [pType, pKey] = this.resolveKeyContext(listKey);
+            const timeEvent = await this.redisClient[this.commands.lpop](pKey);
             if (timeEvent) {
-                //placeholder: there are 3 time-related event triggers: sleep, expire, cron
-                const [type, activityId, ...jobId] = timeEvent.split('::');
-                return [listKey, jobId.join('::'), activityId];
+                //there are 3 time-related event triggers: sleep, expire, interrupt
+                const [_type, activityId, ...jobId] = timeEvent.split('::');
+                return [listKey, jobId.join('::'), activityId, pType];
             }
             await this.redisClient[this.commands.zrem](zsetKey, listKey);
         }
     }
+    /**
+     * when processing time jobs, the target LIST ID returned
+     * from the ZSET query can be prefixed to denote what to
+     * do with the work list. (not everything is known in advance,
+     * so the ZSET key defines HOW to approach the work in the
+     * generic LIST (lists typically contain target job ids)
+     * @param {string} listKey - for example `::INTERRUPT::job123` or `job123`
+     */
+    resolveKeyContext(listKey) {
+        if (listKey.startsWith('::INTERRUPT')) {
+            return ['interrupt', listKey.split('::')[2]];
+        }
+        else if (listKey.startsWith('::EXPIRE')) {
+            return ['expire', listKey.split('::')[2]];
+        }
+        else {
+            return ['sleep', listKey];
+        }
+    }
+    /**
+     * Interrupts a job and sets sets a job error (410), if 'throw'!=false.
+     * This method is called by the engine and not by an activity and is
+     * followed by a call to execute job completion/cleanup tasks
+     * associated with a job completion event.
+     */
+    async interrupt(topic, jobId, options = {}) {
+        try {
+            //verify job exists
+            const status = await this.getStatus(jobId, this.appId);
+            if (status <= 0) {
+                //verify still active; job already completed
+                throw new Error(`Job ${jobId} already completed`);
+            }
+            //decrement job status (:) by 1bil
+            const amount = -1000000000;
+            const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
+            const result = await this.redisClient[this.commands.hincrbyfloat](jobKey, ':', amount);
+            if (result <= amount) {
+                //verify active state; job already interrupted
+                throw new Error(`Job ${jobId} already completed`);
+            }
+            //persist the error unless specifically told not to
+            if (options.throw !== false) {
+                const errKey = `metadata/err`; //job errors are stored at the path `metadata/err`
+                const symbolNames = [`$${topic}`]; //the symbol for `metadata/err` is in redis and stored using the job topic
+                const symKeys = await this.getSymbolKeys(symbolNames);
+                const symVals = await this.getSymbolValues();
+                this.serializer.resetSymbols(symKeys, symVals, {});
+                //persists the standard 410 error (job is `gone`)
+                const err = JSON.stringify({
+                    code: enums_1.STATUS_CODE_INTERRUPT,
+                    message: options.reason ?? `job [${jobId}] interrupted`,
+                    job_id: jobId
+                });
+                const payload = { [errKey]: amount.toString() };
+                const hashData = this.serializer.package(payload, symbolNames);
+                const errSymbol = Object.keys(hashData)[0];
+                await this.redisClient[this.commands.hset](jobKey, errSymbol, err);
+            }
+        }
+        catch (e) {
+            if (!options.suppress) {
+                throw e;
+            }
+            else {
+                this.logger.debug('suppressed-interrupt', { message: e.message });
+            }
+        }
+    }
     async scrub(jobId) {
         const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
         await this.redisClient[this.commands.del](jobKey);

package/build/services/task/index.d.ts

@@ -1,8 +1,9 @@
 /// <reference types="node" />
 import { ILogger } from '../logger';
 import { StoreService } from '../store';
-import { RedisClient, RedisMulti } from '../../types/redis';
 import { HookInterface } from '../../types/hook';
+import { JobCompletionOptions } from '../../types/job';
+import { RedisClient, RedisMulti } from '../../types/redis';
 declare class TaskService {
     store: StoreService<RedisClient, RedisMulti>;
     logger: ILogger;
@@ -10,9 +11,9 @@ declare class TaskService {
     constructor(store: StoreService<RedisClient, RedisMulti>, logger: ILogger);
     processWebHooks(hookEventCallback: HookInterface): Promise<void>;
     enqueueWorkItems(keys: string[]): Promise<void>;
-    registerJobForCleanup(jobId: string, inSeconds?: number): Promise<void>;
-    registerTimeHook(jobId: string, activityId: string, type: 'sleep' | 'expire' | 'cron', inSeconds?: number, multi?: RedisMulti): Promise<void>;
-    processTimeHooks(timeEventCallback: (jobId: string, activityId: string) => Promise<void>, listKey?: string): Promise<void>;
+    registerJobForCleanup(jobId: string, inSeconds: number, options: JobCompletionOptions): Promise<void>;
+    registerTimeHook(jobId: string, activityId: string, type: 'sleep' | 'expire' | 'interrupt', inSeconds?: number, multi?: RedisMulti): Promise<void>;
+    processTimeHooks(timeEventCallback: (jobId: string, activityId: string, type: 'sleep' | 'expire' | 'interrupt') => Promise<void>, listKey?: string): Promise<void>;
     cancelCleanup(): void;
 }
 export { TaskService };

package/build/services/task/index.js

@@ -1,11 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.TaskService = void 0;
+const enums_1 = require("../../modules/enums");
 const utils_1 = require("../../modules/utils");
-//system timer granularity limit (task queues organize)
-const FIDELITY_SECONDS = 15; //note: this can be reduced using 'watch' or scout role
-//default resolution/fidelity when expiring
-const EXPIRATION_FIDELITY_SECONDS = 60;
 class TaskService {
     constructor(store, logger) {
         this.cleanupTimeout = null;
@@ -32,27 +29,28 @@ class TaskService {
     async enqueueWorkItems(keys) {
         await this.store.addTaskQueues(keys);
     }
-    async registerJobForCleanup(jobId, inSeconds = EXPIRATION_FIDELITY_SECONDS) {
-        if (inSeconds > -1) {
+    async registerJobForCleanup(jobId, inSeconds = enums_1.EXPIRE_DURATION, options) {
+        if (inSeconds > 0) {
             await this.store.expireJob(jobId, inSeconds);
+            const expireTimeSlot = Math.floor((Date.now() + (inSeconds * 1000)) / (enums_1.FIDELITY_SECONDS * 1000)) * (enums_1.FIDELITY_SECONDS * 1000); //n second awaken groups
+            await this.store.registerExpireJob(jobId, expireTimeSlot, options);
         }
     }
-    async registerTimeHook(jobId, activityId, type, inSeconds = FIDELITY_SECONDS, multi) {
-        const awakenTimeSlot = Math.floor((Date.now() + inSeconds * 1000) / (FIDELITY_SECONDS * 1000)) * (FIDELITY_SECONDS * 1000); //n second awaken groups
+    async registerTimeHook(jobId, activityId, type, inSeconds = enums_1.FIDELITY_SECONDS, multi) {
+        const awakenTimeSlot = Math.floor((Date.now() + (inSeconds * 1000)) / (enums_1.FIDELITY_SECONDS * 1000)) * (enums_1.FIDELITY_SECONDS * 1000); //n second awaken groups
         await this.store.registerTimeHook(jobId, activityId, type, awakenTimeSlot, multi);
     }
-    //todo: need 'scout' role in quorum to check for this and then alert the quorum to get to work
     async processTimeHooks(timeEventCallback, listKey) {
         try {
-            const job = await this.store.getNextTimeJob(listKey);
-            if (job) {
-                const [listKey, jobId, activityId] = job;
-                await timeEventCallback(jobId, activityId);
+            const timeJob = await this.store.getNextTimeJob(listKey);
+            if (timeJob) {
+                const [listKey, jobId, activityId, type] = timeJob;
+                await timeEventCallback(jobId, activityId, type);
                 await (0, utils_1.sleepFor)(0);
                 this.processTimeHooks(timeEventCallback, listKey);
             }
             else {
-                let sleep = (0, utils_1.XSleepFor)(FIDELITY_SECONDS * 1000);
+                let sleep = (0, utils_1.XSleepFor)(enums_1.FIDELITY_SECONDS * 1000);
                 this.cleanupTimeout = sleep.timerId;
                 await sleep.promise;
                 this.processTimeHooks(timeEventCallback);

package/build/types/activity.d.ts

@@ -1,6 +1,6 @@
 import { MetricTypes } from "./stats";
 import { StreamRetryPolicy } from "./stream";
-type ActivityExecutionType = 'trigger' | 'await' | 'worker' | 'activity' | 'emit' | 'iterate' | 'cycle' | 'signal' | 'hook';
+type ActivityExecutionType = 'trigger' | 'await' | 'worker' | 'activity' | 'emit' | 'interrupt' | 'cycle' | 'signal' | 'hook';
 type Consumes = Record<string, string[]>;
 interface BaseActivity {
     title?: string;
@@ -32,13 +32,33 @@ interface Measure {
     target: string;
 }
 interface TriggerActivityStats {
+    /**
+     * parent job; including this allows the parent's
+     * expiration/interruption events to cascade; set
+     * `expire` in the YAML for the dependent graph
+     * to 0 and provide the parent for dependent,
+     * cascading interruption and cleanup
+     */
+    parent?: string;
     id?: {
         [key: string]: unknown;
     } | string;
     key?: {
         [key: string]: unknown;
     } | string;
+    /**
+     * @deprecated
+     * return 'infinity' to disable; default behavior
+     * is to always segment keys by time to ensure
+     * indexes (Redis LIST) never grow unbounded
+     * as a default behavior; for now, 5m is default
+     * and infinity can be set to override
+     */
     granularity?: string;
+    /**
+     * @deprecated
+     * what to capture
+     */
     measures?: Measure[];
 }
 interface TriggerActivity extends BaseActivity {
@@ -74,10 +94,20 @@ interface SignalActivity extends BaseActivity {
     status?: string;
     code?: number;
 }
-interface IterateActivity extends BaseActivity {
-    type: 'iterate';
+interface InterruptActivity extends BaseActivity {
+    type: 'interrupt';
+    /** Optional Reason; will be used as the error `message` when thrown; NOTE: 410 is the error `code` */
+    reason?: string;
+    /** default is `true` (throw JobInterrupted error upon interrupting) */
+    throw?: boolean;
+    /** TODO: // default is `false` (do not interrupt child jobs) */
+    descend?: boolean;
+    /** target job id (if not present the current job will be targeted) */
+    target?: string;
+    /** topic to publish the interrupt message (if not present the current job topic will be used) */
+    topic?: string;
 }
-type ActivityType = BaseActivity | TriggerActivity | AwaitActivity | WorkerActivity | IterateActivity | HookActivity;
+type ActivityType = BaseActivity | TriggerActivity | AwaitActivity | WorkerActivity | InterruptActivity | HookActivity | SignalActivity | CycleActivity;
 type ActivityData = Record<string, any>;
 type ActivityMetadata = {
     aid: string;
@@ -103,4 +133,4 @@ type ActivityDataType = {
     hook?: Record<string, unknown>;
 };
 type ActivityLeg = 1 | 2;
-export { ActivityContext, ActivityData, ActivityDataType, ActivityDuplex, ActivityLeg, ActivityMetadata, ActivityType, Consumes, TriggerActivityStats, AwaitActivity, CycleActivity, HookActivity, SignalActivity, BaseActivity, IterateActivity, TriggerActivity, WorkerActivity };
+export { ActivityContext, ActivityData, ActivityDataType, ActivityDuplex, ActivityLeg, ActivityMetadata, ActivityType, Consumes, TriggerActivityStats, AwaitActivity, CycleActivity, HookActivity, SignalActivity, BaseActivity, InterruptActivity, TriggerActivity, WorkerActivity };

package/build/types/durable.d.ts

@@ -52,10 +52,12 @@ type WorkflowOptions = {
     entity?: string;
     workflowName?: string;
     parentWorkflowId?: string;
+    originJobId?: string;
     workflowTrace?: string;
     workflowSpan?: string;
     search?: WorkflowSearchOptions;
     config?: WorkflowConfig;
+    expire?: number;
 };
 type HookOptions = {
     namespace?: string;
@@ -83,6 +85,8 @@ type WorkflowDataType = {
     arguments: any[];
     workflowId: string;
     workflowTopic: string;
+    workflowDimension?: string;
+    originJobId?: string;
 };
 type MeshOSClassConfig = {
     namespace: string;

package/build/types/index.d.ts

@@ -1,4 +1,4 @@
-export { ActivityType, ActivityDataType, ActivityContext, ActivityData, ActivityDuplex, ActivityLeg, ActivityMetadata, Consumes, AwaitActivity, BaseActivity, CycleActivity, HookActivity, WorkerActivity, IterateActivity, SignalActivity, TriggerActivity, TriggerActivityStats } from './activity';
+export { ActivityType, ActivityDataType, ActivityContext, ActivityData, ActivityDuplex, ActivityLeg, ActivityMetadata, Consumes, AwaitActivity, BaseActivity, CycleActivity, HookActivity, WorkerActivity, InterruptActivity, SignalActivity, TriggerActivity, TriggerActivityStats } from './activity';
 export { App, AppVID, AppTransitions, AppSubscriptions } from './app';
 export { AsyncSignal } from './async';
 export { CacheMode } from './cache';

package/build/types/job.d.ts

@@ -39,6 +39,18 @@ type JobState = {
         errors: ActivityData;
     };
 };
+type JobInterruptOptions = {
+    /** Optional reason when throwing the error  */
+    reason?: string;
+    /** default is `true` when `undefined` (throw JobInterrupted/410 error) */
+    throw?: boolean;
+    /** default behavior is `false` when `undefined` (do NOT interrupt child jobs) */
+    descend?: boolean;
+    /** default is false; if true, errors related to inactivation (like overage...already inactive) are suppressed/ignored */
+    suppress?: boolean;
+    /** how long to wait in seconds before fully expiring/removing the hash from Redis; the job is inactive, but can remain in the cache indefinitely. minimum 1 second.*/
+    expire?: number;
+};
 type JobOutput = {
     metadata: JobMetadata;
     data: JobData;
@@ -47,4 +59,9 @@ type PartialJobState = {
     metadata: JobMetadata | Pick<JobMetadata, 'jid' | 'dad' | 'aid'>;
     data: JobData;
 };
-export { JobState, JobStatus, JobData, JobsData, JobMetadata, PartialJobState, JobOutput };
+type JobCompletionOptions = {
+    emit?: boolean;
+    interrupt?: boolean;
+    expire?: number;
+};
+export { JobCompletionOptions, JobInterruptOptions, JobData, JobsData, JobMetadata, JobOutput, JobState, JobStatus, PartialJobState, };

package/build/types/quorum.d.ts

@@ -1,11 +1,4 @@
 import { JobOutput } from "./job";
-/**
- * The types in this file are used to define those messages that are sent
- * to hotmesh client instances when a new version is about to be activated.
- * These messages serve to coordinate the cache invalidation and switch-over
- * to the new version without any downtime and a coordinating parent server.
- */
-export type QuorumMessage = PingMessage | PongMessage | ActivateMessage | WorkMessage | JobMessage | ThrottleMessage;
 export interface PingMessage {
     type: 'ping';
     originator: string;
@@ -14,6 +7,10 @@ export interface WorkMessage {
     type: 'work';
     originator: string;
 }
+export interface CronMessage {
+    type: 'cron';
+    originator: string;
+}
 export interface PongMessage {
     type: 'pong';
     originator: string;
@@ -44,3 +41,10 @@ export interface SubscriptionCallback {
 export interface QuorumMessageCallback {
     (topic: string, message: QuorumMessage): void;
 }
+/**
+ * The types in this file are used to define those messages that are sent
+ * to hotmesh client instances when a new version is about to be activated.
+ * These messages serve to coordinate the cache invalidation and switch-over
+ * to the new version without any downtime and a coordinating parent server.
+ */
+export type QuorumMessage = PingMessage | PongMessage | ActivateMessage | WorkMessage | JobMessage | ThrottleMessage | CronMessage;