@hotmeshio/hotmesh 0.0.33 → 0.0.35

package/build/services/hotmesh/index.js

@@ -1,8 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.HotMeshService = void 0;
-const nanoid_1 = require("nanoid");
 const key_1 = require("../../modules/key");
+const utils_1 = require("../../modules/utils");
+const redis_1 = require("../connector/clients/redis");
+const ioredis_1 = require("../connector/clients/ioredis");
 const engine_1 = require("../engine");
 const logger_1 = require("../logger");
 const stream_1 = require("../signaler/stream");
@@ -39,7 +41,7 @@ class HotMeshService {
     }
     static async init(config) {
         const instance = new HotMeshService();
-        instance.guid = (0, nanoid_1.nanoid)();
+        instance.guid = (0, utils_1.guid)();
         instance.verifyAndSetNamespace(config.namespace);
         instance.verifyAndSetAppId(config.appId);
         instance.logger = new logger_1.LoggerService(config.appId, instance.guid, config.name || '', config.logLevel);
@@ -49,7 +51,7 @@ class HotMeshService {
         return instance;
     }
     static guid() {
-        return (0, nanoid_1.nanoid)();
+        return (0, utils_1.guid)();
     }
     async initEngine(config, logger) {
         if (config.engine) {
@@ -98,6 +100,10 @@ class HotMeshService {
         //activation is a quorum operation
         return await this.quorum?.activate(version, delay);
     }
+    async inventory(version, delay) {
+        //get count of all peers
+        return await this.quorum?.inventory(delay);
+    }
     // ************* REPORTER METHODS *************
     async getStats(topic, query) {
         return await this.engine?.getStats(topic, query);
@@ -117,6 +123,10 @@ class HotMeshService {
     async resolveQuery(topic, query) {
         return await this.engine?.resolveQuery(topic, query);
     }
+    // ****************** `INTERRUPT` ACTIVE JOBS *****************
+    async interrupt(topic, jobId, options = {}) {
+        return await this.engine?.interrupt(topic, jobId, options);
+    }
     // ****************** `SCRUB` CLEAN COMPLETED JOBS *****************
     async scrub(jobId) {
         await this.engine?.scrub(jobId);
@@ -128,8 +138,15 @@ class HotMeshService {
     async hookAll(hookTopic, data, query, queryFacets = []) {
         return await this.engine?.hookAll(hookTopic, data, query, queryFacets);
     }
-    async stop() {
-        await stream_1.StreamSignaler.stopConsuming();
+    static async stop() {
+        if (!this.disconnecting) {
+            this.disconnecting = true;
+            await stream_1.StreamSignaler.stopConsuming();
+            await redis_1.RedisConnection.disconnectAll();
+            await ioredis_1.RedisConnection.disconnectAll();
+        }
+    }
+    stop() {
         this.engine?.task.cancelCleanup();
     }
     async compress(terms) {
@@ -137,3 +154,4 @@ class HotMeshService {
     }
 }
 exports.HotMeshService = HotMeshService;
+HotMeshService.disconnecting = false;

package/build/services/quorum/index.d.ts

@@ -29,6 +29,7 @@ declare class QuorumService {
     pub(quorumMessage: ThrottleMessage): Promise<boolean>;
     sub(callback: QuorumMessageCallback): Promise<void>;
     unsub(callback: QuorumMessageCallback): Promise<void>;
+    inventory(delay?: number): Promise<number>;
     activate(version: string, delay?: number): Promise<boolean>;
 }
 export { QuorumService };

package/build/services/quorum/index.js

@@ -82,6 +82,9 @@ class QuorumService {
             else if (message.type === 'job') {
                 self.engine.routeToSubscribers(message.topic, message.job);
             }
+            else if (message.type === 'cron') {
+                self.engine.processTimeHooks();
+            }
             //if there are any callbacks, call them
             if (self.callbacks.length > 0) {
                 self.callbacks.forEach(cb => cb(topic, message));
@@ -114,6 +117,13 @@ class QuorumService {
         this.callbacks = this.callbacks.filter(cb => cb !== callback);
     }
     // ************* COMPILER METHODS *************
+    async inventory(delay = QUORUM_DELAY) {
+        await this.requestQuorum(delay);
+        const q1 = await this.requestQuorum(delay);
+        const q2 = await this.requestQuorum(delay);
+        const q3 = await this.requestQuorum(delay);
+        return Math.round((q1 + q2 + q3) / 3);
+    }
     async activate(version, delay = QUORUM_DELAY) {
         version = version.toString();
         const config = await this.engine.getVID();

package/build/services/signaler/stream.js

@@ -1,22 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.StreamSignaler = void 0;
+const enums_1 = require("../../modules/enums");
 const key_1 = require("../../modules/key");
 const utils_1 = require("../../modules/utils");
 const telemetry_1 = require("../telemetry");
 const stream_1 = require("../../types/stream");
-const MAX_RETRIES = 3; //local retry; 10, 100, 1000ms
-const MAX_TIMEOUT_MS = 60000;
-const GRADUATED_INTERVAL_MS = 5000;
-const BLOCK_DURATION = 15000; //Set to `15` so SIGINT/SIGTERM can interrupt; set to `0` to BLOCK indefinitely
-const TEST_BLOCK_DURATION = 1000; //Set to `1000` so tests can interrupt quickly
-const BLOCK_TIME_MS = process.env.NODE_ENV === 'test' ? TEST_BLOCK_DURATION : BLOCK_DURATION;
-const SYSTEM_STATUS_CODE = 999;
-const UNKNOWN_STATUS_CODE = 500;
-const UNKNOWN_STATUS_MESSAGE = 'unknown';
-const XCLAIM_DELAY_MS = 1000 * 60; //max time a message can be unacked before it is claimed by another
-const XCLAIM_COUNT = 3; //max number of times a message can be claimed by another before it is dead-lettered
-const XPENDING_COUNT = 10;
 class StreamSignaler {
     constructor(config, stream, store, logger) {
         this.throttle = 0;
@@ -28,8 +17,8 @@ class StreamSignaler {
         this.topic = config.topic;
         this.stream = stream;
         this.store = store;
-        this.reclaimDelay = config.reclaimDelay || XCLAIM_DELAY_MS;
-        this.reclaimCount = config.reclaimCount || XCLAIM_COUNT;
+        this.reclaimDelay = config.reclaimDelay || enums_1.XCLAIM_DELAY_MS;
+        this.reclaimCount = config.reclaimCount || enums_1.XCLAIM_COUNT;
         this.logger = logger;
     }
     async createGroup(stream, group) {
@@ -59,7 +48,7 @@ class StreamSignaler {
                 return;
             }
             try {
-                const result = await this.stream.xreadgroup('GROUP', group, consumer, 'BLOCK', BLOCK_TIME_MS, 'STREAMS', stream, '>');
+                const result = await this.stream.xreadgroup('GROUP', group, consumer, 'BLOCK', enums_1.BLOCK_TIME_MS, 'STREAMS', stream, '>');
                 if (this.isStreamMessage(result)) {
                     const [[, messages]] = result;
                     for (const [id, message] of messages) {
@@ -81,7 +70,7 @@ class StreamSignaler {
                 if (this.shouldConsume && process.env.NODE_ENV !== 'test') {
                     this.logger.error(`stream-consume-message-error`, { err, stream, group, consumer });
                     this.errorCount++;
-                    const timeout = Math.min(GRADUATED_INTERVAL_MS * (2 ** this.errorCount), MAX_TIMEOUT_MS);
+                    const timeout = Math.min(enums_1.GRADUATED_INTERVAL_MS * (2 ** this.errorCount), enums_1.MAX_TIMEOUT_MS);
                     setTimeout(consume.bind(this), timeout);
                 }
             }
@@ -101,7 +90,7 @@ class StreamSignaler {
             telemetry.startStreamSpan(input, this.role);
             output = await this.execStreamLeg(input, stream, id, callback.bind(this));
             if (output?.status === stream_1.StreamStatus.ERROR) {
-                telemetry.setStreamError(`Function Status Code ${output.code || UNKNOWN_STATUS_CODE}`);
+                telemetry.setStreamError(`Function Status Code ${output.code || enums_1.STATUS_CODE_UNKNOWN}`);
             }
             this.errorCount = 0;
         }
@@ -140,6 +129,7 @@ class StreamSignaler {
                     await (0, utils_1.sleepFor)(timeout);
                     return await this.publishMessage(input.metadata.topic, {
                         data: input.data,
+                        //note: retain guid (this is a retry attempt)
                         metadata: { ...input.metadata, try: (input.metadata.try || 0) + 1 },
                         policies: input.policies,
                     });
@@ -148,6 +138,12 @@ class StreamSignaler {
                     output = this.structureError(input, output);
                 }
             }
+            else if (typeof output.metadata !== 'object') {
+                output.metadata = { ...input.metadata, guid: (0, utils_1.guid)() };
+            }
+            else {
+                output.metadata.guid = (0, utils_1.guid)();
+            }
             output.type = stream_1.StreamDataType.RESPONSE;
             return await this.publishMessage(null, output);
         }
@@ -157,7 +153,7 @@ class StreamSignaler {
         const errorCode = output.code.toString();
         const policy = policies?.[errorCode];
         const maxRetries = policy?.[0];
-        const tryCount = Math.min(input.metadata.try || 0, MAX_RETRIES);
+        const tryCount = Math.min(input.metadata.try || 0, enums_1.MAX_RETRIES);
         //only possible values for maxRetries are 1, 2, 3
         //only possible values for tryCount are 0, 1, 2
         if (maxRetries > tryCount) {
@@ -172,7 +168,7 @@ class StreamSignaler {
             error.message = err.message;
         }
         else {
-            error.message = UNKNOWN_STATUS_MESSAGE;
+            error.message = enums_1.STATUS_MESSAGE_UNKNOWN;
         }
         if (typeof err.stack === 'string') {
             error.stack = err.stack;
@@ -182,17 +178,17 @@ class StreamSignaler {
         }
         return {
             status: 'error',
-            code: UNKNOWN_STATUS_CODE,
-            metadata: { ...input.metadata },
+            code: enums_1.STATUS_CODE_UNKNOWN,
+            metadata: { ...input.metadata, guid: (0, utils_1.guid)() },
             data: error
         };
     }
     structureUnacknowledgedError(input) {
         const message = 'stream message max delivery count exceeded';
-        const code = SYSTEM_STATUS_CODE;
+        const code = enums_1.STATUS_CODE_UNACKED;
         const data = { message, code };
         const output = {
-            metadata: { ...input.metadata },
+            metadata: { ...input.metadata, guid: (0, utils_1.guid)() },
             status: stream_1.StreamStatus.ERROR,
             code,
             data,
@@ -202,9 +198,9 @@ class StreamSignaler {
         return output;
     }
     structureError(input, output) {
-        const message = output.data?.message ? output.data?.message.toString() : UNKNOWN_STATUS_MESSAGE;
+        const message = output.data?.message ? output.data?.message.toString() : enums_1.STATUS_MESSAGE_UNKNOWN;
         const statusCode = output.code || output.data?.code;
-        const code = isNaN(statusCode) ? UNKNOWN_STATUS_CODE : parseInt(statusCode.toString());
+        const code = isNaN(statusCode) ? enums_1.STATUS_CODE_UNKNOWN : parseInt(statusCode.toString());
         const data = { message, code };
         if (typeof output.data?.error === 'object') {
             data.error = { ...output.data.error };
@@ -212,7 +208,7 @@ class StreamSignaler {
         return {
             status: stream_1.StreamStatus.ERROR,
             code,
-            metadata: { ...input.metadata },
+            metadata: { ...input.metadata, guid: (0, utils_1.guid)() },
             data
         };
     }
@@ -220,12 +216,12 @@ class StreamSignaler {
         for (const instance of [...StreamSignaler.signalers]) {
             instance.stopConsuming();
         }
+        await (0, utils_1.sleepFor)(enums_1.BLOCK_TIME_MS);
     }
     async stopConsuming() {
         this.shouldConsume = false;
         this.logger.info(`stream-consumer-stopping`, this.topic ? { topic: this.topic } : undefined);
         this.cancelThrottle();
-        //await sleepFor(BLOCK_TIME_MS);
     }
     cancelThrottle() {
         if (this.currentTimerId !== undefined) {
@@ -240,7 +236,7 @@ class StreamSignaler {
         this.throttle = delayInMillis;
         this.logger.info(`stream-throttle-reset`, { delay: this.throttle, topic: this.topic });
     }
-    async claimUnacknowledged(stream, group, consumer, idleTimeMs = this.reclaimDelay, limit = XPENDING_COUNT) {
+    async claimUnacknowledged(stream, group, consumer, idleTimeMs = this.reclaimDelay, limit = enums_1.XPENDING_COUNT) {
         let pendingMessages = [];
         const pendingMessagesInfo = await this.stream.xpending(stream, group, '-', '+', limit); //[[ '1688768134881-0', 'testConsumer1', 1017, 1 ]]
         for (const pendingMessageInfo of pendingMessagesInfo) {
@@ -269,7 +265,7 @@ class StreamSignaler {
         //     ii) corrupt hardware/network/transport/etc
         // 3b) system error: Redis unable to accept `xadd` request
         // 4c) system error: Redis unable to accept `xdel`/`xack` request
-        this.logger.error('stream-message-max-delivery-count-exceeded', { id, stream, group, consumer, code: SYSTEM_STATUS_CODE, count });
+        this.logger.error('stream-message-max-delivery-count-exceeded', { id, stream, group, consumer, code: enums_1.STATUS_CODE_UNACKED, count });
         const streamData = reclaimedMessage[0]?.[1]?.[1];
         //fatal risk point 1 of 3): json is corrupt
         const [err, input] = this.parseStreamData(streamData);

package/build/services/store/index.d.ts

@@ -10,6 +10,7 @@ import { SymbolSets, StringStringType, StringAnyType, Symbols } from '../../type
 import { IdsData, JobStatsRange, StatsType } from '../../types/stats';
 import { Transitions } from '../../types/transition';
 import { ReclaimedMessageType } from '../../types/stream';
+import { JobCompletionOptions, JobInterruptOptions } from '../../types/job';
 interface AbstractRedisClient {
     exec(): any;
 }
@@ -54,6 +55,13 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
     setApp(id: string, version: string): Promise<HotMeshApp>;
     activateAppVersion(id: string, version: string): Promise<boolean>;
     registerAppVersion(appId: string, version: string): Promise<any>;
+    /**
+     * Registers jobId with the originJobId that spawned it. In the future,
+     * when originJobId is interrupted or expired, the items in the
+     * list (added via RPUSH) are LPOPed. If origin was expired, then
+     * LPOPed items from the list are likewise expired;
+     */
+    setDependency(originJobId: string, topic: string, jobId: string, multi?: U): Promise<any>;
     setStats(jobKey: string, jobId: string, dateTime: string, stats: StatsType, appVersion: AppVID, multi?: U): Promise<any>;
     hGetAllResult(result: any): any;
     getJobStats(jobKeys: string[]): Promise<JobStatsRange>;
@@ -62,8 +70,8 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
     getStatus(jobId: string, appId: string): Promise<number>;
     setState({ ...state }: StringAnyType, status: number | null, jobId: string, symbolNames: string[], dIds: StringStringType, multi?: U): Promise<string>;
     /**
-     * returns custom search fields and values. The fields param
-     * should not prefix items with an underscore.
+     * Returns custom search fields and values.
+     * NOTE: The `fields` param should NOT prefix items with an underscore.
      */
     getQueryState(jobId: string, fields: string[]): Promise<StringAnyType>;
     getState(jobId: string, consumes: Consumes, dIds: StringStringType): Promise<[StringAnyType, number] | undefined>;
@@ -87,8 +95,36 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
     deleteProcessedTaskQueue(workItemKey: string, key: string, processedKey: string, scrub?: boolean): Promise<void>;
     processTaskQueue(sourceKey: string, destinationKey: string): Promise<any>;
     expireJob(jobId: string, inSeconds: number): Promise<void>;
-    registerTimeHook(jobId: string, activityId: string, type: 'sleep' | 'expire' | 'cron', deletionTime: number, multi?: U): Promise<void>;
-    getNextTimeJob(listKey?: string): Promise<[listKey: string, jobId: string, activityId: string] | void>;
+    /**
+     * register the descendants of an expired origin flow to be
+     * expired at a future date; options indicate whether this
+     * is a standard `expire` or an `interrupt`
+     */
+    registerExpireJob(jobId: string, deletionTime: number, options: JobCompletionOptions): Promise<void>;
+    /**
+     * registers a hook activity to be awakened (uses ZSET to
+     * store the 'sleep group' and LIST to store the events
+     * for the given sleep group. Sleep groups are
+     * organized into 'n'-second blocks (LISTS))
+     */
+    registerTimeHook(jobId: string, activityId: string, type: 'sleep' | 'expire' | 'interrupt', deletionTime: number, multi?: U): Promise<void>;
+    getNextTimeJob(listKey?: string): Promise<[listKey: string, jobId: string, activityId: string, type: 'sleep' | 'expire' | 'interrupt'] | void>;
+    /**
+     * when processing time jobs, the target LIST ID returned
+     * from the ZSET query can be prefixed to denote what to
+     * do with the work list. (not everything is known in advance,
+     * so the ZSET key defines HOW to approach the work in the
+     * generic LIST (lists typically contain target job ids)
+     * @param {string} listKey - for example `::INTERRUPT::job123` or `job123`
+     */
+    resolveKeyContext(listKey: string): [('sleep' | 'expire' | 'interrupt'), string];
+    /**
+     * Interrupts a job and sets sets a job error (410), if 'throw'!=false.
+     * This method is called by the engine and not by an activity and is
+     * followed by a call to execute job completion/cleanup tasks
+     * associated with a job completion event.
+     */
+    interrupt(topic: string, jobId: string, options?: JobInterruptOptions): Promise<void>;
     scrub(jobId: string): Promise<void>;
 }
 export { StoreService };

package/build/services/store/index.js

@@ -28,6 +28,8 @@ const key_1 = require("../../modules/key");
 const serializer_1 = require("../serializer");
 const cache_1 = require("./cache");
 const utils_1 = require("../../modules/utils");
+const enums_1 = require("../../modules/enums");
+const errors_1 = require("../../modules/errors");
 class StoreService {
     constructor(redisClient) {
         this.commands = {
@@ -296,6 +298,21 @@ class StoreService {
         };
         return await this.redisClient[this.commands.hset](key, payload);
     }
+    /**
+     * Registers jobId with the originJobId that spawned it. In the future,
+     * when originJobId is interrupted or expired, the items in the
+     * list (added via RPUSH) are LPOPed. If origin was expired, then
+     * LPOPed items from the list are likewise expired;
+     */
+    async setDependency(originJobId, topic, jobId, multi) {
+        const privateMulti = multi || this.getMulti();
+        const depParams = { appId: this.appId, jobId: originJobId };
+        const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, depParams);
+        privateMulti[this.commands.rpush](depKey, `expire::${topic}::${jobId}`);
+        if (!multi) {
+            return await privateMulti.exec();
+        }
+    }
     async setStats(jobKey, jobId, dateTime, stats, appVersion, multi) {
         const params = { appId: appVersion.id, jobId, jobKey, dateTime };
         const privateMulti = multi || this.getMulti();
@@ -395,8 +412,8 @@ class StoreService {
         return jobId;
     }
     /**
-     * returns custom search fields and values. The fields param
-     * should not prefix items with an underscore.
+     * Returns custom search fields and values.
+     * NOTE: The `fields` param should NOT prefix items with an underscore.
      */
     async getQueryState(jobId, fields) {
         const key = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
@@ -437,7 +454,7 @@ class StoreService {
             return [state, status];
         }
         else {
-            throw new Error(`Job ${jobId} not found`);
+            throw new errors_1.GetStateError(jobId);
         }
     }
     async collate(jobId, activityId, amount, dIds, multi) {
@@ -646,6 +663,25 @@ class StoreService {
             await this.redisClient[this.commands.expire](jobKey, inSeconds);
         }
     }
+    /**
+     * register the descendants of an expired origin flow to be
+     * expired at a future date; options indicate whether this
+     * is a standard `expire` or an `interrupt`
+     */
+    async registerExpireJob(jobId, deletionTime, options) {
+        const depParams = { appId: this.appId, jobId };
+        const depKey = this.mintKey(key_1.KeyType.JOB_DEPENDENTS, depParams);
+        const context = options.interrupt ? 'INTERRUPT' : 'EXPIRE';
+        const depKeyContext = `::${context}::${depKey}`;
+        const zsetKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId });
+        await this.zAdd(zsetKey, deletionTime.toString(), depKeyContext);
+    }
+    /**
+     * registers a hook activity to be awakened (uses ZSET to
+     * store the 'sleep group' and LIST to store the events
+     * for the given sleep group. Sleep groups are
+     * organized into 'n'-second blocks (LISTS))
+     */
     async registerTimeHook(jobId, activityId, type, deletionTime, multi) {
         const listKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId, timeValue: deletionTime });
         const timeEvent = `${type}::${activityId}::${jobId}`;
@@ -657,18 +693,87 @@ class StoreService {
     }
     async getNextTimeJob(listKey) {
         const zsetKey = this.mintKey(key_1.KeyType.TIME_RANGE, { appId: this.appId });
-        const now = Date.now();
-        listKey = listKey || await this.zRangeByScore(zsetKey, 0, now);
+        listKey = listKey || await this.zRangeByScore(zsetKey, 0, Date.now());
         if (listKey) {
-            const timeEvent = await this.redisClient[this.commands.lpop](listKey);
+            const [pType, pKey] = this.resolveKeyContext(listKey);
+            const timeEvent = await this.redisClient[this.commands.lpop](pKey);
             if (timeEvent) {
-                //placeholder: there are 3 time-related event triggers: sleep, expire, cron
-                const [type, activityId, ...jobId] = timeEvent.split('::');
-                return [listKey, jobId.join('::'), activityId];
+                //there are 3 time-related event triggers: sleep, expire, interrupt
+                const [_type, activityId, ...jobId] = timeEvent.split('::');
+                return [listKey, jobId.join('::'), activityId, pType];
             }
             await this.redisClient[this.commands.zrem](zsetKey, listKey);
         }
     }
+    /**
+     * when processing time jobs, the target LIST ID returned
+     * from the ZSET query can be prefixed to denote what to
+     * do with the work list. (not everything is known in advance,
+     * so the ZSET key defines HOW to approach the work in the
+     * generic LIST (lists typically contain target job ids)
+     * @param {string} listKey - for example `::INTERRUPT::job123` or `job123`
+     */
+    resolveKeyContext(listKey) {
+        if (listKey.startsWith('::INTERRUPT')) {
+            return ['interrupt', listKey.split('::')[2]];
+        }
+        else if (listKey.startsWith('::EXPIRE')) {
+            return ['expire', listKey.split('::')[2]];
+        }
+        else {
+            return ['sleep', listKey];
+        }
+    }
+    /**
+     * Interrupts a job and sets sets a job error (410), if 'throw'!=false.
+     * This method is called by the engine and not by an activity and is
+     * followed by a call to execute job completion/cleanup tasks
+     * associated with a job completion event.
+     */
+    async interrupt(topic, jobId, options = {}) {
+        try {
+            //verify job exists
+            const status = await this.getStatus(jobId, this.appId);
+            if (status <= 0) {
+                //verify still active; job already completed
+                throw new Error(`Job ${jobId} already completed`);
+            }
+            //decrement job status (:) by 1bil
+            const amount = -1000000000;
+            const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
+            const result = await this.redisClient[this.commands.hincrbyfloat](jobKey, ':', amount);
+            if (result <= amount) {
+                //verify active state; job already interrupted
+                throw new Error(`Job ${jobId} already completed`);
+            }
+            //persist the error unless specifically told not to
+            if (options.throw !== false) {
+                const errKey = `metadata/err`; //job errors are stored at the path `metadata/err`
+                const symbolNames = [`$${topic}`]; //the symbol for `metadata/err` is in redis and stored using the job topic
+                const symKeys = await this.getSymbolKeys(symbolNames);
+                const symVals = await this.getSymbolValues();
+                this.serializer.resetSymbols(symKeys, symVals, {});
+                //persists the standard 410 error (job is `gone`)
+                const err = JSON.stringify({
+                    code: enums_1.STATUS_CODE_INTERRUPT,
+                    message: options.reason ?? `job [${jobId}] interrupted`,
+                    job_id: jobId
+                });
+                const payload = { [errKey]: amount.toString() };
+                const hashData = this.serializer.package(payload, symbolNames);
+                const errSymbol = Object.keys(hashData)[0];
+                await this.redisClient[this.commands.hset](jobKey, errSymbol, err);
+            }
+        }
+        catch (e) {
+            if (!options.suppress) {
+                throw e;
+            }
+            else {
+                this.logger.debug('suppressed-interrupt', { message: e.message });
+            }
+        }
+    }
     async scrub(jobId) {
         const jobKey = this.mintKey(key_1.KeyType.JOB_STATE, { appId: this.appId, jobId });
         await this.redisClient[this.commands.del](jobKey);

package/build/services/task/index.d.ts

@@ -1,8 +1,9 @@
 /// <reference types="node" />
 import { ILogger } from '../logger';
 import { StoreService } from '../store';
-import { RedisClient, RedisMulti } from '../../types/redis';
 import { HookInterface } from '../../types/hook';
+import { JobCompletionOptions } from '../../types/job';
+import { RedisClient, RedisMulti } from '../../types/redis';
 declare class TaskService {
     store: StoreService<RedisClient, RedisMulti>;
     logger: ILogger;
@@ -10,9 +11,9 @@ declare class TaskService {
     constructor(store: StoreService<RedisClient, RedisMulti>, logger: ILogger);
     processWebHooks(hookEventCallback: HookInterface): Promise<void>;
     enqueueWorkItems(keys: string[]): Promise<void>;
-    registerJobForCleanup(jobId: string, inSeconds?: number): Promise<void>;
-    registerTimeHook(jobId: string, activityId: string, type: 'sleep' | 'expire' | 'cron', inSeconds?: number, multi?: RedisMulti): Promise<void>;
-    processTimeHooks(timeEventCallback: (jobId: string, activityId: string) => Promise<void>, listKey?: string): Promise<void>;
+    registerJobForCleanup(jobId: string, inSeconds: number, options: JobCompletionOptions): Promise<void>;
+    registerTimeHook(jobId: string, activityId: string, type: 'sleep' | 'expire' | 'interrupt', inSeconds?: number, multi?: RedisMulti): Promise<void>;
+    processTimeHooks(timeEventCallback: (jobId: string, activityId: string, type: 'sleep' | 'expire' | 'interrupt') => Promise<void>, listKey?: string): Promise<void>;
     cancelCleanup(): void;
 }
 export { TaskService };

package/build/services/task/index.js

@@ -1,11 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.TaskService = void 0;
+const enums_1 = require("../../modules/enums");
 const utils_1 = require("../../modules/utils");
-//system timer granularity limit (task queues organize)
-const FIDELITY_SECONDS = 15; //note: this can be reduced using 'watch' or scout role
-//default resolution/fidelity when expiring
-const EXPIRATION_FIDELITY_SECONDS = 60;
 class TaskService {
     constructor(store, logger) {
         this.cleanupTimeout = null;
@@ -32,27 +29,28 @@ class TaskService {
     async enqueueWorkItems(keys) {
         await this.store.addTaskQueues(keys);
     }
-    async registerJobForCleanup(jobId, inSeconds = EXPIRATION_FIDELITY_SECONDS) {
-        if (inSeconds > -1) {
+    async registerJobForCleanup(jobId, inSeconds = enums_1.EXPIRE_DURATION, options) {
+        if (inSeconds > 0) {
             await this.store.expireJob(jobId, inSeconds);
+            const expireTimeSlot = Math.floor((Date.now() + (inSeconds * 1000)) / (enums_1.FIDELITY_SECONDS * 1000)) * (enums_1.FIDELITY_SECONDS * 1000); //n second awaken groups
+            await this.store.registerExpireJob(jobId, expireTimeSlot, options);
         }
     }
-    async registerTimeHook(jobId, activityId, type, inSeconds = FIDELITY_SECONDS, multi) {
-        const awakenTimeSlot = Math.floor((Date.now() + inSeconds * 1000) / (FIDELITY_SECONDS * 1000)) * (FIDELITY_SECONDS * 1000); //n second awaken groups
+    async registerTimeHook(jobId, activityId, type, inSeconds = enums_1.FIDELITY_SECONDS, multi) {
+        const awakenTimeSlot = Math.floor((Date.now() + (inSeconds * 1000)) / (enums_1.FIDELITY_SECONDS * 1000)) * (enums_1.FIDELITY_SECONDS * 1000); //n second awaken groups
         await this.store.registerTimeHook(jobId, activityId, type, awakenTimeSlot, multi);
     }
-    //todo: need 'scout' role in quorum to check for this and then alert the quorum to get to work
     async processTimeHooks(timeEventCallback, listKey) {
         try {
-            const job = await this.store.getNextTimeJob(listKey);
-            if (job) {
-                const [listKey, jobId, activityId] = job;
-                await timeEventCallback(jobId, activityId);
+            const timeJob = await this.store.getNextTimeJob(listKey);
+            if (timeJob) {
+                const [listKey, jobId, activityId, type] = timeJob;
+                await timeEventCallback(jobId, activityId, type);
                 await (0, utils_1.sleepFor)(0);
                 this.processTimeHooks(timeEventCallback, listKey);
             }
             else {
-                let sleep = (0, utils_1.XSleepFor)(FIDELITY_SECONDS * 1000);
+                let sleep = (0, utils_1.XSleepFor)(enums_1.FIDELITY_SECONDS * 1000);
                 this.cleanupTimeout = sleep.timerId;
                 await sleep.promise;
                 this.processTimeHooks(timeEventCallback);