@hotmeshio/hotmesh 0.0.37 → 0.0.39

package/services/router/index.ts

@@ -1,14 +1,14 @@
 import {
-  BLOCK_TIME_MS,
-  MAX_RETRIES,
-  MAX_TIMEOUT_MS,
-  GRADUATED_INTERVAL_MS,
-  STATUS_CODE_UNACKED,
-  STATUS_CODE_UNKNOWN,
-  STATUS_MESSAGE_UNKNOWN,
-  XCLAIM_COUNT,
-  XCLAIM_DELAY_MS,
-  XPENDING_COUNT } from '../../modules/enums';
+  HMSH_BLOCK_TIME_MS,
+  HMSH_MAX_RETRIES,
+  HMSH_MAX_TIMEOUT_MS,
+  HMSH_GRADUATED_INTERVAL_MS,
+  HMSH_CODE_UNACKED,
+  HMSH_CODE_UNKNOWN,
+  HMSH_STATUS_UNKNOWN,
+  HMSH_XCLAIM_COUNT,
+  HMSH_XCLAIM_DELAY_MS,
+  HMSH_XPENDING_COUNT } from '../../modules/enums';
 import { KeyType } from '../../modules/key';
 import { XSleepFor, guid, sleepFor } from '../../modules/utils';
 import { ILogger } from '../logger';
@@ -50,8 +50,8 @@ class Router {
     this.topic = config.topic;
     this.stream = stream;
     this.store = store;
-    this.reclaimDelay = config.reclaimDelay || XCLAIM_DELAY_MS;
-    this.reclaimCount = config.reclaimCount || XCLAIM_COUNT;
+    this.reclaimDelay = config.reclaimDelay || HMSH_XCLAIM_DELAY_MS;
+    this.reclaimCount = config.reclaimCount || HMSH_XCLAIM_COUNT;
     this.logger = logger;
   }
 
@@ -85,7 +85,9 @@ class Router {
       }
 
       try {
-        const result = await this.stream.xreadgroup('GROUP', group, consumer, 'BLOCK', BLOCK_TIME_MS, 'STREAMS', stream, '>');
+        //randomizer that asymptotes at 150% of `HMSH_BLOCK_TIME_MS`
+        const streamDuration = HMSH_BLOCK_TIME_MS + Math.round((HMSH_BLOCK_TIME_MS * Math.random()));
+        const result = await this.stream.xreadgroup('GROUP', group, consumer, 'BLOCK', streamDuration, 'STREAMS', stream, '>');
         if (this.isStreamMessage(result)) {
           const [[, messages]] = result;
           for (const [id, message] of messages) {
@@ -107,7 +109,7 @@ class Router {
         if (this.shouldConsume && process.env.NODE_ENV !== 'test') {
         this.logger.error(`stream-consume-message-error`, { err, stream, group, consumer });
           this.errorCount++;
-          const timeout = Math.min(GRADUATED_INTERVAL_MS * (2 ** this.errorCount), MAX_TIMEOUT_MS);
+          const timeout = Math.min(HMSH_GRADUATED_INTERVAL_MS * (2 ** this.errorCount), HMSH_MAX_TIMEOUT_MS);
           setTimeout(consume.bind(this), timeout);
         }
       }
@@ -129,7 +131,7 @@ class Router {
       telemetry.startStreamSpan(input, this.role);  
       output = await this.execStreamLeg(input, stream, id, callback.bind(this));
       if (output?.status === StreamStatus.ERROR) {
-        telemetry.setStreamError(`Function Status Code ${ output.code || STATUS_CODE_UNKNOWN }`);
+        telemetry.setStreamError(`Function Status Code ${ output.code || HMSH_CODE_UNKNOWN }`);
       }
       this.errorCount = 0;
     } catch (err) {
@@ -191,7 +193,7 @@ class Router {
     const errorCode = output.code.toString();
     const policy = policies?.[errorCode];
     const maxRetries = policy?.[0];
-    const tryCount = Math.min(input.metadata.try || 0,  MAX_RETRIES);
+    const tryCount = Math.min(input.metadata.try || 0,  HMSH_MAX_RETRIES);
     //only possible values for maxRetries are 1, 2, 3
     //only possible values for tryCount are 0, 1, 2
     if (maxRetries > tryCount) {
@@ -206,7 +208,7 @@ class Router {
     if (typeof err.message === 'string') {
       error.message = err.message;
     } else {
-      error.message = STATUS_MESSAGE_UNKNOWN;
+      error.message = HMSH_STATUS_UNKNOWN;
     }
     if (typeof err.stack === 'string') {
       error.stack = err.stack;
@@ -216,7 +218,7 @@ class Router {
     }
     return {
       status: 'error',
-      code: STATUS_CODE_UNKNOWN,
+      code: HMSH_CODE_UNKNOWN,
       metadata: { ...input.metadata, guid: guid() },
       data: error as StreamError
     } as StreamDataResponse;
@@ -224,7 +226,7 @@ class Router {
 
   structureUnacknowledgedError(input: StreamData) {
     const message = 'stream message max delivery count exceeded';
-    const code = STATUS_CODE_UNACKED;
+    const code = HMSH_CODE_UNACKED;
     const data: StreamError = { message, code };
     const output: StreamDataResponse = { 
       metadata: { ...input.metadata, guid: guid() },
@@ -238,9 +240,9 @@ class Router {
   }
 
   structureError(input: StreamData, output: StreamDataResponse): StreamDataResponse {
-    const message = output.data?.message ? output.data?.message.toString() : STATUS_MESSAGE_UNKNOWN;
+    const message = output.data?.message ? output.data?.message.toString() : HMSH_STATUS_UNKNOWN;
     const statusCode = output.code || output.data?.code;
-    const code = isNaN(statusCode as number) ? STATUS_CODE_UNKNOWN : parseInt(statusCode.toString());
+    const code = isNaN(statusCode as number) ? HMSH_CODE_UNKNOWN : parseInt(statusCode.toString());
     const data: StreamError = { message, code };
     if (typeof output.data?.error === 'object') {
       data.error = { ...output.data.error };
@@ -257,7 +259,7 @@ class Router {
     for (const instance of [...Router.instances]) {
       instance.stopConsuming();
     }
-    await sleepFor(BLOCK_TIME_MS);
+    await sleepFor(HMSH_BLOCK_TIME_MS * 2);
   }
   
   async stopConsuming() {
@@ -281,7 +283,7 @@ class Router {
     this.logger.info(`stream-throttle-reset`, { delay: this.throttle, topic: this.topic });
   }
 
-  async claimUnacknowledged(stream: string, group: string, consumer: string, idleTimeMs = this.reclaimDelay, limit = XPENDING_COUNT): Promise<[string, [string, string]][]> {
+  async claimUnacknowledged(stream: string, group: string, consumer: string, idleTimeMs = this.reclaimDelay, limit = HMSH_XPENDING_COUNT): Promise<[string, [string, string]][]> {
     let pendingMessages = [];
     const pendingMessagesInfo = await this.stream.xpending(stream, group, '-', '+', limit); //[[ '1688768134881-0', 'testConsumer1', 1017, 1 ]]
     for (const pendingMessageInfo of pendingMessagesInfo) {
@@ -310,7 +312,7 @@ class Router {
     //     ii) corrupt hardware/network/transport/etc
     // 3b) system error: Redis unable to accept `xadd` request
     // 4c) system error: Redis unable to accept `xdel`/`xack` request
-    this.logger.error('stream-message-max-delivery-count-exceeded', { id, stream, group, consumer, code: STATUS_CODE_UNACKED, count });
+    this.logger.error('stream-message-max-delivery-count-exceeded', { id, stream, group, consumer, code: HMSH_CODE_UNACKED, count });
     const streamData = reclaimedMessage[0]?.[1]?.[1];
 
     //fatal risk point 1 of 3): json is corrupt

package/services/store/clients/ioredis.ts

@@ -131,6 +131,15 @@ class IORedisStoreService extends StoreService<RedisClientType, RedisMultiType>
       throw error;
     }
   }
+
+  async xlen(key: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
+    try {
+      return await (multi || this.redisClient).xlen(key);
+    } catch (error) {
+      this.logger.error(`Error getting stream depth: ${key}`, { error });
+      throw error;
+    }
+  }
 }
 
 export { IORedisStoreService };

package/services/store/clients/redis.ts

@@ -42,6 +42,7 @@ class RedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
       rpush: 'RPUSH',
       xack: 'XACK',
       xdel: 'XDEL',
+      xlen: 'XLEN',
     };
   }
 
@@ -167,6 +168,21 @@ class RedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
       throw error;
     }
   }
+
+  async xlen(key: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
+    try {
+      if (multi) {
+        multi.XLEN(key);
+        return multi;
+      } else {
+        return await this.redisClient.XLEN(key);
+      }
+    } catch (error) {
+      this.logger.error(`Error getting stream depth: ${key}`, { error });
+      throw error;
+    }
+  }
+
 }
 
 export { RedisStoreService };

package/services/store/index.ts

@@ -29,8 +29,9 @@ import { Transitions } from '../../types/transition';
 import { formatISODate, getSymKey } from '../../modules/utils';
 import { ReclaimedMessageType } from '../../types/stream';
 import { JobCompletionOptions, JobInterruptOptions } from '../../types/job';
-import { SCOUT_INTERVAL_SECONDS, STATUS_CODE_INTERRUPT } from '../../modules/enums';
+import { HMSH_SCOUT_INTERVAL_SECONDS, HMSH_CODE_INTERRUPT } from '../../modules/enums';
 import { GetStateError } from '../../modules/errors';
+import { WorkListTaskType } from '../../types/task';
 
 interface AbstractRedisClient {
   exec(): any;
@@ -118,6 +119,10 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     id: string,
     multi?: U
   ): Promise<number|U>;
+  abstract xlen(
+    key: string,
+    multi?: U
+  ): Promise<number|U>;
 
   constructor(redisClient: T) {
     this.redisClient = redisClient;
@@ -173,7 +178,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
    * check for and process work items in the
    * time and signal task queues.
    */
-  async reserveScoutRole(scoutType: 'time' | 'signal', delay = SCOUT_INTERVAL_SECONDS): Promise<boolean> {
+  async reserveScoutRole(scoutType: 'time' | 'signal', delay = HMSH_SCOUT_INTERVAL_SECONDS): Promise<boolean> {
     const key = this.mintKey(KeyType.WORK_ITEMS, { appId: this.appId, scoutType });
     const success = await this.redisClient[this.commands.setnx](key, `${scoutType}:${formatISODate(new Date())}`);
     if (this.isSuccessful(success)) {
@@ -384,16 +389,45 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
   }
 
   /**
-   * Registers jobId with the originJobId that spawned it. In the future,
-   * when originJobId is interrupted or expired, the items in the
-   * list (added via RPUSH) are LPOPed. If origin was expired, then
-   * LPOPed items from the list are likewise expired;
+   * Registers the job, `jobId`, with `originJobId`. In the future,
+   * when `originJobId` is interrupted/expired, the items in the
+   * list (added via RPUSH) will be interrupted/expired (removed via LPOPed).
    */
-  async setDependency(originJobId: string, topic: string, jobId: string, gId: string, multi? : U): Promise<any> {
+  async registerJobDependency(originJobId: string, topic: string, jobId: string, gId: string, multi? : U): Promise<any> {
     const privateMulti = multi || this.getMulti();
-    const depParams = { appId: this.appId, jobId: originJobId };
-    const depKey = this.mintKey(KeyType.JOB_DEPENDENTS, depParams);
-    privateMulti[this.commands.rpush](depKey, `expire::${topic}::${gId}::${jobId}`);
+    const dependencyParams = {
+      appId: this.appId,
+      jobId: originJobId,
+    };
+    const depKey = this.mintKey(
+      KeyType.JOB_DEPENDENTS,
+      dependencyParams,
+    );
+    //tasks have '4' segments
+    const expireTask = `expire::${topic}::${gId}::${jobId}`;
+    privateMulti[this.commands.rpush](depKey, expireTask);
+    if (!multi) {
+      return await privateMulti.exec();
+    }
+  }
+
+  /**
+   * Ensures a `hook signal` is delisted when its parent activity/job
+   * is interrupted/expired.
+   */
+  async registerSignalDependency(jobId: string, signalKey: string, multi? : U): Promise<any> {
+    const privateMulti = multi || this.getMulti();
+    const dependencyParams = { appId: this.appId, jobId };
+    const dependencyKey = this.mintKey(
+      KeyType.JOB_DEPENDENTS,
+      dependencyParams,
+    );
+    //tasks have '4' segments
+    const delistTask = `delist::signal::${jobId}::${signalKey}`;
+    privateMulti[this.commands.rpush](
+      dependencyKey,
+      delistTask,
+    );
     if (!multi) {
       return await privateMulti.exec();
     }
@@ -706,11 +740,11 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
 
   async setHookSignal(hook: HookSignal, multi?: U): Promise<any> {
     const key = this.mintKey(KeyType.SIGNALS, { appId: this.appId });
-    const { topic, resolved, jobId} = hook;
-    const payload = {
-      [`${topic}:${resolved}`]: jobId
-    };
-    return await (multi || this.redisClient)[this.commands.hset](key, payload);
+    const { topic, resolved, jobId} = hook; //`${activityId}::${dad}::${gId}::${jobId}`
+    const signalKey = `${topic}:${resolved}`;
+    const payload = { [signalKey]: jobId };
+    await (multi || this.redisClient)[this.commands.hset](key, payload);
+    return await this.registerSignalDependency(jobId.split('::')[3], signalKey, multi);
   }
 
   async getHookSignal(topic: string, resolved: string): Promise<string | undefined> {
@@ -778,7 +812,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
    * expired at a future date; options indicate whether this
    * is a standard `expire` or an `interrupt`
    */
-  async registerExpireJob(jobId: string, deletionTime: number, options: JobCompletionOptions): Promise<void> {
+  async registerDependenciesForCleanup(jobId: string, deletionTime: number, options: JobCompletionOptions): Promise<void> {
     const depParams = { appId: this.appId, jobId };
     const depKey = this.mintKey(KeyType.JOB_DEPENDENTS, depParams);
     const context = options.interrupt ? 'INTERRUPT' : 'EXPIRE';
@@ -793,7 +827,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
    * for the given sleep group. Sleep groups are
    * organized into 'n'-second blocks (LISTS))
    */
-  async registerTimeHook(jobId: string, gId: string, activityId: string, type: 'sleep'|'expire'|'interrupt', deletionTime: number, multi?: U): Promise<void> {
+  async registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, deletionTime: number, multi?: U): Promise<void> {
     const listKey = this.mintKey(KeyType.TIME_RANGE, { appId: this.appId, timeValue: deletionTime });
     const timeEvent = `${type}::${activityId}::${gId}::${jobId}`;
     const len = await (multi || this.redisClient)[this.commands.rpush](listKey, timeEvent);
@@ -803,21 +837,25 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     }
   }
 
-  async getNextTimeJob(listKey?: string): Promise<[listKey: string, jobId: string, gId: string, activityId: string, type: 'sleep'|'expire'|'interrupt'] | boolean> {
-    const existing = Boolean(listKey);
+  async getNextTask(listKey?: string): Promise<[listKey: string, jobId: string, gId: string, activityId: string, type: WorkListTaskType] | boolean> {
     const zsetKey = this.mintKey(KeyType.TIME_RANGE, { appId: this.appId });
     listKey = listKey || await this.zRangeByScore(zsetKey, 0, Date.now());
     if (listKey) {
-      const [pType, pKey] = this.resolveKeyContext(listKey);
+      let [pType, pKey] = this.resolveTaskKeyContext(listKey);
       const timeEvent = await this.redisClient[this.commands.lpop](pKey);
       if (timeEvent) {
-        //there are 3 time-related event triggers: sleep, expire, interrupt
-        const [_type, activityId, gId, ...jobId] = timeEvent.split('::');
+        //there are 4 time-related task
+        //1) sleep (awaken), 2) expire, 3) interrupt, 4) delist
+        const [type, activityId, gId, ...jobId] = timeEvent.split('::');
+        if (type === 'delist') {
+          pType = 'delist';
+        }
         return [listKey, jobId.join('::'), gId, activityId, pType];
       }
       await this.redisClient[this.commands.zrem](zsetKey, listKey);
+      return true;
     }
-    return existing;
+    return false;
   }
 
   /**
@@ -828,7 +866,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
    * generic LIST (lists typically contain target job ids)
    * @param {string} listKey - for example `::INTERRUPT::job123` or `job123`
    */
-  resolveKeyContext(listKey: string): [('sleep'|'expire'|'interrupt'), string] {
+  resolveTaskKeyContext(listKey: string): [('sleep'|'expire'|'interrupt'|'delist'), string] {
     if (listKey.startsWith('::INTERRUPT')) {
       return ['interrupt', listKey.split('::')[2]];
     } else if (listKey.startsWith('::EXPIRE')) {
@@ -870,7 +908,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
 
         //persists the standard 410 error (job is `gone`)
         const err = JSON.stringify({
-          code: STATUS_CODE_INTERRUPT,
+          code: HMSH_CODE_INTERRUPT,
           message: options.reason ?? `job [${jobId}] interrupted`,
           job_id: jobId
         });

package/services/stream/clients/ioredis.ts

@@ -33,16 +33,16 @@ class IORedisStreamService extends StreamService<RedisClientType, RedisMultiType
     if (mkStream === 'MKSTREAM') {
       try {
         return (await this.redisClient.xgroup(command, key, groupName, id, mkStream)) === 'OK';
-      } catch (err) {
+      } catch (error) {
         this.logger.info(`Consumer group not created with MKSTREAM for key: ${key} and group: ${groupName}`);
-        throw err;
+        throw error;
       }
     } else {
       try {
         return (await this.redisClient.xgroup(command, key, groupName, id)) === 'OK';
-      } catch (err) {
+      } catch (error) {
         this.logger.info(`Consumer group not created for key: ${key} and group: ${groupName}`);
-        throw err;
+        throw error;
       }
     }
   }
@@ -50,9 +50,9 @@ class IORedisStreamService extends StreamService<RedisClientType, RedisMultiType
   async xadd(key: string, id: string, messageId: string, messageValue: string, multi?: RedisMultiType): Promise<string | RedisMultiType> {
     try {
       return await (multi || this.redisClient).xadd(key, id, messageId, messageValue);
-    } catch (err) {
-      this.logger.error(`Error publishing 'xadd'; key: ${key}`, err);
-      throw err;
+    } catch (error) {
+      this.logger.error(`Error publishing 'xadd'; key: ${key}`, { error });
+      throw error;
     }
   }
 
@@ -79,9 +79,9 @@ class IORedisStreamService extends StreamService<RedisClientType, RedisMultiType
         streamName,
         id
       );
-    } catch (err) {
-      this.logger.error(`Error reading stream data [Stream ${streamName}] [Group ${groupName}]`, err);
-      throw err;
+    } catch (error) {
+      this.logger.error(`Error reading stream data [Stream ${streamName}] [Group ${groupName}]`, { error });
+      throw error;
     }
   }
 
@@ -101,12 +101,12 @@ class IORedisStreamService extends StreamService<RedisClientType, RedisMultiType
       if (consumer) args.push(consumer);
       try {
         return await this.redisClient.call('XPENDING', ...args) as [string, string, number, number][];
-      } catch (err) {
-        this.logger.error('err, args', err, args);
+      } catch (error) {
+        this.logger.error('err, args', { error }, args);
       }
-    } catch (err) {
-      this.logger.error(`Error in retrieving pending messages for [stream ${key}], [group ${group}]`, err);
-      throw err;
+    } catch (error) {
+      this.logger.error(`Error in retrieving pending messages for [stream ${key}], [group ${group}]`, { error });
+      throw error;
     }
   }
 
@@ -120,27 +120,36 @@ class IORedisStreamService extends StreamService<RedisClientType, RedisMultiType
   ): Promise<ReclaimedMessageType> {
     try {
       return await this.redisClient.xclaim(key, group, consumer, minIdleTime, id, ...args) as unknown as ReclaimedMessageType;
-    } catch (err) {
-      this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, err);
-      throw err;
+    } catch (error) {
+      this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, { error });
+      throw error;
     }
   }
 
   async xack(key: string, group: string, id: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
     try {
       return await (multi || this.redisClient).xack(key, group, id);
-    } catch (err) {
-      this.logger.error(`Error in acknowledging messages in group: ${group} for key: ${key}`, err);
-      throw err;
+    } catch (error) {
+      this.logger.error(`Error in acknowledging messages in group: ${group} for key: ${key}`, { error });
+      throw error;
     }
   }
 
   async xdel(key: string, id: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
     try {
       return await (multi || this.redisClient).xdel(key, id);
-    } catch (err) {
-      this.logger.error(`Error in deleting messages with id: ${id} for key: ${key}`, err);
-      throw err;
+    } catch (error) {
+      this.logger.error(`Error in deleting messages with id: ${id} for key: ${key}`, { error });
+      throw error;
+    }
+  }
+
+  async xlen(key: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
+    try {
+      return await (multi || this.redisClient).xlen(key);
+    } catch (error) {
+      this.logger.error(`Error getting stream depth: ${key}`, { error });
+      throw error;
     }
   }
 }

package/services/stream/clients/redis.ts

@@ -139,6 +139,20 @@ class RedisStreamService extends StreamService<RedisClientType, RedisMultiType>
       throw err;
     }
   }
+
+  async xlen(key: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
+    try {
+      if (multi) {
+        multi.XLEN(key);
+        return multi;
+      } else {
+        return await this.redisClient.XLEN(key);
+      }
+    } catch (error) {
+      this.logger.error(`Error getting stream depth: ${key}`, { error });
+      throw error;
+    }
+  }
 }
 
 export { RedisStreamService };

package/services/stream/index.ts

@@ -52,6 +52,7 @@ abstract class StreamService<T, U> {
     ...args: string[]): Promise<ReclaimedMessageType>;
   abstract xack(key: string, group: string, id: string, multi?: U): Promise<number|U>;
   abstract xdel(key: string, id: string, multi?: U): Promise<number|U>;
+  abstract xlen(key: string, multi?: U): Promise<number|U>;
 }
 
 export { StreamService };

package/services/task/index.ts

@@ -1,14 +1,16 @@
 import {
-  EXPIRE_DURATION,
-  FIDELITY_SECONDS, 
-  SCOUT_INTERVAL_SECONDS} from '../../modules/enums';
+  HMSH_EXPIRE_DURATION,
+  HMSH_FIDELITY_SECONDS, 
+  HMSH_SCOUT_INTERVAL_SECONDS} from '../../modules/enums';
 import { XSleepFor, sleepFor } from '../../modules/utils';
 import { ILogger } from '../logger';
+import { Pipe } from '../pipe';
 import { StoreService } from '../store';
 import { HookInterface, HookRule, HookSignal } from '../../types/hook';
+import { KeyType } from '../../types/hotmesh';
 import { JobCompletionOptions, JobState } from '../../types/job';
 import { RedisClient, RedisMulti } from '../../types/redis';
-import { Pipe } from '../pipe';
+import { WorkListTaskType } from '../../types/task';
 
 class TaskService {
   store: StoreService<RedisClient, RedisMulti>;
@@ -35,7 +37,12 @@ class TaskService {
         //todo: don't use 'id', make configurable using hook rule
         await hookEventCallback(topic, { ...data, id: jobId });
       } else {
-        await this.store.deleteProcessedTaskQueue(workItemKey, sourceKey, destinationKey, scrub === 'true');
+        await this.store.deleteProcessedTaskQueue(
+          workItemKey,
+          sourceKey,
+          destinationKey,
+          scrub === 'true'
+        );
       }
       setImmediate(() => this.processWebHooks(hookEventCallback));
     }
@@ -45,21 +52,45 @@ class TaskService {
     await this.store.addTaskQueues(keys);
   }
 
-  async registerJobForCleanup(jobId: string, inSeconds = EXPIRE_DURATION, options: JobCompletionOptions): Promise<void> {
+  async registerJobForCleanup(jobId: string, inSeconds = HMSH_EXPIRE_DURATION, options: JobCompletionOptions): Promise<void> {
     if (inSeconds > 0) {
       await this.store.expireJob(jobId, inSeconds);
-      const expireTimeSlot = Math.floor((Date.now() + (inSeconds * 1000)) / (FIDELITY_SECONDS * 1000)) * (FIDELITY_SECONDS * 1000); //n second awaken groups
-      await this.store.registerExpireJob(jobId, expireTimeSlot, options);
+      const fromNow = Date.now() + (inSeconds * 1000);
+      const fidelityMS = HMSH_FIDELITY_SECONDS * 1000;
+      const timeSlot = Math.floor(fromNow / fidelityMS) * fidelityMS;
+      await this.store.registerDependenciesForCleanup(
+        jobId,
+        timeSlot,
+        options,
+      );
     }
   }
 
-  async registerTimeHook(jobId: string, gId: string, activityId: string, type: 'sleep'|'expire'|'interrupt', inSeconds = FIDELITY_SECONDS, multi?: RedisMulti): Promise<void> {
-    const awakenTimeSlot = Math.floor((Date.now() + (inSeconds * 1000)) / (FIDELITY_SECONDS * 1000)) * (FIDELITY_SECONDS * 1000); //n second awaken groups
-    await this.store.registerTimeHook(jobId, gId, activityId, type, awakenTimeSlot, multi);
+  async registerTimeHook(
+    jobId: string,
+    gId: string,
+    activityId: string,
+    type: WorkListTaskType,
+    inSeconds = HMSH_FIDELITY_SECONDS,
+    multi?: RedisMulti,
+  ): Promise<void> {
+    const fromNow = Date.now() + (inSeconds * 1000);
+    const fidelityMS = HMSH_FIDELITY_SECONDS * 1000;
+    const awakenTimeSlot = Math.floor(fromNow / fidelityMS) * fidelityMS;
+    await this.store.registerTimeHook(
+      jobId,
+      gId,
+      activityId,
+      type,
+      awakenTimeSlot,
+      multi,
+    );
   }
 
   /**
-   * Should this engine instance play the role of 'scout' for the quorum.
+   * Should this engine instance play the role of 'scout' on behalf
+   * of the entire quorum? The scout role is responsible for processing
+   * task lists on behalf of the collective.
    */
   async shouldScout() {
     const wasScout = this.isScout;
@@ -68,30 +99,41 @@ class TaskService {
       if (!wasScout) {
         setTimeout(() => {
           this.isScout = false;
-        }, SCOUT_INTERVAL_SECONDS * 1_000);
+        }, HMSH_SCOUT_INTERVAL_SECONDS * 1_000);
       }
       return true;
     }
     return false;
   }
 
-  async processTimeHooks(timeEventCallback: (jobId: string, gId: string, activityId: string, type: 'sleep'|'expire'|'interrupt') => Promise<void>, listKey?: string): Promise<void> {
+  /**
+   * Callback handler that takes an item from a work list and
+   * processes according to its type
+   */
+  async processTimeHooks(timeEventCallback: (jobId: string, gId: string, activityId: string, type: WorkListTaskType) => Promise<void>, listKey?: string): Promise<void> {
     if (await this.shouldScout()) {
       try {
-        const timeJob = await this.store.getNextTimeJob(listKey);
-        if (Array.isArray(timeJob)) {
-          //a queue had a job; try again immediately
-          const [listKey, jobId, gId, activityId, type] = timeJob;
-          await timeEventCallback(jobId, gId, activityId, type);
+        const workListTask = await this.store.getNextTask(listKey);
+
+        if (Array.isArray(workListTask)) {
+          const [listKey, target, gId, activityId, type] = workListTask;
+          if (type === 'delist') {
+            //delist the signalKey (target)
+            const key = this.store.mintKey(KeyType.SIGNALS, { appId: this.store.appId });
+            await this.store.redisClient[this.store.commands.hdel](key, target);
+          } else {
+            //awaken/expire/interrupt
+            await timeEventCallback(target, gId, activityId, type);
+          }
           await sleepFor(0);
           this.processTimeHooks(timeEventCallback, listKey);
-        } else if (timeJob) {
-          //a queue was just emptied; try again immediately
+        } else if (workListTask) {
+          //a worklist was just emptied; try again immediately
           await sleepFor(0);
           this.processTimeHooks(timeEventCallback);
         } else {
-          //all queues are empty; sleep before checking
-          let sleep = XSleepFor(FIDELITY_SECONDS * 1000);
+          //no worklists exist; sleep before checking
+          let sleep = XSleepFor(HMSH_FIDELITY_SECONDS * 1000);
           this.cleanupTimeout = sleep.timerId;
           await sleep.promise;
           this.processTimeHooks(timeEventCallback);
@@ -102,7 +144,7 @@ class TaskService {
       }
     } else {
       //didn't get the scout role; try again in 'one-ish' minutes
-      let sleep = XSleepFor(SCOUT_INTERVAL_SECONDS * 1_000 * 2 * Math.random());
+      let sleep = XSleepFor(HMSH_SCOUT_INTERVAL_SECONDS * 1_000 * 2 * Math.random());
       this.cleanupTimeout = sleep.timerId;
       await sleep.promise;
       this.processTimeHooks(timeEventCallback);