@hotmeshio/hotmesh 0.0.48 → 0.0.50

package/services/durable/workflow.ts

@@ -21,6 +21,7 @@ import {
 import { JobInterruptOptions, JobOutput, JobState } from '../../types/job';
 import { StreamStatus } from '../../types/stream';
 import { deterministicRandom } from '../../modules/utils';
+import { StringStringType } from '../../types';
 
 export class WorkflowService {
 
@@ -92,6 +93,10 @@ export class WorkflowService {
     const COUNTER = store.get('counter');
     const execIndex = COUNTER.counter = COUNTER.counter + 1;
     const sessionId = `-start${workflowDimension}-${execIndex}-`;
+    const replay = store.get('replay') as StringStringType;
+    if (sessionId in replay) {
+      return replay[sessionId];
+    }
     //NOTE: this is the hash prefix; necessary for the search index to locate the entity
     const entityOrEmptyString = options.entity ?? '';
     //If the workflowId is not provided, it is generated from the entity and the workflow name
@@ -193,6 +198,8 @@ export class WorkflowService {
   static getContext(): WorkflowContext {
     const store = asyncLocalStorage.getStore();
     const workflowId = store.get('workflowId');
+    const replay = store.get('replay');
+    const cursor = store.get('cursor');
     const workflowDimension = store.get('workflowDimension') ?? '';
     const workflowTopic = store.get('workflowTopic');
     const namespace = store.get('namespace');
@@ -201,7 +208,9 @@ export class WorkflowService {
     const COUNTER = store.get('counter');
     return {
       counter: COUNTER.counter,
+      cursor,
       namespace,
+      replay,
       workflowId,
       workflowDimension,
       workflowTopic,
@@ -223,6 +232,10 @@ export class WorkflowService {
     const COUNTER = store.get('counter');
     const execIndex = COUNTER.counter = COUNTER.counter + 1;
     const sessionId = `-${prefix}${workflowDimension}-${execIndex}-`;
+    const replay = store.get('replay') as StringStringType;
+    if (sessionId in replay) {
+      return false;
+    }
     const keyParams = {
       appId: hotMeshClient.appId,
       jobId: workflowId
@@ -300,6 +313,7 @@ export class WorkflowService {
       workflowTopic: store.get('workflowTopic'),
       workflowDimension: store.get('workflowDimension') ?? '',
       counter: store.get('counter'),
+      replay: store.get('replay'),
     }
   }
 
@@ -317,9 +331,13 @@ export class WorkflowService {
       workflowTopic,
       workflowDimension,
       counter: COUNTER,
+      replay,
     } = WorkflowService.getLocalState();
     const execIndex = COUNTER.counter = COUNTER.counter + 1;
     const sessionId = `-once${workflowDimension}-${execIndex}-`;
+    if (sessionId in replay) {
+      return JSON.parse(replay[sessionId]);
+    }
     const hotMeshClient = await WorkerService.getHotMesh(workflowTopic, { namespace });
     const keyParams = {
       appId: hotMeshClient.appId,

package/services/engine/index.ts

@@ -1,4 +1,4 @@
-import { KeyType } from '../../modules/key';
+import { KeyType, VALSEP } from '../../modules/key';
 import {
   HMSH_OTT_WAIT_TIME,
   HMSH_CODE_SUCCESS,
@@ -386,9 +386,10 @@ class EngineService {
         pg: streamData.metadata.gid,
         pd: streamData.metadata.dad,
         pa: streamData.metadata.aid,
+        px: streamData.metadata.await === false, //sever the parent connection (px)
         trc: streamData.metadata.trc,
         spn: streamData.metadata.spn,
-       };
+      };
       const activityHandler = await this.initActivity(
         streamData.metadata.topic,
         streamData.data,
@@ -459,7 +460,10 @@ class EngineService {
       return (await this.router?.publishMessage(null, streamData)) as string;
     }
   }
-  hasParentJob(context: JobState): boolean {
+  hasParentJob(context: JobState, checkSevered = false): boolean {
+    if (checkSevered) {
+      return Boolean(context.metadata.pj && context.metadata.pa && !context.metadata.px);
+    }
     return Boolean(context.metadata.pj && context.metadata.pa);
   }
   resolveError(metadata: JobMetadata): StreamError | undefined {
@@ -537,7 +541,11 @@ class EngineService {
         const taskService = new TaskService(this.store, this.logger);
         await taskService.enqueueWorkItems(
           workItems.map(
-            workItem => `${hookTopic}::${workItem}::${keyResolver.scrub || false}::${JSON.stringify(data)}`
+            workItem => [
+              hookTopic,
+              workItem,
+              keyResolver.scrub || false,
+              JSON.stringify(data)].join(VALSEP)
         ));
         this.store.publish(
           KeyType.QUORUM,
@@ -663,7 +671,7 @@ class EngineService {
   // ********** JOB COMPLETION/CLEANUP (AND JOB EMIT) ***********
   async runJobCompletionTasks(context: JobState, options: JobCompletionOptions = {}): Promise<string | void> {
     //'emit' indicates the job is still active
-    const isAwait = this.hasParentJob(context);
+    const isAwait = this.hasParentJob(context, true);
     const isOneTimeSub = this.hasOneTimeSubscription(context);
     const topic = await this.getPublishesTopic(context);
     let msgId: string;

package/services/exporter/index.ts

@@ -12,6 +12,7 @@ import {
   JobExport } from '../../types/exporter';
 import { SerializerService } from '../serializer';
 import { restoreHierarchy } from '../../modules/utils';
+import { VALSEP } from '../../modules/key';
 
 /**
  * Downloads job data from Redis (hscan, hmget, hgetall)
@@ -108,8 +109,8 @@ class ExporterService {
     const hookReg = /([0-9,]+)-(\d+)$/;
     const flowReg = /-(\d+)$/;
     return data.map((dependency, index: number): DependencyExport => {
-      const [action, topic, gid, ...jid] = dependency.split('::');
-      const jobId = jid.join('::');
+      const [action, topic, gid, _pd, ...jid] = dependency.split(VALSEP);
+      const jobId = jid.join(VALSEP);
       const match = jobId.match(hookReg);
       let prefix: string;
       let type: 'hook' | 'flow' | 'other';

package/services/hotmesh/index.ts

@@ -222,6 +222,10 @@ class HotMeshService {
 
   stop() {
     this.engine?.taskService.cancelCleanup();
+    this.quorum?.stop();
+    this.workers?.forEach((worker: WorkerService) => {
+      worker.stop();
+    });
   }
 
   async compress(terms: string[]): Promise<boolean> {

package/services/quorum/index.ts

@@ -1,7 +1,9 @@
 import {
   HMSH_ACTIVATION_MAX_RETRY,
-  HMSH_QUORUM_DELAY_MS } from '../../modules/enums';
+  HMSH_QUORUM_DELAY_MS, 
+  HMSH_QUORUM_ROLLCALL_CYCLES} from '../../modules/enums';
 import {
+  XSleepFor,
   formatISODate,
   getSystemHealth,
   identifyRedisType,
@@ -22,6 +24,7 @@ import {
   QuorumMessage,
   QuorumMessageCallback,
   QuorumProfile,
+  RollCallMessage,
   SubscriptionCallback,
   ThrottleMessage
 } from '../../types/quorum';
@@ -41,6 +44,7 @@ class QuorumService {
   untilVersion: string | null = null;
   quorum: number | null = null;
   callbacks: QuorumMessageCallback[] = [];
+  rollCallInterval: NodeJS.Timeout;
 
   static async init(
     namespace: string,
@@ -136,6 +140,8 @@ class QuorumService {
         self.engine.routeToSubscribers(message.topic, message.job)
       } else if (message.type === 'cron') {
         self.engine.processTimeHooks();
+      } else if (message.type === 'rollcall') {
+        self.doRollCall(message);
       }
       //if there are any callbacks, call them
       if (self.callbacks.length > 0) {
@@ -193,10 +199,40 @@ class QuorumService {
     return quorum;
   }
 
+  /**
+   * A quorum-wide command to broadcaset system details.
+   * 
+   */
+  async doRollCall(message: RollCallMessage) {
+    let iteration = 0;
+    let max = !isNaN(message.max) ? message.max : HMSH_QUORUM_ROLLCALL_CYCLES;
+    if (this.rollCallInterval) clearTimeout(this.rollCallInterval);
+    const base = (message.interval / 2);
+    const amount = base + Math.ceil(Math.random() * base);
+    do {
+      await sleepFor(Math.ceil(Math.random() * 1000));
+      await this.sayPong(this.appId, this.guid, null, true);
+      if (!message.interval) return;
+      const { promise, timerId } = XSleepFor(amount * 1000);
+      this.rollCallInterval = timerId;
+      await promise;
+    } while (this.rollCallInterval && iteration++ < max - 1);
+  }
+
+  cancelRollCall() {
+    if (this.rollCallInterval) {
+      clearTimeout(this.rollCallInterval);
+      delete this.rollCallInterval;
+    }
+  }
+
+  stop() {
+    this.cancelRollCall();
+  }
 
   // ************* PUB/SUB METHODS *************
   //publish a message to the quorum
-  async pub(quorumMessage: ThrottleMessage) {
+  async pub(quorumMessage: QuorumMessage) {
     return await this.store.publish(KeyType.QUORUM, quorumMessage, this.appId, quorumMessage.topic || quorumMessage.guid);
   }
   //subscribe user to quorum messages

package/services/router/index.ts

@@ -10,7 +10,7 @@ import {
   HMSH_XCLAIM_DELAY_MS,
   HMSH_XPENDING_COUNT } from '../../modules/enums';
 import { KeyType } from '../../modules/key';
-import { XSleepFor, guid, sleepFor } from '../../modules/utils';
+import { guid, sleepFor } from '../../modules/utils';
 import { ILogger } from '../logger';
 import { StoreService } from '../store';
 import { StreamService } from '../stream';
@@ -43,6 +43,10 @@ class Router {
   counts: { [key: string]: number } = {};
   currentTimerId: NodeJS.Timeout | null = null;
   shouldConsume: boolean;
+  sleepPromiseResolve: (() => void) | null = null;
+  innerPromiseResolve: (() => void) | null = null;
+  isSleeping: boolean = false;
+  sleepTimout: NodeJS.Timeout | null = null;
 
   constructor(config: StreamConfig, stream: StreamService<RedisClient, RedisMulti>, store: StoreService<RedisClient, RedisMulti>, logger: ILogger) {
     this.appId = config.appId;
@@ -54,6 +58,14 @@ class Router {
     this.reclaimDelay = config.reclaimDelay || HMSH_XCLAIM_DELAY_MS;
     this.reclaimCount = config.reclaimCount || HMSH_XCLAIM_COUNT;
     this.logger = logger;
+    this.resetThrottleState();
+  }
+
+  private resetThrottleState() {
+    this.sleepPromiseResolve = null;
+    this.innerPromiseResolve = null;
+    this.isSleeping = false;
+    this.sleepTimout = null;
   }
 
   async createGroup(stream: string, group: string) {
@@ -72,6 +84,36 @@ class Router {
     return await this.store.xadd(stream, '*', 'message', JSON.stringify(streamData), multi);
   }
 
+  /**
+   * An adjustable throttle that will interrupt a sleeping
+   * router if the throttle is reduced and the sleep time
+   * has elapsed. If the throttle is increased, or if
+   * the sleep time has not elapsed, the router will continue
+   * to sleep until the new termination point. This
+   * allows for dynamic, elastic throttling with smooth
+   * acceleration and deceleration.
+   */
+  public async customSleep(): Promise<void> {
+    if (this.throttle === 0) return;
+    if (this.isSleeping) return;
+    this.isSleeping = true;
+    let startTime = Date.now(); //anchor the origin
+
+    await new Promise<void>(async (outerResolve) => {
+      this.sleepPromiseResolve = outerResolve;
+      let elapsedTime = Date.now() - startTime;
+      while (elapsedTime < this.throttle) {
+        await new Promise<void>((innerResolve) => {
+          this.innerPromiseResolve = innerResolve;
+          this.sleepTimout = setTimeout(innerResolve, this.throttle - elapsedTime);
+        });
+        elapsedTime = Date.now() - startTime;
+      }
+      this.resetThrottleState();
+      outerResolve();
+    });
+  }
+
   async consumeMessages(stream: string, group: string, consumer: string, callback: (streamData: StreamData) => Promise<StreamDataResponse|void>): Promise<void> {
     this.logger.info(`stream-consumer-starting`, { group, consumer, stream });
     Router.instances.add(this);
@@ -80,9 +122,7 @@ class Router {
     let lastCheckedPendingMessagesAt = Date.now();
 
     async function consume() {
-      let sleep = XSleepFor(this.throttle);
-      this.currentTimerId = sleep.timerId;
-      await sleep.promise;
+      await this.customSleep();
       if (!this.shouldConsume) {
         this.logger.info(`stream-consumer-stopping`, { group, consumer, stream });
         return;
@@ -273,18 +313,28 @@ class Router {
   }
 
   cancelThrottle() {
-    if (this.currentTimerId !== undefined) {
-      clearTimeout(this.currentTimerId);
-      this.currentTimerId = undefined;
+    if (this.sleepTimout) {
+      clearTimeout(this.sleepTimout);
     }
+    this.resetThrottleState();
   }
 
-  setThrottle(delayInMillis: number) {
+  public setThrottle(delayInMillis: number): void {
     if (!Number.isInteger(delayInMillis) || delayInMillis < 0) {
       throw new Error('Throttle must be a non-negative integer');
     }
+    const wasDecreased = delayInMillis < this.throttle;
     this.throttle = delayInMillis;
-    this.logger.info(`stream-throttle-reset`, { delay: this.throttle, topic: this.topic });
+
+    // If the throttle was decreased, and we're in the middle of a sleep cycle, adjust immediately
+    if (wasDecreased) {
+      if (this.sleepTimout)  {
+        clearTimeout(this.sleepTimout);
+      }
+      if (this.innerPromiseResolve) {
+        this.innerPromiseResolve();
+      }
+    }
   }
 
   async claimUnacknowledged(stream: string, group: string, consumer: string, idleTimeMs = this.reclaimDelay, limit = HMSH_XPENDING_COUNT): Promise<[string, [string, string]][]> {

package/services/serializer/index.ts

@@ -19,7 +19,7 @@ export const MDATA_SYMBOLS = {
     KEYS: ['au', 'err', 'l2s']
   },
   JOB: {
-    KEYS: ['ngn', 'tpc', 'pj', 'pg', 'pd', 'pa', 'key', 'app', 'vrs', 'jid', 'gid', 'aid', 'ts', 'jc', 'ju', 'js', 'err', 'trc']
+    KEYS: ['ngn', 'tpc', 'pj', 'pg', 'pd', 'px', 'pa', 'key', 'app', 'vrs', 'jid', 'gid', 'aid', 'ts', 'jc', 'ju', 'js', 'err', 'trc']
   },
   JOB_UPDATE: {
     KEYS: ['ju', 'err']

package/services/store/clients/redis.ts

@@ -22,6 +22,7 @@ class RedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
       setnx: 'SETNX',
       del: 'DEL',
       expire: 'EXPIRE',
+      hscan: 'HSCAN',
       hset: 'HSET',
       hsetnx: 'HSETNX',
       hincrby: 'HINCRBY',
@@ -41,6 +42,7 @@ class RedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
       lpop: 'LPOP',
       rename: 'RENAME',
       rpush: 'RPUSH',
+      scan: 'SCAN',
       xack: 'XACK',
       xdel: 'XDEL',
       xlen: 'XLEN',

package/services/store/index.ts

@@ -2,7 +2,9 @@ import {
   KeyService,
   KeyStoreParams,
   KeyType, 
-  HMNS} from '../../modules/key';
+  HMNS,
+  VALSEP,
+  TYPSEP} from '../../modules/key';
 import { ILogger } from '../logger';
 import { MDATA_SYMBOLS, SerializerService as Serializer } from '../serializer';
 import { Cache } from './cache';
@@ -50,6 +52,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     del: 'del',
     expire: 'expire',
     hset: 'hset',
+    hscan: 'hscan',
     hsetnx: 'hsetnx',
     hincrby: 'hincrby',
     hdel: 'hdel',
@@ -68,6 +71,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     lrange: 'lrange',
     rename: 'rename',
     rpush: 'rpush',
+    scan: 'scan',
     xack: 'xack',
     xdel: 'xdel',
   };
@@ -426,7 +430,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
    * when `originJobId` is interrupted/expired, the items in the
    * list (added via RPUSH) will be interrupted/expired (removed via LPOPed).
    */
-  async registerJobDependency(depType: WorkListTaskType, originJobId: string, topic: string, jobId: string, gId: string, multi? : U): Promise<any> {
+  async registerJobDependency(depType: WorkListTaskType, originJobId: string, topic: string, jobId: string, gId: string, pd = '',  multi? : U): Promise<any> {
     const privateMulti = multi || this.getMulti();
     const dependencyParams = {
       appId: this.appId,
@@ -436,8 +440,13 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
       KeyType.JOB_DEPENDENTS,
       dependencyParams,
     );
-    //items listed as job dependencies have different relationships
-    const expireTask = `${depType}::${topic}::${gId}::${jobId}`;
+    const expireTask = [
+      depType,
+      topic,
+      gId,
+      pd,
+      jobId,
+    ].join(VALSEP);
     privateMulti[this.commands.rpush](depKey, expireTask);
     if (!multi) {
       return await privateMulti.exec();
@@ -448,15 +457,20 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
    * Ensures a `hook signal` is delisted when its parent activity/job
    * is interrupted/expired.
    */
-  async registerSignalDependency(jobId: string, signalKey: string, multi? : U): Promise<any> {
+  async registerSignalDependency(jobId: string, signalKey: string, dad: string, multi? : U): Promise<any> {
     const privateMulti = multi || this.getMulti();
     const dependencyParams = { appId: this.appId, jobId };
     const dependencyKey = this.mintKey(
       KeyType.JOB_DEPENDENTS,
       dependencyParams,
     );
-    //tasks have '4' segments
-    const delistTask = `delist::signal::${jobId}::${signalKey}`;
+    //persiste dependency tasks as multi-segment composite keys
+    const delistTask = [
+      'delist',
+      'signal',
+      jobId,
+      dad,
+      signalKey].join(VALSEP);
     privateMulti[this.commands.rpush](
       dependencyKey,
       delistTask,
@@ -797,11 +811,14 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
 
   async setHookSignal(hook: HookSignal, multi?: U): Promise<any> {
     const key = this.mintKey(KeyType.SIGNALS, { appId: this.appId });
-    const { topic, resolved, jobId} = hook; //`${activityId}::${dad}::${gId}::${jobId}`
+    //destructure the hook key
+    const { topic, resolved, jobId} = hook;
     const signalKey = `${topic}:${resolved}`;
     const payload = { [signalKey]: jobId };
     await (multi || this.redisClient)[this.commands.hset](key, payload);
-    return await this.registerSignalDependency(jobId.split('::')[3], signalKey, multi);
+    //jobId needs even more destructuring
+    const [_aid, dad, _gid, jid] = jobId.split(VALSEP);
+    return await this.registerSignalDependency(jid, signalKey, dad, multi);
   }
 
   async getHookSignal(topic: string, resolved: string): Promise<string | undefined> {
@@ -873,7 +890,7 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     const depParams = { appId: this.appId, jobId };
     const depKey = this.mintKey(KeyType.JOB_DEPENDENTS, depParams);
     const context = options.interrupt ? 'INTERRUPT' : 'EXPIRE';
-    const depKeyContext = `::${context}::${depKey}`;
+    const depKeyContext = `${TYPSEP}${context}${TYPSEP}${depKey}`;
     const zsetKey = this.mintKey(KeyType.TIME_RANGE, { appId: this.appId });
     await this.zAdd(zsetKey, deletionTime.toString(), depKeyContext);
   }
@@ -890,9 +907,15 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
    * for the given sleep group. Sleep groups are
    * organized into 'n'-second blocks (LISTS))
    */
-  async registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, deletionTime: number, multi?: U): Promise<void> {
+  async registerTimeHook(jobId: string, gId: string, activityId: string, type: WorkListTaskType, deletionTime: number, dad: string, multi?: U): Promise<void> {
     const listKey = this.mintKey(KeyType.TIME_RANGE, { appId: this.appId, timeValue: deletionTime });
-    const timeEvent = `${type}::${activityId}::${gId}::${jobId}`;
+    //construct the composite key (the key has enough info to signal the hook)
+    const timeEvent = [
+      type,
+      activityId,
+      gId,
+      dad,
+      jobId].join(VALSEP);
     const len = await (multi || this.redisClient)[this.commands.rpush](listKey, timeEvent);
     if (multi || len === 1) {
       const zsetKey = this.mintKey(KeyType.TIME_RANGE, { appId: this.appId });
@@ -907,17 +930,23 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
       let [pType, pKey] = this.resolveTaskKeyContext(listKey);
       const timeEvent = await this.redisClient[this.commands.lpop](pKey);
       if (timeEvent) {
-        //there are task types
-        //1) sleep (awaken), 2) expire (OR expire-child), 3) interrupt, 4) delist, 5) child (just an index helper; no work to do)
-        let [type, activityId, gId, ...jobId] = timeEvent.split('::');
+        //deconstruct composite key
+        let [
+          type,
+          activityId,
+          gId,
+          _pd,
+          ...jobId] = timeEvent.split(VALSEP);
+        const jid = jobId.join(VALSEP);
+
         if (type === 'delist') {
           pType = 'delist';
         } else if (type === 'child') {
           pType = 'child';
         } else if (type === 'expire-child') {
-          type = 'expire'; //use the same logic as 'expire'
+          type = 'expire';
         }
-        return [listKey, jobId.join('::'), gId, activityId, pType];
+        return [listKey, jid, gId, activityId, pType];
       }
       await this.redisClient[this.commands.zrem](zsetKey, listKey);
       return true;
@@ -931,13 +960,13 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
    * do with the work list. (not everything is known in advance,
    * so the ZSET key defines HOW to approach the work in the
    * generic LIST (lists typically contain target job ids)
-   * @param {string} listKey - for example `::INTERRUPT::job123` or `job123`
+   * @param {string} listKey - composite key
    */
   resolveTaskKeyContext(listKey: string): [WorkListTaskType, string] {
-    if (listKey.startsWith('::INTERRUPT')) {
-      return ['interrupt', listKey.split('::')[2]];
-    } else if (listKey.startsWith('::EXPIRE')) {
-      return ['expire', listKey.split('::')[2]];
+    if (listKey.startsWith(`${TYPSEP}INTERRUPT`)) {
+      return ['interrupt', listKey.split(TYPSEP)[2]];
+    } else if (listKey.startsWith(`${TYPSEP}EXPIRE`)) {
+      return ['expire', listKey.split(TYPSEP)[2]];
     } else {
       return ['sleep', listKey];
     }
@@ -998,6 +1027,63 @@ abstract class StoreService<T, U extends AbstractRedisClient> {
     const jobKey = this.mintKey(KeyType.JOB_STATE, { appId: this.appId, jobId });
     await this.redisClient[this.commands.del](jobKey);
   }
+
+  async findJobs(queryString: string = '*', limit: number = 1000, batchSize: number = 1000): Promise<[string, string[]]> {
+    const matchKey = this.mintKey(KeyType.JOB_STATE, { appId: this.appId, jobId: queryString });
+    let cursor = '0';
+    let keys: string[];
+    const matchingKeys: string[] = [];
+    do {
+      const output = await this.exec(
+        'SCAN',
+        cursor,
+        'MATCH',
+        matchKey,
+        'COUNT',
+        batchSize.toString(),
+      ) as unknown as [string, string[]];
+      if (Array.isArray(output)) {
+        [cursor, keys] = output;
+        for (let key of [...keys]) {
+          matchingKeys.push(key);
+        }
+        if (matchingKeys.length >= limit) {
+          break;
+        }
+      } else {
+        break;
+      }
+    } while (cursor !== '0');
+    return [cursor, matchingKeys];
+  }
+
+  async findJobFields(jobId: string, fieldMatchPattern: string = '*', limit: number = 1000, batchSize: number = 1000, cursor = '0'): Promise<[string, StringStringType]> {
+    let fields: string[] = [];
+    const matchingFields: StringStringType = {};
+    const jobKey = this.mintKey(KeyType.JOB_STATE, { appId: this.appId, jobId });
+    let len = 0;
+    do {
+      const output = await this.exec(
+        'HSCAN',
+        jobKey,
+        cursor,
+        'MATCH',
+        fieldMatchPattern,
+        'COUNT',
+        batchSize.toString(),
+      ) as unknown as [string, string[]];
+      if (Array.isArray(output)) {
+        [cursor, fields] = output;
+        for (let i = 0; i < fields.length; i += 2) {
+          len++;
+          matchingFields[fields[i]] = fields[i + 1];
+        }
+      } else {
+        break;
+      }
+    } while (cursor !== '0' && len < limit);
+    return [cursor, matchingFields];
+  }  
 }
 
 export { StoreService };