@hotmeshio/hotmesh 0.0.1

package/services/stream/clients/ioredis.ts

@@ -0,0 +1,148 @@
+import { KeyService, KeyStoreParams, KeyType, PSNS } from '../../../modules/key';
+import { ILogger } from '../../logger';
+import { StreamService } from '../index';
+import { RedisClientType, RedisMultiType } from '../../../types/ioredisclient';
+import { ReclaimedMessageType } from '../../../types/stream';
+
+class IORedisStreamService extends StreamService<RedisClientType, RedisMultiType> {
+  redisClient: RedisClientType;
+  namespace: string;
+  logger: ILogger;
+  appId: string;
+
+  constructor(redisClient: RedisClientType) {
+    super(redisClient);
+  }
+
+  async init(namespace = PSNS, appId: string, logger: ILogger): Promise<void> {
+    this.namespace = namespace;
+    this.logger = logger;
+    this.appId = appId;
+  }
+
+  getMulti(): RedisMultiType {
+    return this.redisClient.multi();
+  }
+
+  mintKey(type: KeyType, params: KeyStoreParams): string {
+    if (!this.namespace) throw new Error('namespace not set');
+    return KeyService.mintKey(this.namespace, type, params);
+  }
+
+  async xgroup(command: 'CREATE', key: string, groupName: string, id: string, mkStream?: 'MKSTREAM'): Promise<boolean> {
+    if (mkStream === 'MKSTREAM') {
+      try {
+        return (await this.redisClient.xgroup(command, key, groupName, id, mkStream)) === 'OK';
+      } catch (err) {
+        this.logger.info(`Consumer group not created with MKSTREAM for key: ${key} and group: ${groupName}`);
+        throw err;
+      }
+    } else {
+      try {
+        return (await this.redisClient.xgroup(command, key, groupName, id)) === 'OK';
+      } catch (err) {
+        this.logger.info(`Consumer group not created for key: ${key} and group: ${groupName}`);
+        throw err;
+      }
+    }
+  }
+
+  async xadd(key: string, id: string, messageId: string, messageValue: string, multi?: RedisMultiType): Promise<string | RedisMultiType> {
+    try {
+      return await (multi || this.redisClient).xadd(key, id, messageId, messageValue);
+    } catch (err) {
+      this.logger.error(`Error publishing 'xadd'; key: ${key}`, err);
+      throw err;
+    }
+  }
+
+  async xreadgroup(
+    command: 'GROUP',
+    groupName: string,
+    consumerName: string,
+    blockOption: 'BLOCK'|'COUNT',
+    blockTime: number|string,
+    streamsOption: 'STREAMS',
+    streamName: string,
+    id: string
+  ): Promise<string[][][] | null | unknown[]> {
+    try {
+      //@ts-ignore
+      return await this.redisClient.xreadgroup(
+        command,
+        groupName,
+        consumerName,
+        // @ts-ignore
+        blockOption,
+        blockTime,
+        streamsOption,
+        streamName,
+        id
+      );
+    } catch (err) {
+      this.logger.error(`Error reading stream data [Stream ${streamName}] [Group ${groupName}]`, err);
+      throw err;
+    }
+  }
+
+  async xpending(
+    key: string,
+    group: string,
+    start?: string,
+    end?: string,
+    count?: number,
+    consumer?: string
+  ): Promise<[string, string, number, [string, number][]][] | [string, string, number, number] | unknown[]> {
+    try {
+      const args = [key, group];
+      if (start) args.push(start);
+      if (end) args.push(end);
+      if (count !== undefined) args.push(count.toString());
+      if (consumer) args.push(consumer);
+      try {
+        return await this.redisClient.call('XPENDING', ...args) as [string, string, number, number][];
+      } catch (err) {
+        this.logger.error('err, args', err, args);
+      }
+    } catch (err) {
+      this.logger.error(`Error in retrieving pending messages for [stream ${key}], [group ${group}]`, err);
+      throw err;
+    }
+  }
+
+  async xclaim(
+    key: string,
+    group: string,
+    consumer: string,
+    minIdleTime: number,
+    id: string,
+    ...args: string[]
+  ): Promise<ReclaimedMessageType> {
+    try {
+      return await this.redisClient.xclaim(key, group, consumer, minIdleTime, id, ...args) as unknown as ReclaimedMessageType;
+    } catch (err) {
+      this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, err);
+      throw err;
+    }
+  }
+
+  async xack(key: string, group: string, id: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
+    try {
+      return await (multi || this.redisClient).xack(key, group, id);
+    } catch (err) {
+      this.logger.error(`Error in acknowledging messages in group: ${group} for key: ${key}`, err);
+      throw err;
+    }
+  }
+
+  async xdel(key: string, id: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
+    try {
+      return await (multi || this.redisClient).xdel(key, id);
+    } catch (err) {
+      this.logger.error(`Error in deleting messages with id: ${id} for key: ${key}`, err);
+      throw err;
+    }
+  }
+}
+
+export { IORedisStreamService };

package/services/stream/clients/redis.ts

@@ -0,0 +1,144 @@
+import { KeyService, KeyStoreParams, KeyType, PSNS } from '../../../modules/key';
+import { ILogger } from '../../logger';
+import { StreamService } from '../index';
+import { RedisClientType, RedisMultiType } from '../../../types/redisclient';
+import { ReclaimedMessageType } from '../../../types/stream';
+
+class RedisStreamService extends StreamService<RedisClientType, RedisMultiType> {
+  redisClient: RedisClientType;
+  namespace: string;
+  logger: ILogger;
+  appId: string;
+
+  constructor(redisClient: RedisClientType) {
+    super(redisClient);
+  }
+
+  async init(namespace = PSNS, appId: string, logger: ILogger): Promise<void> {
+    this.namespace = namespace;
+    this.logger = logger;
+    this.appId = appId;
+  }
+
+  getMulti(): RedisMultiType {
+    return this.redisClient.MULTI() as unknown as RedisMultiType;
+  }
+
+  mintKey(type: KeyType, params: KeyStoreParams): string {
+    if (!this.namespace) throw new Error('namespace not set');
+    return KeyService.mintKey(this.namespace, type, params);
+  }
+
+  async xgroup(command: 'CREATE', key: string, groupName: string, id: string, mkStream?: 'MKSTREAM'): Promise<boolean> {
+    const args = mkStream === 'MKSTREAM' ? ['MKSTREAM'] : [];
+    try {
+      return (await this.redisClient.sendCommand(['XGROUP', 'CREATE', key, groupName, id, ...args])) === 1;
+    } catch (err) {
+      const streamType = mkStream === 'MKSTREAM' ? 'with MKSTREAM' : 'without MKSTREAM';
+      this.logger.error(`x-group-error ${streamType} for key: ${key} and group: ${groupName}`, err);
+      throw err;
+    }
+  }
+
+  async xadd(key: string, id: string, ...args: any[]): Promise<string | RedisMultiType> {
+    let multi: RedisMultiType;
+    if (typeof args[args.length - 1] !== 'string') {
+      multi = args.pop() as RedisMultiType;
+    }
+    try {
+      return await (multi || this.redisClient).XADD(key, id, { [args[0]]: args[1] });
+    } catch (err) {
+      this.logger.error(`Error publishing 'xadd'; key: ${key}`, err);
+      throw err;
+    }
+  }
+
+  async xreadgroup(
+    command: 'GROUP',
+    groupName: string,
+    consumerName: string,
+    blockOption: 'BLOCK'|'COUNT',
+    blockTime: number|string,
+    streamsOption: 'STREAMS',
+    streamName: string,
+    id: string
+  ): Promise<string[][][] | null> {
+    try {
+      return await this.redisClient.sendCommand(['XREADGROUP', command, groupName, consumerName, blockOption, blockTime.toString(), streamsOption, streamName, id]);
+    } catch (err) {
+      this.logger.error(`Error in reading data from group: ${groupName} in stream: ${streamName}`, err);
+      throw err;
+    }
+  }
+
+  async xpending(
+    key: string,
+    group: string,
+    start?: string,
+    end?: string,
+    count?: number,
+    consumer?: string
+  ): Promise<[string, string, number, [string, number][]][] | [string, string, number, number]> {
+    try {
+      const args = [key, group];
+      if (start) args.push(start);
+      if (end) args.push(end);
+      if (count !== undefined) args.push(count.toString());
+      if (consumer) args.push(consumer);
+      try {
+        return await this.redisClient.sendCommand(['XPENDING', ...args]);
+      } catch (err) {
+        this.logger.error('err, args', err, args);
+      }
+    } catch (err) {
+      this.logger.error(`Error in retrieving pending messages for group: ${group} in key: ${key}`, err);
+      throw err;
+    }
+  }
+  
+  async xclaim(
+    key: string,
+    group: string,
+    consumer: string,
+    minIdleTime: number,
+    id: string,
+    ...args: string[]
+  ): Promise<ReclaimedMessageType> {
+    try {
+      return await this.redisClient.sendCommand(['XCLAIM', key, group, consumer, minIdleTime.toString(), id, ...args]) as unknown as ReclaimedMessageType;
+    } catch (err) {
+      this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, err);
+      throw err;
+    }
+  }
+
+  async xack(key: string, group: string, id: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
+    try {
+      if (multi) {
+        multi.XACK(key, group, id);
+        return multi;
+      } else {
+        return await this.redisClient.XACK(key, group, id);
+      }
+    } catch (err) {
+      this.logger.error(`Error in acknowledging messages in group: ${group} for key: ${key}`, err);
+      throw err;
+    }
+  }
+
+  async xdel(key: string, id: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
+    try {
+      if (multi) {
+        multi.XDEL(key, id);
+        return multi;
+      } else {
+        return await this.redisClient.XDEL(key, id);
+      }
+    } catch (err) {
+      this.logger.error(`Error in deleting messages with ids: ${id} for key: ${key}`, err);
+      throw err;
+    }
+  }
+}
+
+export { RedisStreamService };

package/services/stream/index.ts

@@ -0,0 +1,57 @@
+import { KeyStoreParams, KeyType } from '../../modules/key';
+import { ReclaimedMessageType } from '../../types/stream';
+import { ILogger } from '../logger';
+
+abstract class StreamService<T, U> {
+  redisClient: T;
+  namespace: string;
+  logger: ILogger;
+  appId: string;
+
+  constructor(redisClient: T) {
+    this.redisClient = redisClient;
+  }
+
+  abstract init(namespace: string, appId: string, logger: ILogger): Promise<void>;
+  abstract getMulti(): U;
+  abstract mintKey(type: KeyType, params: KeyStoreParams): string;
+  abstract xgroup(
+    command: 'CREATE',
+    key: string,
+    groupName: string,
+    id: string,
+    mkStream?: 'MKSTREAM'): Promise<boolean>;
+  abstract xadd(
+    key: string,
+    id: string,
+    messageId: string,
+    messageValue: string,
+    multi?: U): Promise<string | U>;
+  abstract xreadgroup(
+    command: 'GROUP',
+    groupName: string,
+    consumerName: string,
+    blockOption: 'BLOCK'|'COUNT',
+    blockTime: number|string,
+    streamsOption: 'STREAMS',
+    streamName: string,
+    id: string): Promise<string[][][] | null | unknown[]>;
+  abstract xpending(
+    key: string,
+    group: string,
+    start?: string,
+    end?: string,
+    count?: number,
+    consumer?: string): Promise<[string, string, number, [string, number][]][] | [string, string, number, number] | unknown[]>;
+  abstract xclaim(
+    key: string,
+    group: string,
+    consumer: string,
+    minIdleTime: number,
+    id: string,
+    ...args: string[]): Promise<ReclaimedMessageType>;
+  abstract xack(key: string, group: string, id: string, multi?: U): Promise<number|U>;
+  abstract xdel(key: string, id: string, multi?: U): Promise<number|U>;
+}
+
+export { StreamService };

package/services/sub/clients/ioredis.ts

@@ -0,0 +1,83 @@
+import { KeyService, KeyStoreParams, KeyType, PSNS } from '../../../modules/key';
+import { ILogger } from '../../logger';
+import { SubService } from '../index';
+import { RedisClientType, RedisMultiType } from '../../../types/ioredisclient';
+import { SubscriptionCallback } from '../../../types/quorum';
+
+class IORedisSubService extends SubService<RedisClientType, RedisMultiType> {
+  redisClient: RedisClientType;
+  namespace: string;
+  logger: ILogger;
+  appId: string
+
+  constructor(redisClient: RedisClientType) {
+    super(redisClient);
+  }
+
+  async init(namespace = PSNS, appId: string, engineId: string, logger: ILogger): Promise<void> {
+    this.namespace = namespace;
+    this.logger = logger;
+    this.appId = appId;
+  }
+
+  getMulti(): RedisMultiType {
+    return this.redisClient.multi();
+  }
+
+  mintKey(type: KeyType, params: KeyStoreParams): string {
+    if (!this.namespace) throw new Error('namespace not set');
+    return KeyService.mintKey(this.namespace, type, params);
+  }
+
+  async subscribe(keyType: KeyType.QUORUM, callback: SubscriptionCallback, appId: string, engineId?: string): Promise<void> {
+    const self = this;
+    const topic = this.mintKey(keyType, { appId, engineId });
+    await this.redisClient.subscribe(topic, (err) => {
+      if (err) {
+        self.logger.error(`Error subscribing to: ${topic}`, err);
+      }
+    });
+    this.redisClient.on('message', (channel, message) => {
+      if (channel === topic) {
+        try {
+          const payload = JSON.parse(message);
+          callback(topic, payload);
+        } catch (e) {
+          self.logger.error(`Error parsing message: ${message}`, e);
+        }
+      }
+    });
+  }
+
+  async unsubscribe(keyType: KeyType.QUORUM, appId: string, engineId?: string): Promise<void> {
+    const topic = this.mintKey(keyType, { appId, engineId });
+    await this.redisClient.unsubscribe(topic);
+  }
+
+  async psubscribe(keyType: KeyType.QUORUM, callback: SubscriptionCallback, appId: string, engineId?: string): Promise<void> {
+    const self = this;
+    const topic = this.mintKey(keyType, { appId, engineId });
+    await this.redisClient.psubscribe(topic, (err) => {
+      if (err) {
+        self.logger.error(`Error subscribing to: ${topic}`, err);
+      }
+    });
+    this.redisClient.on('pmessage', (pattern, channel, message) => {
+      if (pattern === topic) {
+        try {
+          const payload = JSON.parse(message);
+          callback(channel, payload);
+        } catch (e) {
+          self.logger.error(`Error parsing message: ${message}`, e);
+        }
+      }
+    });
+  }
+
+  async punsubscribe(keyType: KeyType.QUORUM, appId: string, engineId?: string): Promise<void> {
+    const topic = this.mintKey(keyType, { appId, engineId });
+    await this.redisClient.punsubscribe(topic);
+  }
+}
+
+export { IORedisSubService };

package/services/sub/clients/redis.ts

@@ -0,0 +1,74 @@
+import { KeyService, KeyStoreParams, KeyType, PSNS } from '../../../modules/key';
+import { ILogger } from '../../logger';
+import { SubService } from '../index';
+import { RedisClientType, RedisMultiType } from '../../../types/redisclient';
+import { SubscriptionCallback } from '../../../types/quorum';
+
+class RedisSubService extends SubService<RedisClientType, RedisMultiType> {
+  redisClient: RedisClientType;
+  namespace: string;
+  logger: ILogger;
+  appId: string;
+
+  constructor(redisClient: RedisClientType) {
+    super(redisClient);
+  }
+
+  async init(namespace = PSNS, appId: string, engineId: string, logger: ILogger): Promise<void> {
+    this.namespace = namespace;
+    this.logger = logger;
+    this.appId = appId;
+  }
+
+  getMulti(): RedisMultiType {
+    const multi = this.redisClient.MULTI();
+    return multi as unknown as RedisMultiType;
+  }
+
+  mintKey(type: KeyType, params: KeyStoreParams): string {
+    if (!this.namespace) throw new Error('namespace not set');
+    return KeyService.mintKey(this.namespace, type, params);
+  }
+
+  async subscribe(keyType: KeyType.QUORUM, callback: SubscriptionCallback, appId: string, engineId?: string): Promise<void> {
+    if (this.redisClient) {
+      const self = this;
+      const topic = this.mintKey(keyType, { appId, engineId });
+      await this.redisClient.subscribe(topic, (message) => {
+        try {
+          const payload = JSON.parse(message);
+          callback(topic, payload);
+        } catch (e) {
+          self.logger.error(`Error parsing message: ${message}`, e);
+        }
+      });
+    }
+  }
+
+  async unsubscribe(keyType: KeyType.QUORUM, appId: string, engineId?: string): Promise<void> {
+    const topic = this.mintKey(keyType, { appId, engineId });
+    await this.redisClient.unsubscribe(topic);
+  }
+
+  async psubscribe(keyType: KeyType.QUORUM, callback: SubscriptionCallback, appId: string, engineId?: string): Promise<void> {
+    if (this.redisClient) {
+      const self = this;
+      const topic = this.mintKey(keyType, { appId, engineId });
+      await this.redisClient.pSubscribe(topic, (message, channel) => {
+        try {
+          const payload = JSON.parse(message);
+          callback(channel, payload);
+        } catch (e) {
+          self.logger.error(`Error parsing message: ${message}`, e);
+        }
+      });
+    }
+  }
+
+  async punsubscribe(keyType: KeyType.QUORUM, appId: string, engineId?: string): Promise<void> {
+    const topic = this.mintKey(keyType, { appId, engineId });
+    await this.redisClient.pUnsubscribe(topic);
+  }
+}
+
+export { RedisSubService };

package/services/sub/index.ts

@@ -0,0 +1,25 @@
+import { KeyStoreParams, KeyType } from '../../modules/key';
+import { ILogger } from '../logger';
+import { SubscriptionCallback } from '../../types/quorum';
+
+abstract class SubService<T, U> {
+  redisClient: T;
+  namespace: string;
+  logger: ILogger;
+  appId: string;
+
+  constructor(redisClient: T) {
+    this.redisClient = redisClient;
+  }
+
+  abstract init(namespace: string, appId: string, engineId: string, logger: ILogger): Promise<void>;
+  abstract getMulti(): U;
+  abstract mintKey(type: KeyType, params: KeyStoreParams): string;
+  abstract subscribe(keyType: KeyType.QUORUM, callback: SubscriptionCallback, appId: string, engineId?: string): Promise<void>;
+  abstract unsubscribe(keyType: KeyType.QUORUM, appId: string, engineId?: string): Promise<void>;
+  abstract psubscribe(keyType: KeyType.QUORUM, callback: SubscriptionCallback, appId: string, engineId?: string): Promise<void>;
+  abstract punsubscribe(keyType: KeyType.QUORUM, appId: string, engineId?: string): Promise<void>;
+  //NOTE: `publish` happens in the 'StoreService' as Redis subscription clients must be read-only
+}
+
+export { SubService };

package/services/task/index.ts

@@ -0,0 +1,86 @@
+import { ILogger } from '../logger';
+import { StoreService } from '../store';
+import { RedisClient, RedisMulti } from '../../types/redis';
+import { HookInterface } from '../../types/hook';
+import { XSleepFor, sleepFor } from '../../modules/utils';
+
+//system timer granularity limit (task queues organize)
+const FIDELITY_SECONDS = 15; //note: this can be reduced using 'watch' or scout role
+//default resolution/fidelity when expiring
+const EXPIRATION_FIDELITY_SECONDS = 60;
+
+class TaskService {
+  store: StoreService<RedisClient, RedisMulti>;
+  logger: ILogger;
+  cleanupTimeout: NodeJS.Timeout | null = null;
+
+  constructor(
+    store: StoreService<RedisClient, RedisMulti>,
+    logger: ILogger
+  ) {
+    this.logger = logger;
+    this.store = store;
+  }
+
+  async processWebHooks(hookEventCallback: HookInterface): Promise<void> {
+    const workItemKey = await this.store.getActiveTaskQueue();
+    if (workItemKey) {
+      const [topic, sourceKey, ...sdata] = workItemKey.split('::');
+      const data = JSON.parse(sdata.join('::'));
+      const destinationKey = `${sourceKey}:processed`;
+      const jobId = await this.store.processTaskQueue(sourceKey, destinationKey);
+      if (jobId) {
+        await hookEventCallback(topic, { ...data, id: jobId });
+        //todo: do final checksum count (values are tracked in the stats hash)
+      } else {
+        await this.store.deleteProcessedTaskQueue(workItemKey, sourceKey, destinationKey);
+      }
+      setImmediate(() => this.processWebHooks(hookEventCallback));
+    }
+  }
+
+  async enqueueWorkItems(keys: string[]): Promise<void> {
+    await this.store.addTaskQueues(keys);
+  }
+
+  async registerJobForCleanup(jobId: string, inSeconds = EXPIRATION_FIDELITY_SECONDS): Promise<void> {
+    if (inSeconds > -1) {
+      await this.store.expireJob(jobId, inSeconds);
+    }
+  }
+
+  async registerTimeHook(jobId: string, activityId: string, type: 'sleep'|'expire'|'cron', inSeconds = FIDELITY_SECONDS, multi?: RedisMulti): Promise<void> {
+    const awakenTimeSlot = Math.floor((Date.now() + inSeconds * 1000) / (FIDELITY_SECONDS * 1000)) * (FIDELITY_SECONDS * 1000); //n second awaken groups
+    await this.store.registerTimeHook(jobId, activityId, type, awakenTimeSlot, multi);
+  }
+
+  //todo: need 'scout' role in quorum to check for this and then alert the quorum to get to work
+  async processTimeHooks(timeEventCallback: (jobId: string, activityId: string) => Promise<void>, listKey?: string): Promise<void> {
+    try {
+      const job = await this.store.getNextTimeJob(listKey);
+      if (job) {
+        const [listKey, jobId, activityId] = job;
+        await timeEventCallback(jobId, activityId);
+        await sleepFor(0);
+        this.processTimeHooks(timeEventCallback, listKey);
+      } else {
+        let sleep = XSleepFor(FIDELITY_SECONDS * 1000);
+        this.cleanupTimeout = sleep.timerId;
+        await sleep.promise;
+        this.processTimeHooks(timeEventCallback)
+      }
+    } catch (err) {
+      //todo: retry connect to redis
+      this.logger.error('task-process-timehooks-error', err);
+    }
+  }
+
+  cancelCleanup() {
+    if (this.cleanupTimeout !== undefined) {
+      clearTimeout(this.cleanupTimeout);
+      this.cleanupTimeout = undefined;
+    }
+  }
+}
+
+export { TaskService };