@hotmeshio/hotmesh 0.0.1

package/services/store/cache.ts

@@ -0,0 +1,172 @@
+/**
+ * The Cache is a key/value store and used to store commonly accessed Redis metadata
+ * (mainly the execution rules for the app) to save time accessing them as they
+ * are immutable per verison. The only time the rules are ejected are when
+ * a new version is deployed to the quorum and the cache is invalidated/cleared.
+ */
+
+import { ActivityType } from "../../types/activity";
+import { HookRule } from "../../types/hook";
+import { HotMeshApp, HotMeshSettings } from "../../types/hotmesh";
+import { Symbols } from "../../types/serializer";
+import { Transitions } from "../../types/transition";
+
+class Cache {
+  settings: HotMeshSettings;
+  appId: string;
+  apps: Record<string, HotMeshApp>;
+  schemas: Record<string, ActivityType>;
+  subscriptions: Record<string, Record<string, string>>;
+  symbols: Record<string, Symbols>;
+  symvals: Record<string, Symbols>;
+  transitions: Record<string, Record<string, unknown>>;
+  hookRules: Record<string, Record<string, HookRule[]>>;
+  workItems: Record<string, string>;
+
+  /**
+   * The cache is ALWAYS initialized with HotMeshSettings. The other parameters are optional.
+   * @param settings 
+   * @param apps 
+   * @param schemas 
+   * @param subscriptions 
+   * @param transitions 
+   * @param hookRules 
+   */
+  constructor(appId: string, settings: HotMeshSettings, apps: Record<string, HotMeshApp> = {}, schemas: Record<string, ActivityType> = {}, subscriptions: Record<string, Record<string, string>> = {}, symbols: Record<string, Symbols> = {}, symvals: Record<string, Symbols> = {}, transitions: Record<string, Record<string, unknown>> = {}, hookRules: Record<string, Record<string, HookRule[]>> = {}, workItems: Record<string, string> = {}) {
+    this.appId = appId;
+    this.settings = settings;
+    this.apps = apps;
+    this.schemas = schemas;
+    this.subscriptions = subscriptions;
+    this.symbols = symbols;
+    this.symvals = symvals;
+    this.transitions = transitions;
+    this.hookRules = hookRules;
+    this.workItems = workItems;
+  }
+
+  /**
+   * invalidate the cache; settings are not invalidated!
+   */
+  invalidate(): void {
+    this.apps = {} as Record<string, HotMeshApp>;
+    this.schemas = {};
+    this.subscriptions = {};
+    this.transitions = {};
+    this.hookRules = {};
+  }
+
+  getSettings(): HotMeshSettings {
+    return this.settings;
+  }
+
+  setSettings(settings: HotMeshSettings): void {
+    this.settings = settings;
+  }
+
+  getApps(): Record<string, HotMeshApp> {
+    return this.apps;
+  }
+
+  getApp(appId: string): HotMeshApp {
+    return this.apps[appId] as HotMeshApp;
+  }
+
+  setApps(apps: Record<string, HotMeshApp>): void {
+    this.apps = apps;
+  }
+
+  setApp(appId: string, app: HotMeshApp): void {
+    this.apps[appId] = app;
+  }
+
+  getSchemas(appId: string, version: string): Record<string, ActivityType> {
+    return this.schemas[`${appId}/${version}`] as unknown as Record<string, ActivityType>;
+  }
+
+  getSchema(appId: string, version: string, activityId: string): ActivityType {
+    return this.schemas?.[`${appId}/${version}`]?.[activityId] as ActivityType;
+  }
+
+  setSchemas(appId: string, version: string, schemas: Record<string, ActivityType>): void {
+    this.schemas[`${appId}/${version}`] = schemas as unknown as Record<string, ActivityType>;
+  }
+
+  setSchema(appId: string, version: string, activityId: string, schema: ActivityType): void {
+    this.schemas[`${appId}/${version}`][activityId] = schema;
+  }
+
+  getSubscriptions(appId: string, version: string): Record<string, string> {
+    return this.subscriptions[`${appId}/${version}`];
+  }
+
+  getSubscription(appId: string, version: string, topic: string): unknown {
+    return this.subscriptions?.[`${appId}/${version}`]?.[topic];
+  }
+
+  setSubscriptions(appId: string, version: string, subscriptions: Record<string, string>): void {
+    this.subscriptions[`${appId}/${version}`] = subscriptions;
+  }
+
+  getSymbols(appId: string, targetEntityId: string): Symbols {
+    return this.symbols[`${appId}/${targetEntityId}`] as Symbols;
+  }
+
+  setSymbols(appId: string, targetEntityId: string, symbols: Symbols): void {
+    this.symbols[`${appId}/${targetEntityId}`] = symbols;
+  }
+
+  deleteSymbols(appId: string, targetEntityId: string): void {
+    delete this.symbols[`${appId}/${targetEntityId}`];
+  }
+
+  getSymbolValues(appId: string): Symbols {
+    return this.symvals[`${appId}`] as Symbols;
+  }
+
+  setSymbolValues(appId: string, symvals: Symbols): void {
+    this.symvals[`${appId}`] = symvals;
+  }
+
+  deleteSymbolValues(appId: string): void {
+    delete this.symvals[`${appId}`];
+  }
+
+  getTransitions(appId: string, version: string): Transitions {
+    return this.transitions[`${appId}/${version}`] as Transitions;
+  }
+
+  setTransitions(appId: string, version: string, transitions: Transitions): void {
+    this.transitions[`${appId}/${version}`] = transitions;
+  }
+
+  getHookRules(appId: string): Record<string, HookRule[]> {
+    return this.hookRules[`${appId}`];
+  }
+
+  setHookRules(appId: string, hookRules: Record<string, HookRule[]>): void {
+    this.hookRules[`${appId}`] = hookRules;
+  }
+
+  getSignals(appId: string, version: string): Record<string, unknown> {
+    throw new Error("SIGNAL (getHooks) is not supported");
+  }
+
+  setSignals(appId: string, version: string): Record<string, unknown> {
+    throw new Error("SIGNAL (setHook) is not supported");
+  }
+
+  getActiveTaskQueue(appId: string): string {
+    return this.workItems[appId];
+  }
+
+  setWorkItem(appId: string, workItem: string): void {
+    this.workItems[appId] = workItem;
+  }
+
+  removeWorkItem(appId: string): void {
+    delete this.workItems[appId];
+  }
+}
+
+export { Cache };

package/services/store/clients/ioredis.ts

@@ -0,0 +1,123 @@
+import { KeyType } from '../../../modules/key';
+import { ILogger } from '../../logger';
+import { SerializerService as Serializer } from '../../serializer';
+import { Cache } from '../cache';
+import { StoreService } from '../index';
+import { RedisClientType, RedisMultiType } from '../../../types/ioredisclient';
+import { ReclaimedMessageType } from '../../../types/stream';
+
+class IORedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
+  redisClient: RedisClientType;
+  cache: Cache;
+  namespace: string;
+  appId: string;
+  logger: ILogger;
+  serializer: Serializer;
+
+  constructor(redisClient: RedisClientType) {
+    super(redisClient);
+  }
+
+  getMulti(): RedisMultiType {
+    return this.redisClient.multi();
+  }
+
+  hGetAllResult(result: any) {
+    //ioredis response signature is [null, {}] or [null, null]
+    return result[1];
+  }
+
+  async addTaskQueues(keys: string[]): Promise<void> {
+    const multi = this.redisClient.multi();
+    const zsetKey = this.mintKey(KeyType.WORK_ITEMS, { appId: this.appId });
+    for (const key of keys) {
+      multi.zadd(zsetKey, 'NX', Date.now(), key);
+    }
+    await multi.exec();
+  }
+
+  async publish(keyType: KeyType.QUORUM, message: Record<string, any>, appId: string, engineId?: string): Promise<boolean> {
+    const topic = this.mintKey(keyType, { appId, engineId });
+    const status: number = await this.redisClient.publish(topic, JSON.stringify(message));
+    return status === 1;
+  }
+
+  async xgroup(command: 'CREATE', key: string, groupName: string, id: string, mkStream?: 'MKSTREAM'): Promise<boolean> {
+    if (mkStream === 'MKSTREAM') {
+      try {
+        return (await this.redisClient.xgroup(command, key, groupName, id, mkStream)) === 'OK';
+      } catch (err) {
+        this.logger.info(`Consumer group not created with MKSTREAM for key: ${key} and group: ${groupName}`);
+        throw err;
+      }
+    } else {
+      try {
+        return (await this.redisClient.xgroup(command, key, groupName, id)) === 'OK';
+      } catch (err) {
+        this.logger.info(`Consumer group not created for key: ${key} and group: ${groupName}`);
+        throw err;
+      }
+    }
+  }
+
+  async xadd(key: string, id: string, messageId: string, messageValue: string, multi?: RedisMultiType): Promise<string | RedisMultiType> {
+    try {
+      return await (multi || this.redisClient).xadd(key, id, messageId, messageValue);
+    } catch (err) {
+      this.logger.error(`Error publishing 'xadd'; key: ${key}`, err);
+      throw err;
+    }
+  }
+
+  async xpending(
+    key: string,
+    group: string,
+    start?: string,
+    end?: string,
+    count?: number,
+    consumer?: string
+  ): Promise<[string, string, number, [string, number][]][] | [string, string, number, number] | unknown[]> {
+    try {
+      return await this.redisClient.xpending(key, group, start, end, count, consumer);
+    } catch (err) {
+      this.logger.error(`Error in retrieving pending messages for [stream ${key}], [group ${group}]`, err);
+      throw err;
+    }
+  }
+
+  async xclaim(
+    key: string,
+    group: string,
+    consumer: string,
+    minIdleTime: number,
+    id: string,
+    ...args: string[]
+  ): Promise<ReclaimedMessageType> {
+    try {
+      return await this.redisClient.xclaim(key, group, consumer, minIdleTime, id, ...args) as unknown as ReclaimedMessageType;
+    } catch (err) {
+      this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, err);
+      throw err;
+    }
+  }
+
+  async xack(key: string, group: string, id: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
+    try {
+      return await (multi || this.redisClient).xack(key, group, id);
+    } catch (err) {
+      this.logger.error(`Error in acknowledging messages in group: ${group} for key: ${key}`, err);
+      throw err;
+    }
+  }
+
+  async xdel(key: string, id: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
+    try {
+      return await (multi || this.redisClient).xdel(key, id);
+    } catch (err) {
+      this.logger.error(`Error in deleting messages with id: ${id} for key: ${key}`, err);
+      throw err;
+    }
+  }
+}
+
+export { IORedisStoreService };

package/services/store/clients/redis.ts

@@ -0,0 +1,169 @@
+import { KeyType } from '../../../modules/key';
+import { ILogger } from '../../logger';
+import { SerializerService as Serializer } from '../../serializer';
+import { Cache } from '../cache';
+import { StoreService } from '../index';
+import { RedisClientType, RedisMultiType } from '../../../types/redisclient';
+import { ReclaimedMessageType } from '../../../types/stream';
+
+class RedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
+  redisClient: RedisClientType;
+  cache: Cache;
+  namespace: string;
+  appId: string;
+  logger: ILogger;
+  serializer: Serializer;
+  commands: Record<string, string>;
+
+  constructor(redisClient: RedisClientType) {
+    super(redisClient);
+    this.commands = {
+      setnx: 'SETNX',
+      del: 'DEL',
+      expire: 'EXPIRE',
+      hset: 'HSET',
+      hsetnx: 'HSETNX',
+      hincrby: 'HINCRBY',
+      hdel: 'HDEL',
+      hget: 'HGET',
+      hmget: 'HMGET',
+      hgetall: 'HGETALL',
+      hincrbyfloat: 'HINCRBYFLOAT',
+      zrange: 'ZRANGE',
+      zrangebyscore_withscores: 'ZRANGEBYSCORE_WITHSCORES',
+      zrangebyscore: 'ZRANGEBYSCORE',
+      zrem: 'ZREM',
+      zadd: 'ZADD',
+      lmove: 'LMOVE',
+      lrange: 'LRANGE',
+      llen: 'LLEN',
+      lpop: 'LPOP',
+      rename: 'RENAME',
+      rpush: 'RPUSH',
+      xack: 'XACK',
+      xdel: 'XDEL',
+    };
+  }
+
+  getMulti(): RedisMultiType {
+    const multi = this.redisClient.MULTI();
+    return multi as unknown as RedisMultiType;
+  }
+
+  async publish(keyType: KeyType.QUORUM, message: Record<string, any>, appId: string, engineId?: string): Promise<boolean> {
+    const topic = this.mintKey(keyType, { appId, engineId });
+    const status: number = await this.redisClient.publish(topic, JSON.stringify(message));
+    return this.isSuccessful(status);
+  }
+
+  async zAdd(key: string, score: number | string, value: string | number, redisMulti?: RedisMultiType): Promise<any> {
+    return await (redisMulti || this.redisClient)[this.commands.zadd](key, { score: score, value: value.toString() } as any);
+  }
+
+  async zRangeByScoreWithScores(key: string, score: number | string, value: string | number): Promise<string | null> {
+    const result = await this.redisClient[this.commands.zrangebyscore_withscores](key, score, value);
+    if (result?.length > 0) {
+      return result[0];
+    }
+    return null;
+  }
+
+  async zRangeByScore(key: string, score: number | string, value: string | number): Promise<string | null> {
+    const result = await this.redisClient[this.commands.zrangebyscore](key, score, value);
+    if (result?.length > 0) {
+      return result[0];
+    }
+    return null;
+  }
+
+  async xgroup(command: 'CREATE', key: string, groupName: string, id: string, mkStream?: 'MKSTREAM'): Promise<boolean> {
+    const args = mkStream === 'MKSTREAM' ? ['MKSTREAM'] : [];
+    try {
+      return (await this.redisClient.sendCommand(['XGROUP', 'CREATE', key, groupName, id, ...args])) === 1;
+    } catch (err) {
+      const streamType = mkStream === 'MKSTREAM' ? 'with MKSTREAM' : 'without MKSTREAM';
+      this.logger.warn(`x-group-error ${streamType} for key: ${key} and group: ${groupName}`, err);
+      throw err;
+    }
+  }
+
+  async xadd(key: string, id: string, ...args: any[]): Promise<string | RedisMultiType> {
+    let multi: RedisMultiType;
+    if (typeof args[args.length - 1] !== 'string') {
+      multi = args.pop() as RedisMultiType;
+    }
+    try {
+      return await (multi || this.redisClient).XADD(key, id, { [args[0]]: args[1] });
+    } catch (err) {
+      this.logger.error(`Error publishing 'xadd'; key: ${key}`, err);
+      throw err;
+    }
+  }
+
+  async xpending(
+    key: string,
+    group: string,
+    start?: string,
+    end?: string,
+    count?: number,
+    consumer?: string
+  ): Promise<[string, string, number, [string, number][]][] | [string, string, number, number]> {
+    try {
+      const args = [key, group];
+      if (start) args.push(start);
+      if (end) args.push(end);
+      if (count !== undefined) args.push(count.toString());
+      if (consumer) args.push(consumer);
+      return await this.redisClient.sendCommand(['XPENDING', ...args]);
+    } catch (err) {
+      this.logger.error(`Error in retrieving pending messages for group: ${group} in key: ${key}`, err);
+      throw err;
+    }
+  }
+  
+  async xclaim(
+    key: string,
+    group: string,
+    consumer: string,
+    minIdleTime: number,
+    id: string,
+    ...args: string[]
+  ): Promise<ReclaimedMessageType> {
+    try {
+      return await this.redisClient.sendCommand(['XCLAIM', key, group, consumer, minIdleTime.toString(), id, ...args]) as unknown as ReclaimedMessageType;
+    } catch (err) {
+      this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, err);
+      throw err;
+    }
+  }
+
+  async xack(key: string, group: string, id: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
+    try {
+      if (multi) {
+        multi[this.commands.xack](key, group, id);
+        return multi;
+      } else {
+        return await this.redisClient[this.commands.xack](key, group, id);
+      }
+    } catch (err) {
+      this.logger.error(`Error in acknowledging messages in group: ${group} for key: ${key}`, err);
+      throw err;
+    }
+  }
+
+  async xdel(key: string, id: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
+    try {
+      if (multi) {
+        multi[this.commands.xdel](key, id);
+        return multi;
+      } else {
+        return await this.redisClient[this.commands.xdel](key, id);
+      }
+    } catch (err) {
+      this.logger.error(`Error in deleting messages with ids: ${id} for key: ${key}`, err);
+      throw err;
+    }
+  }
+}
+
+export { RedisStoreService };