@hotmeshio/hotmesh 0.0.54 → 0.0.56

package/services/serializer/index.ts

@@ -1,285 +0,0 @@
-import { getSymVal } from '../../modules/utils';
-import { Consumes } from '../../types/activity';
-import {
-  StringStringType,
-  StringAnyType,
-  SymbolMap,
-  SymbolMaps,
-  SymbolSets, 
-  Symbols } from '../../types/serializer';
-
-const dateReg = /^"\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z)?"$/;
-
-export const MDATA_SYMBOLS = {
-  SLOTS: 26,
-  ACTIVITY: {
-    KEYS: ['aid', 'dad', 'as', 'atp', 'stp', 'ac', 'au', 'err','l1s','l2s']
-  },
-  ACTIVITY_UPDATE: {
-    KEYS: ['au', 'err', 'l2s']
-  },
-  JOB: {
-    KEYS: ['ngn', 'tpc', 'pj', 'pg', 'pd', 'px', 'pa', 'key', 'app', 'vrs', 'jid', 'gid', 'aid', 'ts', 'jc', 'ju', 'js', 'err', 'trc']
-  },
-  JOB_UPDATE: {
-    KEYS: ['ju', 'err']
-  }
-};
-
-export class SerializerService {
-  dIds: StringStringType;
-  symKeys: SymbolMaps;
-  symReverseKeys: SymbolMaps;
-  symValMaps: SymbolMap;
-  symValReverseMaps: SymbolMap;
-
-  constructor() {
-    this.resetSymbols({}, {}, {});
-  }
-
-  abbreviate(consumes: Consumes, symbolNames: string[], fields: string[] = []): string[] {
-    for (const symbolName of symbolNames) {
-      const symbolSet = this.symKeys.get(symbolName);
-      const symbolPaths = consumes[symbolName];
-      for (const symbolPath of symbolPaths) {
-        const abbreviation = symbolSet.get(symbolPath);
-        if (abbreviation) {
-          const dimensionalIndex = this.resolveDimensionalIndex(symbolPath);
-          fields.push(`${abbreviation}${dimensionalIndex}`);
-        } else {
-          fields.push(symbolPath);
-        }
-      }
-    }
-    return fields;
-  }
-
-  resolveDimensionalIndex(path: string): string {
-    if (this.isJobPath(path)) {
-      return '';
-    } else {
-      const [activityId] = path.split('/');
-      if (activityId in this.dIds) {
-        return this.dIds[activityId];
-      } else if ('$ADJACENT' in this.dIds) {
-        //else=> pre-authorizing adjacent activity entry
-        return this.dIds['$ADJACENT'];
-      }
-      return ',0';
-    }
-  }
-
-  isJobPath(path: string): boolean {
-    return path.startsWith('data/') || path.startsWith('metadata/');
-  }
-
-  resetSymbols(symKeys: SymbolSets, symVals: Symbols, dIds: StringStringType): void {
-    this.symKeys = new Map();
-    this.symReverseKeys = new Map();
-    for (const id in symKeys) {
-      this.symKeys.set(id, new Map(Object.entries(symKeys[id])));
-    }
-    this.symValMaps = new Map(Object.entries(symVals));
-    this.symValReverseMaps = this.getReverseValueMap(this.symValMaps);
-    this.dIds = dIds;
-  }
-
-  getReverseKeyMap(keyMap: SymbolMap, id?: string): SymbolMap {
-    let map = this.symReverseKeys.get(id);
-    if (!map) {
-      map = new Map();
-      for (let [key, val] of keyMap.entries()) {
-        map.set(val, key);
-      }
-      this.symReverseKeys.set(id, map);
-    }
-    return map;
-  }
-
-  getReverseValueMap(valueMap: SymbolMap): SymbolMap {
-    const map = new Map();
-    for (let [key, val] of valueMap.entries()) {
-      map.set(val, key);
-    }
-    return map;
-  }
-
-  static filterSymVals(startIndex: number, maxIndex: number, existingSymbolValues: Symbols,  proposedValues: Set<string>): Symbols {
-    let newSymbolValues: Symbols = {};
-    let currentSymbolValues: Symbols = { ...existingSymbolValues };
-    let currentValuesSet: Set<string> = new Set(Object.values(currentSymbolValues));
-    for (let value of  proposedValues) {
-      if (!currentValuesSet.has(value)) {
-        if (startIndex > maxIndex) {
-          return newSymbolValues;
-        }
-        const symbol = getSymVal(startIndex);
-        startIndex++;
-        newSymbolValues[symbol] = value;
-        currentValuesSet.add(value);
-      }
-    }
-    return newSymbolValues;
-  }
-
-  compress(document: StringStringType, ids: string[]): StringStringType {
-    if (this.symKeys.size === 0) {
-      return document;
-    }
-    let source: StringStringType = { ...document };
-    let result: StringStringType = { };
-
-    const compressWithMap = (abbreviationMap: SymbolMap, id: string) => {
-      for (let key in source) {
-        if (key.startsWith(`${id}/`) || (id.startsWith('$') && ['data', 'metadata'].includes(key.split('/')[0]))) {
-          const dimensionalIndex = this.resolveDimensionalIndex(key);
-          let shortKey = abbreviationMap.get(key) || key;
-          const shortDimensionalKey = `${shortKey}${dimensionalIndex}`;
-          result[shortDimensionalKey] = source[key];
-        } else if (!(key in result) && this.isLiteralKeyType(key)) {
-          //mark (-) and search (_)
-          result[key] = source[key];
-        }  
-      }
-    };
-    for (let id of ids) {
-      const abbreviationMap = this.symKeys.get(id);
-      if (abbreviationMap) {
-        compressWithMap(abbreviationMap, id);
-      }
-    }
-    return result;
-  }
-
-  isLiteralKeyType(key: string): boolean {
-    return key.startsWith('-') || key.startsWith('_');
-  }
-
-  decompress(document: StringStringType, ids: string[]): StringStringType {
-    if (this.symKeys.size === 0) {
-      return document;
-    }
-    let result: StringStringType = { ...document };
-
-    const inflateWithMap = (abbreviationMap: SymbolMap, id: string) => {
-      const reversedAbbreviationMap = this.getReverseKeyMap(abbreviationMap, id);
-      for (let key in result) {
-        //strip dimensional index from key
-        const shortKey = key.split(',')[0];
-        let longKey = reversedAbbreviationMap.get(shortKey);
-        if (longKey) {
-          result[longKey] = result[key];
-          delete result[key];
-        }
-      }
-    };
-    for (let id of ids) {
-      const abbreviationMap = this.symKeys.get(id);
-      if (abbreviationMap) {
-        inflateWithMap(abbreviationMap, id);
-      }
-    }
-    return result;
-  }
-
-  //stringify: convert a multi-dimensional document to a 2-d hash
-  stringify(document: Record<string, any>): StringStringType {
-    let result: StringStringType = {};
-    for (let key in document) {
-      let value = SerializerService.toString(document[key]);
-      if (value) {
-        if (/^:*[a-zA-Z]{2}$/.test(value)) {
-          value = ':' + value;
-        } else if (this.symValReverseMaps.has(value)) {
-          value = this.symValReverseMaps.get(value);
-        }
-        result[key] = value;
-      }
-    }
-    return result;
-  }
-
-  //parse: convert a 2-d hash to a multi-dimensional document
-  parse(document: StringStringType): any {
-    let result: any = {};
-    for (let [key, value] of Object.entries(document)) {
-      if (value === undefined || value === null) continue;
-      if (/^:+[a-zA-Z]{2}$/.test(value)) {
-        result[key] = value.slice(1);
-      } else {
-        if (value?.length === 2 && this.symValMaps.has(value)) {
-          value = this.symValMaps.get(value);
-        }
-        result[key] = SerializerService.fromString(value);
-      }
-    }
-    return result;
-  }
-
-  static toString(value: any): string|undefined {
-    switch (typeof value) {
-      case 'string':
-        break;
-      case 'boolean':
-        value = value ? '/t' : '/f';
-        break;
-      case 'number':
-        value = '/d' + value.toString();
-        break;
-      case 'undefined':
-        return undefined;
-      case 'object':
-        if (value === null) {
-          value = '/n';
-        } else {
-          value = '/s' + JSON.stringify(value);
-        }
-        break;
-    }
-    return value;
-  }
-
-  static fromString(value: string|undefined): any {
-    if (typeof value !== 'string') return undefined;
-    const prefix = value.slice(0, 2);
-    const rest = value.slice(2);
-    switch (prefix) {
-      case '/t': // boolean true
-        return true;
-      case '/f': // boolean false
-        return false;
-      case '/d': // number
-        return Number(rest);
-      case '/n': // null
-        return null;
-      case '/s': // object (JSON string)
-        if (dateReg.exec(rest)) {
-          return new Date(JSON.parse(rest));
-        }
-        return JSON.parse(rest);
-      default: // string
-        return value;
-    }
-  }
-
-  public package(multiDimensionalDocument: StringAnyType, ids: string[]): StringStringType {
-    const flatDocument = this.stringify(multiDimensionalDocument);
-    return this.compress(flatDocument, ids);
-  }
-
-  public unpackage(document: StringStringType, ids: string[]): StringAnyType {
-    const multiDimensionalDocument = this.decompress(document, ids);
-    return this.parse(multiDimensionalDocument);
-  }
-
-  public export(): SymbolSets {
-    const obj: {[key: string]: StringStringType} = {};
-    for (const [id, map] of this.symKeys.entries()) {
-      obj[id] = {};
-      for (const [key, value] of map.entries()) {
-        obj[id][key] = value;
-      }
-    }
-    return obj;
-  }
-}

package/services/store/cache.ts

@@ -1,172 +0,0 @@
-/**
- * The Cache is a key/value store and used to store commonly accessed Redis metadata
- * (mainly the execution rules for the app) to save time accessing them as they
- * are immutable per verison. The only time the rules are ejected are when
- * a new version is deployed to the quorum and the cache is invalidated/cleared.
- */
-
-import { ActivityType } from "../../types/activity";
-import { HookRule } from "../../types/hook";
-import { HotMeshApp, HotMeshSettings } from "../../types/hotmesh";
-import { Symbols } from "../../types/serializer";
-import { Transitions } from "../../types/transition";
-
-class Cache {
-  settings: HotMeshSettings;
-  appId: string;
-  apps: Record<string, HotMeshApp>;
-  schemas: Record<string, ActivityType>;
-  subscriptions: Record<string, Record<string, string>>;
-  symbols: Record<string, Symbols>;
-  symvals: Record<string, Symbols>;
-  transitions: Record<string, Record<string, unknown>>;
-  hookRules: Record<string, Record<string, HookRule[]>>;
-  workItems: Record<string, string>;
-
-  /**
-   * The cache is ALWAYS initialized with HotMeshSettings. The other parameters are optional.
-   * @param settings 
-   * @param apps 
-   * @param schemas 
-   * @param subscriptions 
-   * @param transitions 
-   * @param hookRules 
-   */
-  constructor(appId: string, settings: HotMeshSettings, apps: Record<string, HotMeshApp> = {}, schemas: Record<string, ActivityType> = {}, subscriptions: Record<string, Record<string, string>> = {}, symbols: Record<string, Symbols> = {}, symvals: Record<string, Symbols> = {}, transitions: Record<string, Record<string, unknown>> = {}, hookRules: Record<string, Record<string, HookRule[]>> = {}, workItems: Record<string, string> = {}) {
-    this.appId = appId;
-    this.settings = settings;
-    this.apps = apps;
-    this.schemas = schemas;
-    this.subscriptions = subscriptions;
-    this.symbols = symbols;
-    this.symvals = symvals;
-    this.transitions = transitions;
-    this.hookRules = hookRules;
-    this.workItems = workItems;
-  }
-
-  /**
-   * invalidate the cache; settings are not invalidated!
-   */
-  invalidate(): void {
-    this.apps = {} as Record<string, HotMeshApp>;
-    this.schemas = {};
-    this.subscriptions = {};
-    this.transitions = {};
-    this.hookRules = {};
-  }
-
-  getSettings(): HotMeshSettings {
-    return this.settings;
-  }
-
-  setSettings(settings: HotMeshSettings): void {
-    this.settings = settings;
-  }
-
-  getApps(): Record<string, HotMeshApp> {
-    return this.apps;
-  }
-
-  getApp(appId: string): HotMeshApp {
-    return this.apps[appId] as HotMeshApp;
-  }
-
-  setApps(apps: Record<string, HotMeshApp>): void {
-    this.apps = apps;
-  }
-
-  setApp(appId: string, app: HotMeshApp): void {
-    this.apps[appId] = app;
-  }
-
-  getSchemas(appId: string, version: string): Record<string, ActivityType> {
-    return this.schemas[`${appId}/${version}`] as unknown as Record<string, ActivityType>;
-  }
-
-  getSchema(appId: string, version: string, activityId: string): ActivityType {
-    return this.schemas?.[`${appId}/${version}`]?.[activityId] as ActivityType;
-  }
-
-  setSchemas(appId: string, version: string, schemas: Record<string, ActivityType>): void {
-    this.schemas[`${appId}/${version}`] = schemas as unknown as Record<string, ActivityType>;
-  }
-
-  setSchema(appId: string, version: string, activityId: string, schema: ActivityType): void {
-    this.schemas[`${appId}/${version}`][activityId] = schema;
-  }
-
-  getSubscriptions(appId: string, version: string): Record<string, string> {
-    return this.subscriptions[`${appId}/${version}`];
-  }
-
-  getSubscription(appId: string, version: string, topic: string): unknown {
-    return this.subscriptions?.[`${appId}/${version}`]?.[topic];
-  }
-
-  setSubscriptions(appId: string, version: string, subscriptions: Record<string, string>): void {
-    this.subscriptions[`${appId}/${version}`] = subscriptions;
-  }
-
-  getSymbols(appId: string, targetEntityId: string): Symbols {
-    return this.symbols[`${appId}/${targetEntityId}`] as Symbols;
-  }
-
-  setSymbols(appId: string, targetEntityId: string, symbols: Symbols): void {
-    this.symbols[`${appId}/${targetEntityId}`] = symbols;
-  }
-
-  deleteSymbols(appId: string, targetEntityId: string): void {
-    delete this.symbols[`${appId}/${targetEntityId}`];
-  }
-
-  getSymbolValues(appId: string): Symbols {
-    return this.symvals[`${appId}`] as Symbols;
-  }
-
-  setSymbolValues(appId: string, symvals: Symbols): void {
-    this.symvals[`${appId}`] = symvals;
-  }
-
-  deleteSymbolValues(appId: string): void {
-    delete this.symvals[`${appId}`];
-  }
-
-  getTransitions(appId: string, version: string): Transitions {
-    return this.transitions[`${appId}/${version}`] as Transitions;
-  }
-
-  setTransitions(appId: string, version: string, transitions: Transitions): void {
-    this.transitions[`${appId}/${version}`] = transitions;
-  }
-
-  getHookRules(appId: string): Record<string, HookRule[]> {
-    return this.hookRules[`${appId}`];
-  }
-
-  setHookRules(appId: string, hookRules: Record<string, HookRule[]>): void {
-    this.hookRules[`${appId}`] = hookRules;
-  }
-
-  getSignals(appId: string, version: string): Record<string, unknown> {
-    throw new Error("SIGNAL (getHooks) is not supported");
-  }
-
-  setSignals(appId: string, version: string): Record<string, unknown> {
-    throw new Error("SIGNAL (setHook) is not supported");
-  }
-
-  getActiveTaskQueue(appId: string): string {
-    return this.workItems[appId];
-  }
-
-  setWorkItem(appId: string, workItem: string): void {
-    this.workItems[appId] = workItem;
-  }
-
-  removeWorkItem(appId: string): void {
-    delete this.workItems[appId];
-  }
-}
-
-export { Cache };

package/services/store/clients/ioredis.ts

@@ -1,145 +0,0 @@
-import { KeyType } from '../../../modules/key';
-import { ILogger } from '../../logger';
-import { SerializerService as Serializer } from '../../serializer';
-import { Cache } from '../cache';
-import { StoreService } from '../index';
-import { RedisClientType, RedisMultiType } from '../../../types/ioredisclient';
-import { ReclaimedMessageType } from '../../../types/stream';
-
-class IORedisStoreService extends StoreService<RedisClientType, RedisMultiType> {
-  redisClient: RedisClientType;
-  cache: Cache;
-  namespace: string;
-  appId: string;
-  logger: ILogger;
-  serializer: Serializer;
-
-  constructor(redisClient: RedisClientType) {
-    super(redisClient);
-  }
-
-  getMulti(): RedisMultiType {
-    return this.redisClient.multi();
-  }
-
-  async exec(...args: any[]): Promise<string|string[]|string[][]> {
-    const response = await this.redisClient.call.apply(this.redisClient, args as any);
-    if (typeof response === 'string') {
-      return response as string;
-    } else if (Array.isArray(response)) {
-      if (Array.isArray(response[0])) {
-        return response as string[][];
-      }
-      return response as string[];
-    }
-    return response;
-  }
-
-  hGetAllResult(result: any) {
-    //ioredis response signature is [null, {}] or [null, null]
-    return result[1];
-  }
-
-  async addTaskQueues(keys: string[]): Promise<void> {
-    const multi = this.redisClient.multi();
-    const zsetKey = this.mintKey(KeyType.WORK_ITEMS, { appId: this.appId });
-    for (const key of keys) {
-      multi.zadd(zsetKey, 'NX', Date.now(), key);
-    }
-    await multi.exec();
-  }
-
-  async publish(keyType: KeyType.QUORUM, message: Record<string, any>, appId: string, engineId?: string): Promise<boolean> {
-    const topic = this.mintKey(keyType, { appId, engineId });
-    const status: number = await this.redisClient.publish(topic, JSON.stringify(message));
-    return status === 1;
-  }
-
-  async xgroup(command: 'CREATE', key: string, groupName: string, id: string, mkStream?: 'MKSTREAM'): Promise<boolean> {
-    if (mkStream === 'MKSTREAM') {
-      try {
-        return (await this.redisClient.xgroup(command, key, groupName, id, mkStream)) === 'OK';
-      } catch (err) {
-        this.logger.debug(`Consumer group not created with MKSTREAM for key: ${key} and group: ${groupName}`);
-        throw err;
-      }
-    } else {
-      try {
-        return (await this.redisClient.xgroup(command, key, groupName, id)) === 'OK';
-      } catch (err) {
-        this.logger.debug(`Consumer group not created for key: ${key} and group: ${groupName}`);
-        throw err;
-      }
-    }
-  }
-
-  async xadd(key: string, id: string, messageId: string, messageValue: string, multi?: RedisMultiType): Promise<string | RedisMultiType> {
-    try {
-      return await (multi || this.redisClient).xadd(key, id, messageId, messageValue);
-    } catch (error) {
-      this.logger.error(`Error publishing 'xadd'; key: ${key}`, { ...error });
-      throw error;
-    }
-  }
-
-  async xpending(
-    key: string,
-    group: string,
-    start?: string,
-    end?: string,
-    count?: number,
-    consumer?: string
-  ): Promise<[string, string, number, [string, number][]][] | [string, string, number, number] | unknown[]> {
-    try {
-      return await this.redisClient.xpending(key, group, start, end, count, consumer);
-    } catch (error) {
-      this.logger.error(`Error in retrieving pending messages for [stream ${key}], [group ${group}]`, { ...error });
-      throw error;
-    }
-  }
-
-  async xclaim(
-    key: string,
-    group: string,
-    consumer: string,
-    minIdleTime: number,
-    id: string,
-    ...args: string[]
-  ): Promise<ReclaimedMessageType> {
-    try {
-      return await this.redisClient.xclaim(key, group, consumer, minIdleTime, id, ...args) as unknown as ReclaimedMessageType;
-    } catch (error) {
-      this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${key}`, { ...error });
-      throw error;
-    }
-  }
-
-  async xack(key: string, group: string, id: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
-    try {
-      return await (multi || this.redisClient).xack(key, group, id);
-    } catch (error) {
-      this.logger.error(`Error in acknowledging messages in group: ${group} for key: ${key}`, { ...error });
-      throw error;
-    }
-  }
-
-  async xdel(key: string, id: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
-    try {
-      return await (multi || this.redisClient).xdel(key, id);
-    } catch (error) {
-      this.logger.error(`Error in deleting messages with id: ${id} for key: ${key}`, { ...error });
-      throw error;
-    }
-  }
-
-  async xlen(key: string, multi? : RedisMultiType): Promise<number|RedisMultiType> {
-    try {
-      return await (multi || this.redisClient).xlen(key);
-    } catch (error) {
-      this.logger.error(`Error getting stream depth: ${key}`, { ...error });
-      throw error;
-    }
-  }
-}
-
-export { IORedisStoreService };