@hotmeshio/hotmesh 0.0.55 → 0.0.56

package/services/pipe/functions/string.ts

@@ -1,86 +0,0 @@
-class StringHandler {
-  split(input: string, delimiter: string): string[] {
-    return input.split(delimiter);
-  }
-
-  charAt(input: string, index: number): string {
-    return input.charAt(index);
-  }
-
-  concat(...strings: string[]): string {
-    return strings.join('');
-  }
-
-  includes(input: string, searchString: string, position?: number): boolean {
-    return input.includes(searchString, position);
-  }
-
-  indexOf(input: string, searchString: string, fromIndex?: number): number {
-    return input.indexOf(searchString, fromIndex);
-  }
-
-  lastIndexOf(input: string, searchString: string, fromIndex?: number): number {
-    return input.lastIndexOf(searchString, fromIndex);
-  }
-
-  slice(input: string, start?: number, end?: number): string {
-    return input.slice(start, end);
-  }
-
-  toLowerCase(input: string): string {
-    return input.toLowerCase();
-  }
-
-  toUpperCase(input: string): string {
-    return input.toUpperCase();
-  }
-
-  trim(input: string): string {
-    return input.trim();
-  }
-
-  trimStart(input: string): string {
-    return input.trimStart();
-  }
-
-  trimEnd(input: string): string {
-    return input.trimEnd();
-  }
-
-  padStart(input: string, maxLength: number, padString?: string): string {
-    return input.padStart(maxLength, padString);
-  }
-
-  padEnd(input: string, maxLength: number, padString?: string): string {
-    return input.padEnd(maxLength, padString);
-  }
-
-  replace(input: string, searchValue: string | RegExp, replaceValue: string): string {
-    return input.replace(searchValue, replaceValue);
-  }
-
-  search(input: string, regexp: RegExp): number {
-    return input.search(regexp);
-  }
-
-  substring(input: string, start: number, end?: number): string {
-    return input.substring(start, end);
-  }
-
-  startsWith(str: string, searchString: string, position?: number): boolean {
-    return str.startsWith(searchString, position);
-  }
-
-  endsWith(str: string, searchString: string, length?: number): boolean {
-    return str.endsWith(searchString, length);
-  }
-
-  repeat(str: string, count: number): string {
-    if (count < 0 || count === Infinity) {
-      throw new RangeError('Invalid repeat count. Must be a positive finite number.');
-    }
-    return str.repeat(count);
-  }
-}
-  
-export { StringHandler };

package/services/pipe/functions/symbol.ts

@@ -1,39 +0,0 @@
-class SymbolHandler {
-  null(): null {
-    return null;
-  }
-
-  undefined(): undefined {
-    return undefined;
-  }
-
-  whitespace(): string {
-    return ' ';
-  }
-
-  object(): object {
-    return {};
-  }
-
-  array(): any[] {
-    return [];
-  }
-
-  posInfinity(): number {
-    return Infinity;
-  }
-
-  negInfinity(): number {
-    return -Infinity;
-  }
-
-  NaN(): number {
-    return NaN;
-  }
-
-  date(): Date {
-    return new Date();
-  }
-}
-
-export { SymbolHandler };

package/services/pipe/functions/unary.ts

@@ -1,19 +0,0 @@
-class UnaryHandler {
-  not(value: boolean): boolean {
-    return !value;
-  }
-
-  positive(value: number): number {
-    return +value;
-  }
-
-  negative(value: number): number {
-    return -value;
-  }
-
-  bitwise_not(value: number): number {
-    return ~value;
-  }
-}
-
-export { UnaryHandler };

package/services/pipe/index.ts

@@ -1,216 +0,0 @@
-import FUNCTIONS from './functions'
-import { JobState, JobData, JobsData } from '../../types/job';
-import {
-  PipeContext,
-  PipeItem,
-  PipeItems,
-  PipeObject,
-  Pipe as PipeType,
-  ReduceObject } from '../../types/pipe';
-
-class Pipe {
-  rules: PipeType;
-  jobData: JobData;
-  context: PipeContext;
-
-  constructor(rules: PipeType, jobData: JobData, context?: PipeContext) {
-    this.rules = rules;
-    this.jobData = jobData;
-    this.context = context;
-  }
-
-  private isPipeType(currentRow: PipeItem[] | PipeType | PipeObject | ReduceObject): currentRow is PipeType {
-    return !Array.isArray(currentRow) && '@pipe' in currentRow;
-  }
-
-  private isreduceType(currentRow: PipeItem[] | PipeType | PipeObject | ReduceObject): currentRow is PipeType {
-    return !Array.isArray(currentRow) && '@reduce' in currentRow;
-  }
-
-
-  static isPipeObject(obj: { [key: string]: unknown } | PipeItem): boolean {
-    return typeof obj === 'object' && obj !== null && !Array.isArray(obj) && '@pipe' in obj;
-  }
-
-  static resolve(unresolved: { [key: string]: unknown }|PipeItem, context: Partial<JobState>): any {
-    let pipe: Pipe;
-    if (Pipe.isPipeObject(unresolved)) {
-      pipe = new Pipe(unresolved['@pipe'], context);
-    } else {
-      pipe = new Pipe([[unresolved as unknown as PipeItem]], context);
-    }
-    return pipe.process();
-  }
-
-  /**
-   * loop through each PipeItem row in this Pipe, resolving and transforming line by line
-   * @returns {any} the result of the pipe
-   */
-  process(resolved: unknown[] | null = null): any {
-    let index = 0;
-    if (!(resolved || this.isPipeType(this.rules[0]) || this.isreduceType(this.rules[0]))) {
-      resolved = this.processCells(this.rules[0] as PipeItem[]); // Add type assertion
-      index = 1;
-    }
-    const len = this.rules.length;
-    const subPipeQueue = [];
-    for (let i = index; i < len; i++) {
-      resolved = this.processRow(this.rules[i], resolved, subPipeQueue);
-    }
-    return resolved[0];
-  }
-
-  /**
-   * Transforms iterable `input` into a single value. Vars $output, $item, $key
-   * and $input are available. The final statement in the iterator (the reduction)
-   * is assumed to be the return value. A default $output object may be provided
-   * to the iterator by placing the the second cell of the preceding row. Otherwise,
-   * construct the object during first run and ensure it is the first cell of the
-   * last row of the iterator, so it is returned as the $output for the next cycle
-   * @param {unknown[]} input 
-   * @returns {unknown}
-   * @private
-   */
-  reduce(input: Array<unknown[]>): unknown {
-    let resolved = input[1] ?? null;
-
-    if (Array.isArray(input[0])) {
-      for (let index = 0; index < input[0].length; index++) {
-        this.context = { $input: input[0], $output: resolved, $item: input[0][index], $key: index.toString(), $index: index };
-        resolved = this.process([resolved]);
-      }
-    } else {
-      let index = -1;
-      for (let $key in (input[0] as Record<string, unknown>)) {
-        index++;
-        this.context = { $input: input[0], $output: resolved, $item: input[0][$key], $key, $index: index };
-        resolved = this.process([resolved]);
-      }
-    }
-    return [resolved];
-  }
-
-  private processRow(currentRow: PipeItem[] | PipeType | PipeObject | ReduceObject , resolvedPriorRow: unknown[]|null, subPipeQueue: unknown[]): PipeItem[] {
-    if (resolvedPriorRow && this.isreduceType(currentRow)) {
-      //reduce the resolvedPriorRow and return the output from the reducer
-      const subPipe = new Pipe(currentRow['@reduce'], this.jobData);
-      const reduced = subPipe.reduce(resolvedPriorRow as Array<unknown[]>)
-      return reduced as PipeItem[];
-    } else if (this.isPipeType(currentRow)) {
-      //process subPipe and push to subPipeQueue; echo resolvedPriorRow
-      const subPipe = new Pipe(currentRow['@pipe'], this.jobData, this.context);
-      subPipeQueue.push(subPipe.process());
-      return resolvedPriorRow as PipeItem[];
-    } else {
-      //pivot the subPipeQueue into the arguments array (resolvedPriorRow)
-      if (subPipeQueue.length > 0) {
-        resolvedPriorRow = [...subPipeQueue];
-        subPipeQueue.length = 0;
-      }
-
-      if (!resolvedPriorRow) {
-        //if no prior row, use current row as prior row
-        return [].concat(this.processCells(Array.isArray(currentRow) ? [...currentRow] as PipeItem[] : []));
-      } else {
-        const [functionName, ...params] = currentRow as PipeItem[]; // Add type assertion
-        //use resolved values from prior row (n - 1) as input params to cell 1 function
-        let resolvedValue: unknown;
-        if (this.isContextVariable(functionName)) {
-          resolvedValue = this.resolveContextValue(functionName as string);
-        } else {
-          resolvedValue = Pipe.resolveFunction(functionName as string)(...resolvedPriorRow);
-        }
-        //resolve remaining cells in row and return concatenated with resolvedValue
-        return [(resolvedValue as PipeItem)].concat(this.processCells([...params]));
-      }
-    }
-  }
-
-  static resolveFunction(functionName: string) {
-    let [prefix, suffix] = functionName.split('.');
-    prefix = prefix.substring(2);
-    suffix = suffix.substring(0, suffix.length - 1);
-    const domain = FUNCTIONS[prefix];
-    if (!domain) {
-      throw new Error(`Unknown domain name [${functionName}]: ${prefix}`);
-    }
-    if (!domain[suffix]) {
-      throw new Error(`Unknown domain function [${functionName}]: ${prefix}.${suffix}`);
-    }
-    return domain[suffix];
-  }
-
-  processCells(cells: PipeItems): unknown[] {
-    const resolved = [];
-    if (Array.isArray(cells)) {
-      for (const currentCell of cells) {
-        resolved.push(this.resolveCellValue(currentCell));
-      }
-    }
-    return resolved;
-  }
-
-  private isFunction(currentCell: PipeItem): boolean {
-    return typeof currentCell === 'string' && currentCell.startsWith('{@') && currentCell.endsWith('}');
-  }
-
-  private isContextVariable(currentCell: PipeItem): boolean {
-    if(typeof currentCell === 'string' && currentCell.endsWith('}')) {
-      return currentCell.startsWith('{$item') || currentCell.startsWith('{$key') || currentCell.startsWith('{$index') || currentCell.startsWith('{$input') || currentCell.startsWith('{$output');
-    }
-  }
-
-  private isMappable(currentCell: PipeItem): boolean {
-    return typeof currentCell === 'string' &&  currentCell.startsWith('{') && currentCell.endsWith('}');
-  }
-
-  resolveCellValue(currentCell: PipeItem): unknown {
-    if (this.isFunction(currentCell)) {
-      const fn = Pipe.resolveFunction(currentCell as string);
-      return fn.call();
-    } else if (this.isContextVariable(currentCell)) {
-      return this.resolveContextValue(currentCell as string);
-    } else if (this.isMappable(currentCell)) {
-      return this.resolveMappableValue(currentCell as string);
-    } else {
-      return currentCell;
-    }
-  }
-
-  private getNestedProperty(obj: JobsData|unknown, path: string): any {
-    const pathParts = path.split('.');
-    let current = obj;
-    for (const part of pathParts) {
-      if (current === null || typeof current !== 'object' || !current.hasOwnProperty(part)) {
-        return undefined;
-      }
-      current = current[part];
-    }
-
-    return current;
-  }
-
-  resolveMappableValue(currentCell: string): unknown {
-    const term = this.resolveMapTerm(currentCell);
-    return this.getNestedProperty(this.jobData, term);
-  }
-
-  resolveContextValue(currentCell: string): unknown {
-    const term = this.resolveContextTerm(currentCell);
-    return this.getNestedProperty(this.context, term);
-  }
-
-  resolveContextTerm(currentCell: string): string {
-    return currentCell.substring(1, currentCell.length - 1);
-  }
-
-  resolveFunctionTerm(currentCell: string): string {
-    return currentCell.substring(2, currentCell.length - 1);
-  }
-
-  resolveMapTerm(currentCell: string): string {
-    return currentCell.substring(1, currentCell.length - 1);
-  }
-}
-
-export { Pipe };

package/services/quorum/index.ts

@@ -1,319 +0,0 @@
-import {
-  HMSH_ACTIVATION_MAX_RETRY,
-  HMSH_QUORUM_DELAY_MS, 
-  HMSH_QUORUM_ROLLCALL_CYCLES} from '../../modules/enums';
-import {
-  XSleepFor,
-  formatISODate,
-  getSystemHealth,
-  identifyRedisType,
-  sleepFor } from '../../modules/utils';
-import { CompilerService } from '../compiler';
-import { EngineService } from '../engine';
-import { ILogger } from '../logger';
-import { StoreService } from '../store';
-import { IORedisStoreService as IORedisStore } from '../store/clients/ioredis';
-import { RedisStoreService as RedisStore } from '../store/clients/redis';
-import { SubService } from '../sub';
-import { IORedisSubService as IORedisSub } from '../sub/clients/ioredis';
-import { RedisSubService as RedisSub } from '../sub/clients/redis';
-import { CacheMode } from '../../types/cache';
-import { HotMeshConfig, KeyType } from '../../types/hotmesh';
-import { RedisClientType as IORedisClientType } from '../../types/ioredisclient';
-import {
-  QuorumMessage,
-  QuorumMessageCallback,
-  QuorumProfile,
-  RollCallMessage,
-  SubscriptionCallback } from '../../types/quorum';
-import { RedisClient, RedisMulti } from '../../types/redis';
-import { RedisClientType } from '../../types/redisclient';
-
-class QuorumService {
-  namespace: string;
-  appId: string;
-  guid: string;
-  engine: EngineService;
-  profiles: QuorumProfile[] = [];
-  store: StoreService<RedisClient, RedisMulti> | null;
-  subscribe: SubService<RedisClient, RedisMulti> | null;
-  logger: ILogger;
-  cacheMode: CacheMode = 'cache';
-  untilVersion: string | null = null;
-  quorum: number | null = null;
-  callbacks: QuorumMessageCallback[] = [];
-  rollCallInterval: NodeJS.Timeout;
-
-  static async init(
-    namespace: string,
-    appId: string,
-    guid: string,
-    config: HotMeshConfig,
-    engine: EngineService,
-    logger: ILogger
-  ): Promise<QuorumService> {
-    if (config.engine) {
-      const instance = new QuorumService();
-      instance.verifyQuorumFields(config);
-      instance.namespace = namespace;
-      instance.appId = appId;
-      instance.guid = guid;
-      instance.logger = logger;
-      instance.engine = engine;
-
-      //note: `quorum` shares/re-uses the engine's `store`/`sub` Redis clients
-      await instance.initStoreChannel(config.engine.store);
-      await instance.initSubChannel(config.engine.sub);
-      //general quorum subscription
-      await instance.subscribe.subscribe(
-        KeyType.QUORUM,
-        instance.subscriptionHandler(),
-        appId
-      );
-      //app-specific quorum subscription (used for pubsub one-time request/response)
-      await instance.subscribe.subscribe(
-        KeyType.QUORUM,
-        instance.subscriptionHandler(),
-        appId, instance.guid
-      );
-      
-      instance.engine.processWebHooks();
-      instance.engine.processTimeHooks();
-      return instance;
-    }
-  }
-
-  verifyQuorumFields(config: HotMeshConfig) {
-    if (!identifyRedisType(config.engine.store) ||
-    !identifyRedisType(config.engine.sub)) {
-      throw new Error('quorum config must include `store` and `sub` fields.');
-    }
-  }
-
-  async initStoreChannel(store: RedisClient) {
-    if (identifyRedisType(store) === 'redis') {
-      this.store = new RedisStore(store as RedisClientType);
-    } else {
-      this.store = new IORedisStore(store as IORedisClientType);
-    }
-    await this.store.init(
-      this.namespace,
-      this.appId,
-      this.logger
-    );
-  }
-
-  async initSubChannel(sub: RedisClient) {
-    if (identifyRedisType(sub) === 'redis') {
-      this.subscribe = new RedisSub(sub as RedisClientType);
-    } else {
-      this.subscribe = new IORedisSub(sub as IORedisClientType);
-    }
-    await this.subscribe.init(
-      this.namespace,
-      this.appId,
-      this.guid,
-      this.logger
-    );
-  }
-
-  subscriptionHandler(): SubscriptionCallback {
-    const self = this;
-    return async (topic: string, message: QuorumMessage) => {
-      self.logger.debug('quorum-event-received', { topic, type: message.type});
-      if (message.type === 'activate') {
-        self.engine.setCacheMode(message.cache_mode, message.until_version);
-      } else if (message.type === 'ping') {
-        self.sayPong(self.appId, self.guid, message.originator, message.details);
-      } else if (message.type === 'pong' && self.guid === message.originator) {
-        self.quorum = self.quorum + 1;
-        if (message.profile) {
-          self.profiles.push(message.profile);
-        }
-      } else if (message.type === 'throttle') {
-        self.engine.throttle(message.throttle);
-      } else if (message.type === 'work') {
-        self.engine.processWebHooks()
-      } else if (message.type === 'job') {
-        self.engine.routeToSubscribers(message.topic, message.job)
-      } else if (message.type === 'cron') {
-        self.engine.processTimeHooks();
-      } else if (message.type === 'rollcall') {
-        self.doRollCall(message);
-      }
-      //if there are any callbacks, call them
-      if (self.callbacks.length > 0) {
-        self.callbacks.forEach(cb => cb(topic, message));
-      }
-    };
-  }
-
-  async sayPong(appId: string, guid: string, originator: string, details = false) {
-    let profile: QuorumProfile;
-    if (details) {
-      const stream = this.engine.stream.mintKey(
-        KeyType.STREAMS,
-        { appId: this.appId }
-      );
-      profile = {
-        engine_id: this.guid,
-        namespace: this.namespace,
-        app_id: this.appId,
-        stream,
-        counts: this.engine.router.counts,
-        timestamp: formatISODate(new Date()),
-        inited: this.engine.inited,
-        throttle: this.engine.router.throttle,
-        reclaimDelay: this.engine.router.reclaimDelay,
-        reclaimCount: this.engine.router.reclaimCount,
-        system: await getSystemHealth(),
-      };
-    }
-    this.store.publish(
-      KeyType.QUORUM, 
-      {
-        type: 'pong',
-        guid, originator,
-        profile,
-      },
-      appId,
-    );
-  }
-
-  async requestQuorum(delay = HMSH_QUORUM_DELAY_MS, details = false): Promise<number> {
-    const quorum = this.quorum;
-    this.quorum = 0;
-    this.profiles.length = 0;
-    await this.store.publish(
-      KeyType.QUORUM,
-      { 
-        type: 'ping',
-        originator: this.guid,
-        details,
-      },
-      this.appId,
-    );
-    await sleepFor(delay);
-    return quorum;
-  }
-
-  /**
-   * A quorum-wide command to broadcaset system details.
-   * 
-   */
-  async doRollCall(message: RollCallMessage) {
-    let iteration = 0;
-    let max = !isNaN(message.max) ? message.max : HMSH_QUORUM_ROLLCALL_CYCLES;
-    if (this.rollCallInterval) clearTimeout(this.rollCallInterval);
-    const base = (message.interval / 2);
-    const amount = base + Math.ceil(Math.random() * base);
-    do {
-      await sleepFor(Math.ceil(Math.random() * 1000));
-      await this.sayPong(this.appId, this.guid, null, true);
-      if (!message.interval) return;
-      const { promise, timerId } = XSleepFor(amount * 1000);
-      this.rollCallInterval = timerId;
-      await promise;
-    } while (this.rollCallInterval && iteration++ < max - 1);
-  }
-
-  cancelRollCall() {
-    if (this.rollCallInterval) {
-      clearTimeout(this.rollCallInterval);
-      delete this.rollCallInterval;
-    }
-  }
-
-  stop() {
-    this.cancelRollCall();
-  }
-
-  // ************* PUB/SUB METHODS *************
-  //publish a message to the quorum
-  async pub(quorumMessage: QuorumMessage) {
-    return await this.store.publish(KeyType.QUORUM, quorumMessage, this.appId, quorumMessage.topic || quorumMessage.guid);
-  }
-  //subscribe user to quorum messages
-  async sub(callback: QuorumMessageCallback): Promise<void> {
-    //the quorum is always subscribed to the `quorum` topic; just register the fn
-    this.callbacks.push(callback);
-  }
-  //unsubscribe user from quorum messages
-  async unsub(callback: QuorumMessageCallback): Promise<void> {
-    //the quorum is always subscribed to the `quorum` topic; just unregister the fn
-    this.callbacks = this.callbacks.filter(cb => cb !== callback);
-  }
-
-
-  // ************* COMPILER METHODS *************
-  async rollCall(delay = HMSH_QUORUM_DELAY_MS): Promise<QuorumProfile[]> {
-    await this.requestQuorum(delay, true);
-    const targetStreams = [];
-    const multi = this.store.getMulti();
-    this.profiles.forEach((profile: QuorumProfile) => {
-      if (!targetStreams.includes(profile.stream)) {
-        targetStreams.push(profile.stream);
-        this.store.xlen(profile.stream, multi);
-      }
-    });
-    const stream_depths = await multi.exec() as number[];
-    this.profiles.forEach(async (profile: QuorumProfile) => {
-      const index = targetStreams.indexOf(profile.stream);
-      if (index != -1) {
-        profile.stream_depth = Array.isArray(stream_depths[index]) ?
-          stream_depths[index][1] :
-          stream_depths[index];
-      }
-    });
-    return this.profiles;
-  }
-  /**
-   * request a quorum; if successful activate the app version
-   */
-  async activate(version: string, delay = HMSH_QUORUM_DELAY_MS, count = 0): Promise<boolean> {
-    version = version.toString();
-    const canActivate = await this.store.reserveScoutRole('activate', Math.ceil(delay * 6 / 1000) + 1);
-    if (!canActivate) {
-      //another engine is already activating the app version
-      this.logger.debug('quorum-activation-awaiting', { version });
-      await sleepFor(delay * 6);
-      const app = await this.store.getApp(this.appId, true);
-      return app?.active == true && app?.version === version;
-    }
-    const config = await this.engine.getVID();
-    await this.requestQuorum(delay);
-    const q1 = await this.requestQuorum(delay);
-    const q2 = await this.requestQuorum(delay);
-    const q3 = await this.requestQuorum(delay);
-    if (q1 && q1 === q2 && q2 === q3) {
-      this.logger.info('quorum-rollcall-succeeded', { q1, q2, q3 });
-      this.store.publish(
-        KeyType.QUORUM,
-        { type: 'activate', cache_mode: 'nocache', until_version: version },
-        this.appId
-      );
-      await new Promise(resolve => setTimeout(resolve, delay));
-      await this.store.releaseScoutRole('activate');
-      //confirm we received the activation message
-      if (this.engine.untilVersion === version) {
-        this.logger.info('quorum-activation-succeeded', { version });
-        const { id } = config;
-        const compiler = new CompilerService(this.store, this.logger);
-        return await compiler.activate(id, version);
-      } else {
-        this.logger.error('quorum-activation-error', { version });
-        throw new Error(`UntilVersion Not Received. Version ${version} not activated`);
-      }
-    } else {
-      this.logger.warn('quorum-rollcall-error', { q1, q2, q3, count });
-      this.store.releaseScoutRole('activate');
-      if (count < HMSH_ACTIVATION_MAX_RETRY) {
-        //increase the delay (give the quorum time to respond) and try again
-        return await this.activate(version, delay * 2, count + 1);
-      }
-      throw new Error(`Quorum not reached. Version ${version} not activated.`);
-    }
-  }
-}
-
-export { QuorumService }