@hotmeshio/hotmesh 0.0.1

package/services/hotmesh/index.ts

@@ -0,0 +1,182 @@
+import { nanoid } from 'nanoid';
+import { PSNS } from '../../modules/key';
+import { EngineService } from '../engine';
+import { LoggerService, ILogger } from '../logger';
+import { StreamSignaler } from '../signaler/stream';
+import { QuorumService } from '../quorum';
+import { WorkerService } from '../worker';
+import {
+  JobState,
+  JobData,
+  JobOutput, 
+  JobStatus } from '../../types/job';
+import {
+  HotMeshConfig,
+  HotMeshManifest } from '../../types/hotmesh';
+import { JobMessageCallback } from '../../types/quorum';
+import {
+  JobStatsInput,
+  GetStatsOptions,
+  IdsResponse,
+  StatsResponse } from '../../types/stats';
+import { ConnectorService } from '../connector';
+import { StreamData, StreamDataResponse } from '../../types/stream';
+
+class HotMeshService {
+  namespace: string;
+  appId: string;
+  guid: string;
+  engine: EngineService | null = null;
+  quorum: QuorumService | null = null;
+  workers: WorkerService[] = [];
+  logger: ILogger;
+
+  verifyAndSetNamespace(namespace?: string) {
+    if (!namespace) {
+      this.namespace = PSNS;
+    } else if (!namespace.match(/^[A-Za-z0-9-]+$/)) {
+      throw new Error(`config.namespace [${namespace}] is invalid`);
+    } else {
+      this.namespace = namespace;
+    }
+  }
+
+  verifyAndSetAppId(appId: string) {
+    if (!appId?.match(/^[A-Za-z0-9-]+$/)) {
+      throw new Error(`config.appId [${appId}] is invalid`);
+    } else if (appId === 'a') {
+      throw new Error(`config.appId [${appId}] is reserved`);
+    } else {
+      this.appId = appId;
+    }
+  }
+
+  static async init(config: HotMeshConfig) {
+    const instance = new HotMeshService();
+    instance.guid = nanoid();
+    instance.verifyAndSetNamespace(config.namespace);
+    instance.verifyAndSetAppId(config.appId);
+    instance.logger = new LoggerService(config.appId, instance.guid, config.name || '', config.logLevel);
+    await instance.initEngine(config, instance.logger);
+    await instance.initQuorum(config, instance.engine, instance.logger);
+    await instance.initWorkers(config, instance.logger);
+    return instance;
+  }
+
+  async initEngine(config: HotMeshConfig, logger: ILogger): Promise<void> {
+    if (config.engine) {
+      await ConnectorService.initRedisClients(
+        config.engine.redis?.class,
+        config.engine.redis?.options,
+        config.engine,
+      );
+      this.engine = await EngineService.init(
+        this.namespace,
+        this.appId,
+        this.guid,
+        config,
+        logger,
+      );
+    }
+  }
+
+  async initQuorum(config: HotMeshConfig, engine: EngineService, logger: ILogger): Promise<void> {
+    if (engine) {
+      this.quorum = await QuorumService.init(
+        this.namespace,
+        this.appId,
+        this.guid,
+        config,
+        engine,
+        logger
+      );
+    }
+  }
+
+  async initWorkers(config: HotMeshConfig, logger: ILogger) {
+    this.workers = await WorkerService.init(
+      this.namespace,
+      this.appId,
+      this.guid,
+      config,
+      logger
+    );
+  }
+
+  // ************* PUB/SUB METHODS *************
+  async pub(topic: string, data: JobData = {}, context?: JobState) {
+    return await this.engine?.pub(topic, data, context);
+  }
+  async sub(topic: string, callback: JobMessageCallback): Promise<void> {
+    return await this.engine?.sub(topic, callback);
+  }
+  async unsub(topic: string): Promise<void> {
+    return await this.engine?.unsub(topic);
+  }
+  async psub(wild: string, callback: JobMessageCallback): Promise<void> {
+    return await this.engine?.psub(wild, callback);
+  }
+  async punsub(wild: string): Promise<void> {
+    return await this.engine?.punsub(wild);
+  }
+  async pubsub(topic: string, data: JobData = {}, context?: JobState | null, timeout?: number): Promise<JobOutput> {
+    return await this.engine?.pubsub(topic, data, context, timeout);
+  }
+  async add(streamData: StreamData|StreamDataResponse): Promise<string> {
+    return await this.engine.add(streamData) as string;
+  }
+
+  // ************* COMPILER METHODS *************
+  async plan(path: string): Promise<HotMeshManifest> {
+    return await this.engine?.plan(path);
+  }
+  async deploy(pathOrYAML: string): Promise<HotMeshManifest> {
+    return await this.engine?.deploy(pathOrYAML);
+  }
+  async activate(version: string, delay?: number): Promise<boolean> {
+    //activation is a quorum operation
+    return await this.quorum?.activate(version, delay);
+  }
+
+  // ************* REPORTER METHODS *************
+  async getStats(topic: string, query: JobStatsInput): Promise<StatsResponse> {
+    return await this.engine?.getStats(topic, query);
+  }
+  async getStatus(jobId: string): Promise<JobStatus> {
+    return this.engine?.getStatus(jobId);
+  }
+  async getState(topic: string, jobId: string) {
+    return this.engine?.getState(topic, jobId);
+  }
+  async getIds(topic: string, query: JobStatsInput, queryFacets = []): Promise<IdsResponse> {
+    return await this.engine?.getIds(topic, query, queryFacets);
+  }
+  async resolveQuery(topic: string, query: JobStatsInput): Promise<GetStatsOptions> {
+    return await this.engine?.resolveQuery(topic, query);
+  }
+
+  // ****************** `SCRUB` CLEAN COMPLETED JOBS *****************
+  async scrub(jobId: string) {
+    await this.engine?.scrub(jobId);
+  }
+
+  // ****** `HOOK` ACTIVITY RE-ENTRY POINT ******
+  async hook(topic: string, data: JobData): Promise<JobStatus | void> {
+    //return collation int
+    return await this.engine?.hook(topic, data);
+  }
+  async hookAll(hookTopic: string, data: JobData, query: JobStatsInput, queryFacets: string[] = []): Promise<string[]> {
+    return await this.engine?.hookAll(hookTopic, data, query, queryFacets);
+  }
+
+  async stop() {
+    await StreamSignaler.stopConsuming();
+    this.engine?.task.cancelCleanup();
+  }
+
+  async compress(terms: string[]): Promise<boolean> {
+    return await this.engine?.compress(terms);
+  }
+}
+
+export { HotMeshService };

package/services/logger/index.ts

@@ -0,0 +1,79 @@
+import { Logger, createLogger, transports, format } from 'winston';
+import { ILogger } from '../../types/logger';
+
+class LoggerService implements ILogger {
+  private logger: Logger;
+
+  constructor(private appId: string = 'appId', private instanceId: string = 'instanceId', private name: string = 'name', private logLevel: string = 'info', customLogger?: Logger) {
+    this.logger = customLogger || this.createDefaultLogger();
+  }
+
+  private createDefaultLogger(): Logger {
+    return createLogger({
+      level: this.logLevel,
+      format: format.combine(
+        format.colorize(),
+        format.timestamp(),
+        format.printf((info) => {
+          const { timestamp, level, message } = info;
+          // Extract the object from the `info` object's `Symbol(splat)` field
+          const symbols = Object.getOwnPropertySymbols(info);
+          const splatSymbol = symbols.find(symbol => symbol.toString() === 'Symbol(splat)');
+          let splatData = {};
+          if (splatSymbol) {
+            splatData = info[splatSymbol][0] || {};
+          }
+          // Pass it to the `tagify` method
+          const tags = this.tagify(splatData);
+  
+          return `${timestamp} [${level}] [${this.name || this.appId}:${this.instanceId}] ${message} ${tags}`;
+        }),
+      ),
+      transports: [new transports.Console()],
+    });
+  }
+
+  info(message: string, ...meta: any[]): void {
+    this.logger.info(message, ...meta);
+  }
+
+  error(message: string, ...meta: any[]): void {
+    this.logger.error(message, ...meta);
+  }
+
+  warn(message: string, ...meta: any[]): void {
+    this.logger.warn(message, ...meta);
+  }
+
+  debug(message: string, ...meta: any[]): void {
+    this.logger.debug(message, ...meta);
+  }
+
+  tagify(obj: Record<string, unknown>): string {
+    if (!obj) {
+      return '';
+    }
+    const tags: string[] = [];
+    try {
+      Object.entries(obj).forEach(([key, val]) => {
+        let value: any = val;
+        if (typeof val === 'function') {
+          val = val();
+        }
+        if (val instanceof Date) {
+          value = val.toISOString();
+        } else if (typeof val === 'object' && val !== null) {
+          value = JSON.stringify(val);
+        } else {
+          value = value ? value.toString() : value;
+        }
+        tags.push(`${key}:${value}`);
+      });
+    } catch (err) {
+      this.error('tagify-error', err);
+    }
+    return tags.join(' ');
+  }
+}
+
+export { LoggerService, ILogger };

package/services/mapper/index.ts

@@ -0,0 +1,84 @@
+import { matchesStatus, matchesStatusCode } from "../../modules/utils";
+import { Pipe } from "../pipe";
+import { JobState } from "../../types/job";
+import { Pipe as PipeType } from "../../types/pipe";
+import { TransitionMatch, TransitionRule } from "../../types/transition";
+import { StreamCode, StreamStatus } from "../../types";
+
+type RuleType = null | undefined | boolean | string | number | Date | Record<string, any>;
+
+class MapperService {
+  private rules: Record<string, unknown>;
+  private data: JobState;
+
+  constructor(rules: Record<string, unknown>, data: JobState) {
+    this.rules = rules;
+    this.data = data;
+  }
+
+  public mapRules(): Record<string, unknown> {
+    return this.traverseRules(this.rules);
+  }
+
+  private traverseRules(rules: RuleType): Record<string, unknown> {
+    if (typeof rules === 'object' && '@pipe' in rules) {
+      return this.pipe(rules['@pipe'] as PipeType);
+    } if (typeof rules === 'object' && rules !== null) {
+      const mappedRules: Record<string, any> = {};
+      for (const key in rules) {
+        if (Object.prototype.hasOwnProperty.call(rules, key)) {
+          mappedRules[key] = this.traverseRules(rules[key]);
+        }
+      }
+      return mappedRules;
+    } else {
+      return this.resolve(rules);
+    }
+  }
+
+  /**
+   * resolve a pipe expression of the form: { @pipe: [["{data.foo.bar}", 2, false, "hello world"]] }
+   * @param value 
+   * @returns 
+   */
+  private pipe(value: PipeType): any {
+    const pipe = new Pipe(value, this.data);
+    return pipe.process();
+  }
+
+  /**
+   * resolve a simple mapping expression in the form: "{data.foo.bar}" or 2 or false or "hello world"
+   * @param value 
+   * @returns 
+   */
+  private resolve(value: any): any {
+    const pipe = new Pipe([[value]], this.data);
+    return pipe.process();
+  }
+
+  static evaluate(transitionRule: TransitionRule | boolean, context: JobState, code: StreamCode): boolean {
+    if (typeof transitionRule === 'boolean') {
+      return transitionRule;
+    }
+    if (code.toString() === (transitionRule.code || 200).toString()) {
+      const orGate = transitionRule.gate === 'or';
+      let allAreTrue = true;
+      let someAreTrue = false;
+      transitionRule.match.forEach(({ expected, actual }: TransitionMatch) => {
+        if ((orGate && !someAreTrue) || (!orGate && allAreTrue)) {
+          const result = Pipe.resolve(actual, context) === expected;
+          if (orGate && result) {
+            someAreTrue = true;
+          } else if(!orGate && !result) {
+            allAreTrue = false;
+          }
+        }
+      });
+      return orGate ? someAreTrue : allAreTrue;
+    }
+    return false;
+  }
+}
+
+
+export { MapperService }

package/services/pipe/functions/array.ts

@@ -0,0 +1,87 @@
+class ArrayHandler {
+  get(array: any[], index: number): any {
+    return array[index];
+  }
+
+  concat(array1: any[], array2: any[]): any[] {
+    return array1.concat(array2);
+  }
+
+  every(array: any[], callback: (value: any, index: number, array: any[]) => boolean): boolean {
+    return array.every(callback);
+  }
+
+  filter(array: any[], callback: (value: any, index: number, array: any[]) => boolean): any[] {
+    return array.filter(callback);
+  }
+
+  find(array: any[], callback: (value: any, index: number, array: any[]) => boolean): any {
+    return array.find(callback);
+  }
+
+  findIndex(array: any[], callback: (value: any, index: number, array: any[]) => boolean): number {
+    return array.findIndex(callback);
+  }
+
+  forEach(array: any[], callback: (value: any, index: number, array: any[]) => void): void {
+    array.forEach(callback);
+  }
+
+  indexOf(array: any[], searchElement: any, fromIndex?: number): number {
+    return array.indexOf(searchElement, fromIndex);
+  }
+
+  join(array: any[], separator: string): string {
+    return array.join(separator);
+  }
+
+  lastIndexOf(array: any[], searchElement: any, fromIndex?: number): number {
+    return array.lastIndexOf(searchElement, fromIndex);
+  }
+
+  map(array: any[], callback: (value: any, index: number, array: any[]) => any): any[] {
+    return array.map(callback);
+  }
+
+  pop(array: any[]): any {
+    return array.pop();
+  }
+
+  push(array: any[], ...items: any[]): number {
+    return array.push(...items);
+  }
+
+  reduce(array: any[], callback: (accumulator: any, currentValue: any, currentIndex: number, array: any[]) => any, initialValue?: any): any {
+    return array.reduce(callback, initialValue);
+  }
+
+  reverse(array: any[]): any[] {
+    return array.reverse();
+  }
+
+  shift(array: any[]): any {
+    return array.shift();
+  }
+
+  slice(array: any[], start?: number, end?: number): any[] {
+    return array.slice(start, end);
+  }
+
+  some(array: any[], callback: (value: any, index: number, array: any[]) => boolean): boolean {
+    return array.some(callback);
+  }
+
+  sort(array: any[], compareFunction?: (a: any, b: any) => number): any[] {
+    return array.sort(compareFunction);
+  }
+
+  splice(array: any[], start: number, deleteCount?: number, ...items: any[]): any[] {
+    return array.splice(start, deleteCount, ...items);
+  }
+
+  unshift(array: any[], ...items: any[]): number {
+    return array.unshift(...items);
+  }
+}
+
+export { ArrayHandler };

package/services/pipe/functions/bitwise.ts

@@ -0,0 +1,27 @@
+class BitwiseHandler {
+  and(a: number, b: number): number {
+    return a & b;
+  }
+
+  or(a: number, b: number): number {
+    return a | b;
+  }
+
+  xor(a: number, b: number): number {
+    return a ^ b;
+  }
+
+  leftShift(a: number, b: number): number {
+    return a << b;
+  }
+
+  rightShift(a: number, b: number): number {
+    return a >> b;
+  }
+
+  unsignedRightShift(a: number, b: number): number {
+    return a >>> b;
+  }
+}
+
+export { BitwiseHandler };

package/services/pipe/functions/conditional.ts

@@ -0,0 +1,31 @@
+class ConditionalHandler {
+  ternary(condition: boolean, valueIfTrue: any, valueIfFalse: any): any {
+    return condition ? valueIfTrue : valueIfFalse;
+  }
+
+  equality(value1: any, value2: any): boolean {
+    return value1 == value2;
+  }
+
+  strict_equality(value1: any, value2: any): boolean {
+    return value1 === value2;
+  }
+
+  greater_than(value1: number, value2: number): boolean {
+    return value1 > value2;
+  }
+
+  less_than(value1: number, value2: number): boolean {
+    return value1 < value2;
+  }
+
+  greater_than_or_equal(value1: number, value2: number): boolean {
+    return value1 >= value2;
+  }
+
+  less_than_or_equal(value1: number, value2: number): boolean {
+    return value1 <= value2;
+  }
+}
+
+export { ConditionalHandler };

package/services/pipe/functions/date.ts

@@ -0,0 +1,214 @@
+type DateInput = Date | string | number;
+
+class DateHandler {
+
+  /**
+   * It is so common in mapping operations to use a string (ISO) date as input. This helper
+   * method allows for a more-concise mapping ruleset by avoiding date initialization boilerplate
+   * code and instead handles the ISO, Milliseconds, and ECMAScript Date input types.
+   * @param input 
+   * @returns 
+   */
+  static getDateInstance(input: DateInput): Date {
+    const ISO_REGEX = /^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z)?$/;
+
+    if (typeof input === 'string') {
+      if (ISO_REGEX.test(input)) {
+        return new Date(input);
+      }
+
+      const milliseconds = parseInt(input, 10);
+      if (!isNaN(milliseconds)) {
+        return new Date(milliseconds);
+      }
+    } else if (input instanceof Date) {
+      return input;
+    } else if (typeof input === 'number') {
+      return new Date(input);
+    }
+
+    throw new Error("Invalid date format");
+  }
+
+  fromISOString(isoString: string): Date {
+    return new Date(isoString);
+  }
+
+  now(): number {
+    return Date.now();
+  }
+
+  parse(dateString: string): number {
+    return Date.parse(dateString);
+  }
+
+  getDate(date: DateInput): number {
+    return DateHandler.getDateInstance(date).getDate();
+  }
+
+  getDay(date: DateInput): number {
+    return DateHandler.getDateInstance(date).getDay();
+  }
+
+  getFullYear(date: DateInput): number {
+    return DateHandler.getDateInstance(date).getFullYear();
+  }
+
+  getHours(date: DateInput): number {
+    return DateHandler.getDateInstance(date).getHours();
+  }
+
+  getMilliseconds(date: DateInput): number {
+    return DateHandler.getDateInstance(date).getMilliseconds();
+  }
+
+  getMinutes(date: DateInput): number {
+    return DateHandler.getDateInstance(date).getMinutes();
+  }
+
+  getMonth(date: DateInput): number {
+    return DateHandler.getDateInstance(date).getMonth();
+  }
+
+  getSeconds(date: DateInput): number {
+    return DateHandler.getDateInstance(date).getSeconds();
+  }
+
+  getTime(date: DateInput): number {
+    return DateHandler.getDateInstance(date).getTime();
+  }
+
+  getTimezoneOffset(date: DateInput): number {
+    return DateHandler.getDateInstance(date).getTimezoneOffset();
+  }
+
+  getUTCDate(date: DateInput): number {
+    return DateHandler.getDateInstance(date).getUTCDate();
+  }
+
+  getUTCDay(date: DateInput): number {
+    return DateHandler.getDateInstance(date).getUTCDay();
+  }
+
+  getUTCFullYear(date: DateInput): number {
+    return DateHandler.getDateInstance(date).getUTCFullYear();
+  }
+
+  getUTCHours(date: DateInput): number {
+    return DateHandler.getDateInstance(date).getUTCHours();
+  }
+
+  getUTCMilliseconds(date: DateInput): number {
+    return DateHandler.getDateInstance(date).getUTCMilliseconds();
+  }
+
+  getUTCMinutes(date: DateInput): number {
+    return DateHandler.getDateInstance(date).getUTCMinutes();
+  }
+
+  getUTCMonth(date: DateInput): number {
+    return DateHandler.getDateInstance(date).getUTCMonth();
+  }
+
+  getUTCSeconds(date: DateInput): number {
+    return DateHandler.getDateInstance(date).getUTCSeconds();
+  }
+
+  setMilliseconds(date: DateInput, ms: number): number {
+    return DateHandler.getDateInstance(date).setMilliseconds(ms);
+  }
+
+  setMinutes(date: DateInput, minutes: number, seconds?: number, ms?: number): number {
+    return DateHandler.getDateInstance(date).setMinutes(minutes, seconds, ms);
+  }
+
+  setMonth(date: DateInput, month: number, day?: number): number {
+    return DateHandler.getDateInstance(date).setMonth(month, day);
+  }
+
+  setSeconds(date: DateInput, seconds: number, ms?: number): number {
+    return DateHandler.getDateInstance(date).setSeconds(seconds, ms);
+  }
+
+  setTime(date: DateInput, time: number): number {
+    return DateHandler.getDateInstance(date).setTime(time);
+  }
+
+  setUTCDate(date: DateInput, day: number): number {
+    return DateHandler.getDateInstance(date).setUTCDate(day);
+  }
+
+  setUTCFullYear(date: DateInput, year: number, month?: number, day?: number): number {
+    return DateHandler.getDateInstance(date).setUTCFullYear(year, month, day);
+  }
+
+  setUTCHours(date: DateInput, hours: number, minutes?: number, seconds?: number, ms?: number): number {
+    return DateHandler.getDateInstance(date).setUTCHours(hours, minutes, seconds, ms);
+  }
+
+  setUTCMilliseconds(date: DateInput, ms: number): number {
+    return DateHandler.getDateInstance(date).setUTCMilliseconds(ms);
+  }
+
+  setUTCMinutes(date: DateInput, minutes: number, seconds?: number, ms?: number): number {
+    return DateHandler.getDateInstance(date).setUTCMinutes(minutes, seconds, ms);
+  }
+
+  setUTCMonth(date: DateInput, month: number, day?: number): number {
+    return DateHandler.getDateInstance(date).setUTCMonth(month, day);
+  }
+
+  setUTCSeconds(date: DateInput, seconds: number, ms?: number): number {
+    return DateHandler.getDateInstance(date).setUTCSeconds(seconds, ms);
+  }
+  
+  setDate(date: DateInput, day: number): number {
+    return DateHandler.getDateInstance(date).setDate(day);
+  }
+
+  setFullYear(date: DateInput, year: number, month?: number, day?: number): number {
+    return DateHandler.getDateInstance(date).setFullYear(year, month, day);
+  }
+
+  setHours(date: DateInput, hours: number, minutes?: number, seconds?: number, ms?: number): number {
+    return DateHandler.getDateInstance(date).setHours(hours, minutes, seconds, ms);
+  }
+  toDateString(date: DateInput): string {
+    return DateHandler.getDateInstance(date).toDateString();
+  }
+
+  toISOString(date: DateInput): string {
+    return DateHandler.getDateInstance(date).toISOString();
+  }
+
+  toJSON(date: DateInput): string {
+    return DateHandler.getDateInstance(date).toJSON();
+  }
+
+  toLocaleDateString(date: DateInput, locales?: string | string[], options?: Intl.DateTimeFormatOptions): string {
+    return DateHandler.getDateInstance(date).toLocaleDateString(locales, options);
+  }
+
+  toLocaleString(date: DateInput, locales?: string | string[], options?: Intl.DateTimeFormatOptions): string {
+    return DateHandler.getDateInstance(date).toLocaleString(locales, options);
+  }
+
+  toLocaleTimeString(date: DateInput, locales?: string | string[], options?: Intl.DateTimeFormatOptions): string {
+    return DateHandler.getDateInstance(date).toLocaleTimeString(locales, options);
+  }
+
+  toString(date: DateInput): string {
+    return DateHandler.getDateInstance(date).toString();
+  }
+
+  UTC(year: number, month: number, date?: number, hours?: number, minutes?: number, seconds?: number, ms?: number): number {
+    return Date.UTC(year, month, date, hours, minutes, seconds, ms);
+  }
+
+  valueOf(date: DateInput): number {
+    return DateHandler.getDateInstance(date).valueOf();
+  }
+
+}
+
+export { DateHandler };