@hotmeshio/hotmesh 0.0.55 → 0.0.57

package/services/compiler/validator.ts

@@ -1,158 +0,0 @@
-import { Pipe } from "../pipe";
-import { StoreService } from '../store';
-import { MappingStatements } from "../../types/map";
-import { HotMeshManifest } from "../../types/hotmesh";
-import { RedisClient, RedisMulti } from "../../types/redis";
-
-class Validator {
-  manifest: HotMeshManifest | null = null;
-  activityIds: string[] = [];
-  mappingStatements: MappingStatements = {};
-  store: StoreService<RedisClient, RedisMulti> | null = null;
-
-  static SYS_VARS = ['$app', '$self', '$graph', '$job'];
-  static CONTEXT_VARS = ['{$input}', '{$output}', '{$item}', '{$key}', '{$index}'];
-  
-  constructor(manifest: HotMeshManifest) {
-    this.manifest = manifest;
-  }
-
-  /**
-   * validate the manifest file
-   */
-  async validate(store: StoreService<RedisClient, RedisMulti>) {
-    this.store = store;
-    this.getMappingStatements();
-    this.validateActivityIds();
-    this.validateReferencedActivityIds();
-    this.validateMappingStatements();
-    this.validateTransitions();
-    this.validateTransitionConditions();
-    this.validateStats();
-    this.validateSchemas();
-    this.validateUniqueHandledTopics();
-    this.validateGraphPublishSubscribe();
-    this.validateHooks();
-    this.validateConditionalStatements();
-  }
-
-  // 1.1) Validate the manifest file activity ids are unique (no duplicates)
-  validateActivityIds() {
-    const activityIdsSet: Set<string> = new Set();
-    this.manifest.app.graphs.forEach((graph) => {
-      const ids = Object.keys(graph.activities);
-      // Check for duplicates and add ids to the set
-      ids.forEach((id) => {
-        if (activityIdsSet.has(id)) {
-          throw new Error(`Duplicate activity id found: ${id}`);
-        } else {
-          activityIdsSet.add(id);
-        }
-      });
-    });
-    this.activityIds = Array.from(activityIdsSet);
-  }
-
-  isMappingStatement(value: string): boolean {
-    return typeof value === 'string' && value.startsWith('{') && value.endsWith('}');
-  }
-
-  extractMappingStatements(obj: any, result: MappingStatements, currentActivityId: string): void {
-    for (const key in obj) {
-      if (typeof obj[key] === 'object' && obj[key] !== null) {
-        this.extractMappingStatements(obj[key], result, currentActivityId);
-      } else if (this.isMappingStatement(obj[key])) {
-        if (!result[currentActivityId]) {
-          result[currentActivityId] = [];
-        }
-        result[currentActivityId].push(obj[key]);
-      }
-    }
-  }
-  
-  getMappingStatements() {
-    const mappingStatements: MappingStatements = {};
-    this.manifest.app.graphs.forEach((graph) => {
-      const activities = graph.activities;
-      for (const activityId in activities) {
-        const activity = activities[activityId];
-        this.extractMappingStatements(activity, mappingStatements, activityId);
-      }
-    });
-    this.mappingStatements = mappingStatements;
-  }
-
-  // 1.2) Validate no activity ids are referenced that don't exist
-  validateReferencedActivityIds() {
-    // get list of all mapping statements and validate
-    const mappingStatements = this.mappingStatements;
-    const activityIds = this.activityIds;
-    for (const activity in mappingStatements) {
-      const statements = mappingStatements[activity];
-      statements.forEach((statement) => {
-        if (statement.startsWith('{') && statement.endsWith('}')) {
-          const statementParts = statement.slice(1, -1).split('.');
-          const referencedActivityId = statementParts[0];
-          if (!(Validator.SYS_VARS.includes(referencedActivityId) || activityIds.includes(referencedActivityId) || this.isFunction(statement) || this.isContextVariable(statement))) {
-            throw new Error(`Mapping statement references non-existent activity: ${statement}`);
-          }
-        }
-      });
-    }
-  }
-
-  isFunction(value: string): boolean {
-    return value.startsWith('{@') && Pipe.resolveFunction(value);
-  }
-
-  isContextVariable(value: string): boolean {
-    return ['{$input}', '{$output}', '{$item}', '{$key}', '{$index}'].includes(value);
-  }
-
-  // 1.3) Validate the mapping/@pipe statements are valid
-  validateMappingStatements() {
-    // Implement the method content
-  }
-
-  // 1.4) Validate the transitions are valid
-  validateTransitions() {
-    // Implement the method content
-  }
-
-  // 1.5) Validate the transition conditions are valid
-  validateTransitionConditions() {
-    // Implement the method content
-  }
-
-  // 1.6) Validate the stats
-  validateStats() {
-    // Implement the method content
-  }
-
-  // 1.7) Validate the schemas
-  validateSchemas() {
-    // Implement the method content
-  }
-
-  // 1.8) Validate the topics are unique and handled
-  validateUniqueHandledTopics() {
-    // Implement the method content
-  }
-
-  // 1.9) Validate that every graph has publishes and subscribes
-  validateGraphPublishSubscribe() {
-    // Implement the method content
-  }
-
-  // 1.10) Validate hooks, including mapping statements
-  validateHooks() {
-    // Implement the method content
-  }
-
-  // 1.11) Validate conditional statements
-  validateConditionalStatements() {
-    // Implement the method content
-  }
-}
-
-export { Validator };

package/services/connector/clients/ioredis.ts

@@ -1,57 +0,0 @@
-import {
-  RedisClientOptions,
-  RedisClassType,
-  RedisClientType } from '../../../types/ioredisclient';
-
-class RedisConnection {
-  private connection: any | null = null;
-  private static instances: Map<string, RedisConnection> = new Map();
-  private id: string | null = null;
-
-  private static clientOptions: RedisClientOptions = {
-    host: 'localhost',
-    port: 6379,
-    //password: config.REDIS_PASSWORD,
-    //db: config.REDIS_DATABASE,
-  };
-
-  private async createConnection(Redis: RedisClassType, options: RedisClientOptions): Promise<any> {
-    return new Redis(options);
-  }
-
-  public getClient(): RedisClientType {
-    if (!this.connection) {
-      throw new Error('Redis client is not connected');
-    }
-    return this.connection;
-  }
-
-  public async disconnect(): Promise<void> {
-    if (this.connection) {
-      await this.connection.quit();
-      this.connection = null;
-    }
-    if (this.id) {
-      RedisConnection.instances.delete(this.id);
-    }
-  }
-
-  public static async connect(id: string, Redis: RedisClassType, options?: RedisClientOptions): Promise<RedisConnection> {
-    if (this.instances.has(id)) {
-      return this.instances.get(id) as RedisConnection;
-    }
-    const instance = new RedisConnection();
-    const opts = options ? { ...options } : { ...this.clientOptions };
-    instance.connection = await instance.createConnection(Redis, opts);
-    instance.id = id;
-    this.instances.set(id, instance);
-    return instance;
-  }
-
-  public static async disconnectAll(): Promise<void> {
-    await Promise.all(Array.from(this.instances.values()).map((instance) => instance.disconnect()));
-    this.instances.clear();
-  }
-}
-
-export { RedisConnection };

package/services/connector/clients/redis.ts

@@ -1,72 +0,0 @@
-import {
-  RedisClientType,
-  RedisClientOptions,
-  RedisClassType } from '../../../types/redisclient';
-
-class RedisConnection {
-  private connection: RedisClientType | null = null;
-  private static instances: Map<string, RedisConnection> = new Map();
-  private id: string | null = null;
-
-  private static clientOptions: RedisClientOptions = {
-    socket: {
-      host: 'localhost',
-      port: 6379,
-      tls: false,
-    },
-    //password: config.REDIS_PASSWORD,
-    //database: config.REDIS_DATABASE,
-  };
-
-  private async createConnection(Redis: RedisClassType, options: RedisClientOptions): Promise<RedisClientType> {
-    return new Promise((resolve, reject) => {
-      const client = Redis.createClient(options);
-
-      client.on('error', (error: any) => {
-        reject(error);
-      });
-
-      client.on('ready', () => {
-        resolve(client);
-      });
-
-      client.connect();
-    });
-  }
-
-  public getClient(): RedisClientType {
-    if (!this.connection) {
-      throw new Error('Redis client is not connected');
-    }
-    return this.connection;
-  }
-
-  public async disconnect(): Promise<void> {
-    if (this.connection) {
-      await this.connection.quit();
-      this.connection = null;
-    }
-    if (this.id) {
-      RedisConnection.instances.delete(this.id);
-    }
-  }
-
-  public static async connect(id: string, Redis: RedisClassType, options?: RedisClientOptions): Promise<RedisConnection> {
-    if (this.instances.has(id)) {
-      return this.instances.get(id)!;
-    }
-    const instance = new RedisConnection();
-    const opts = options ? { ...options } : { ...this.clientOptions };
-    instance.connection = await instance.createConnection(Redis, opts);
-    instance.id = id;
-    this.instances.set(id, instance);
-    return instance;
-  }
-
-  public static async disconnectAll(): Promise<void> {
-    await Promise.all(Array.from(this.instances.values()).map((instance) => instance.disconnect()));
-    this.instances.clear();
-  }
-}
-
-export { RedisConnection, RedisClientType };

package/services/connector/index.ts

@@ -1,42 +0,0 @@
-import { guid, identifyRedisTypeFromClass } from '../../modules/utils';
-import { RedisConnection as IORedisConnection } from '../connector/clients/ioredis';
-import { RedisConnection } from '../connector/clients/redis';
-import {
-  RedisClassType as IORedisClassType,
-  RedisClientOptions as IORedisClientOptions } from '../../types/ioredisclient';
-import {
-  HotMeshEngine,
-  HotMeshWorker } from '../../types/hotmesh';
-import { RedisClass, RedisOptions } from '../../types/redis';
-import {
-  RedisClassType,
-  RedisClientOptions } from '../../types/redisclient';
-
-export class ConnectorService {
-  //1) Initialize `store`, `stream`, and `subscription` Redis clients.
-  //2) Bind to the target if not already present
-  static async initRedisClients(Redis: RedisClass, options: RedisOptions, target: HotMeshEngine | HotMeshWorker): Promise<void> {
-    if (!target.store || !target.stream || !target.sub) {
-      const instances = [];
-      if (identifyRedisTypeFromClass(Redis) === 'redis') {
-        for (let i = 1; i <= 3; i++) {
-          instances.push(RedisConnection.connect(
-            guid(),
-            Redis as RedisClassType,
-            options as RedisClientOptions));
-        }
-      } else {
-        for (let i = 1; i <= 3; i++) {
-          instances.push(IORedisConnection.connect(
-            guid(),
-            Redis as IORedisClassType,
-            options as IORedisClientOptions));
-        }
-      }
-      const [store, stream, sub] = await Promise.all(instances);
-      target.store = target.store || store.getClient();
-      target.stream = target.stream || stream.getClient();
-      target.sub = target.sub || sub.getClient();
-    }
-  }
-}

package/services/durable/client.ts

@@ -1,266 +0,0 @@
-import ms from 'ms';
-
-import {
-  APP_ID,
-  APP_VERSION,
-  getWorkflowYAML } from './schemas/factory';
-import {
-  HMSH_LOGLEVEL,
-  HMSH_EXPIRE_JOB_SECONDS,
-  HMSH_QUORUM_DELAY_MS,
-  HMSH_DURABLE_EXP_BACKOFF,
-  HMSH_DURABLE_MAX_ATTEMPTS,
-  HMSH_DURABLE_MAX_INTERVAL } from '../../modules/enums';
-import { sleepFor } from '../../modules/utils';
-import { WorkflowHandleService } from './handle';
-import { HotMeshService as HotMesh } from '../hotmesh';
-import {
-  ClientConfig,
-  Connection,
-  HookOptions,
-  WorkflowOptions, 
-  WorkflowSearchOptions} from '../../types/durable';
-import { JobState } from '../../types/job';
-import { KeyService, KeyType } from '../../modules/key';
-import { Search } from './search';
-import { StreamStatus } from '../../types';
-
-export class ClientService {
-
-  connection: Connection;
-  options: WorkflowOptions;
-  static topics: string[] = [];
-  static instances = new Map<string, HotMesh | Promise<HotMesh>>();
-
-  constructor(config: ClientConfig) {
-    this.connection = config.connection;
-  }
-
-  getHotMeshClient = async (workflowTopic: string, namespace?: string) => {
-    const targetNS = namespace ?? APP_ID;
-    if (ClientService.instances.has(targetNS)) {
-      const hotMeshClient = await ClientService.instances.get(targetNS);
-      await this.verifyWorkflowActive(hotMeshClient, targetNS);
-      if (!ClientService.topics.includes(workflowTopic)) {
-        ClientService.topics.push(workflowTopic);
-        await ClientService.createStream(hotMeshClient, workflowTopic, namespace);
-      }
-      return hotMeshClient;
-    }
-
-    //create and cache an instance
-    const hotMeshClient = HotMesh.init({
-      appId: targetNS,
-      logLevel: HMSH_LOGLEVEL,
-      engine: {
-        redis: {
-          class: this.connection.class,
-          options: this.connection.options,
-        }
-      }
-    });
-    ClientService.instances.set(targetNS, hotMeshClient);
-    await ClientService.createStream(await hotMeshClient, workflowTopic, namespace);
-    await this.activateWorkflow(await hotMeshClient, targetNS);
-    return hotMeshClient;
-  }
-
-  /**
-   * Creates a stream (Redis `XGROUP.CREATE`) where events can be published (XADD).
-   * It is possible that the worker that will read from this stream channel
-   * has not yet been initialized, so this call ensures that the channel
-   * exists and is ready to serve as a container for events.
-   */
-  static createStream = async(hotMeshClient: HotMesh, workflowTopic: string, namespace?: string) => {
-    const store = hotMeshClient.engine.store;
-    const params = { appId: namespace ?? APP_ID, topic: workflowTopic };
-    const streamKey = store.mintKey(KeyType.STREAMS, params);
-    try {
-      await store.xgroup('CREATE', streamKey, 'WORKER', '$', 'MKSTREAM');
-    } catch (err) {
-      //ignore if already exists
-    }
-  }
-
-  /**
-   * It is possible for a client to invoke a workflow without first
-   * creating the stream. This method will verify that the stream
-   * exists and if not, create it.
-   */
-  static verifyStream = async(workflowTopic: string, namespace?: string) => {
-    const targetNS = namespace ?? APP_ID;
-    if (ClientService.instances.has(targetNS)) {
-      const hotMeshClient = await ClientService.instances.get(targetNS);
-      if (!ClientService.topics.includes(workflowTopic)) {
-        ClientService.topics.push(workflowTopic);
-        await ClientService.createStream(hotMeshClient, workflowTopic, namespace);
-      }
-      return hotMeshClient;
-    }
-  }
-
-  /**
-   * For those deployments with a redis stack backend (with the FT module),
-   * this method will configure the search index for the workflow.
-   */
-  configureSearchIndex = async (hotMeshClient: HotMesh, search?: WorkflowSearchOptions): Promise<void> => {
-    if (search?.schema) {
-      const store = hotMeshClient.engine.store;
-      const schema: string[] = [];
-      for (const [key, value] of Object.entries(search.schema)) {
-        //prefix with an underscore (avoids collisions with hotmesh reserved symbols)
-        schema.push(`_${key}`);
-        schema.push(value.type);
-        if (value.sortable) {
-          schema.push('SORTABLE');
-        }
-      }
-      try {
-        const keyParams = {
-          appId: hotMeshClient.appId,
-          jobId: ''
-        }
-        const hotMeshPrefix = KeyService.mintKey(hotMeshClient.namespace, KeyType.JOB_STATE, keyParams);
-        const prefixes = search.prefix.map((prefix) => `${hotMeshPrefix}${prefix}`);
-        await store.exec('FT.CREATE', `${search.index}`, 'ON', 'HASH', 'PREFIX', prefixes.length, ...prefixes, 'SCHEMA', ...schema);
-      } catch (error) {
-        hotMeshClient.engine.logger.info('durable-client-search-err', { ...error });
-      }
-    }
-  }
-
-  search = async (hotMeshClient: HotMesh, index: string, query: string[]): Promise<string[]> => {
-    const store = hotMeshClient.engine.store;
-    if (query[0]?.startsWith('FT.')) {
-      return await store.exec(...query) as string[];
-    }
-    return await store.exec('FT.SEARCH', index, ...query) as string[];
-  }
-
-  workflow = {
-    start: async (options: WorkflowOptions): Promise<WorkflowHandleService> => {
-      const taskQueueName = options.entity ?? options.taskQueue;
-      const workflowName = options.entity ?? options.workflowName;
-      const trc = options.workflowTrace;
-      const spn = options.workflowSpan;
-      //NOTE: HotMesh 'workflowTopic' is a created by concatenating
-      //     the taskQueue and workflowName used by the Durable module
-      const workflowTopic = `${taskQueueName}-${workflowName}`;
-      const hotMeshClient = await this.getHotMeshClient(workflowTopic, options.namespace);
-      this.configureSearchIndex(hotMeshClient, options.search);
-      const payload = {
-        arguments: [...options.args],
-        originJobId: options.originJobId,
-        expire: options.expire ?? HMSH_EXPIRE_JOB_SECONDS,
-        parentWorkflowId: options.parentWorkflowId,
-        workflowId: options.workflowId || HotMesh.guid(),
-        workflowTopic: workflowTopic,
-        backoffCoefficient: options.config?.backoffCoefficient || HMSH_DURABLE_EXP_BACKOFF,
-        maximumAttempts: options.config?.maximumAttempts || HMSH_DURABLE_MAX_ATTEMPTS,
-        maximumInterval: ms(options.config?.maximumInterval || HMSH_DURABLE_MAX_INTERVAL) / 1000,
-      }
-
-      const context = { metadata: { trc, spn }, data: {}};
-      const jobId = await hotMeshClient.pub(
-        `${options.namespace ?? APP_ID}.execute`,
-        payload,
-        context as JobState,
-        { search: options?.search?.data, marker: options?.marker},
-      );
-      return new WorkflowHandleService(hotMeshClient, workflowTopic, jobId);
-    },
-
-    /**
-     * send a message to a running workflow that is paused and awaiting the signal
-     */
-    signal: async (signalId: string, data: Record<any, any>, namespace?: string): Promise<string> => {
-      const topic = `${namespace ?? APP_ID}.wfs.signal`;
-      return await (await this.getHotMeshClient(topic, namespace)).hook(topic, { id: signalId, data });
-    },
-
-    /**
-     * send a message to spawn an parallel in-process thread of execution
-     * with the same job state as the main thread but bound to a different
-     * handler function. All job state will be journaled to the same hash
-     * as is used by the main thread.
-     */
-    hook: async (options: HookOptions): Promise<string> => {
-      const workflowTopic = `${options.taskQueue}-${options.workflowName}`;
-      const payload = {
-        arguments: [...options.args],
-        id: options.workflowId,
-        workflowTopic,
-        backoffCoefficient: options.config?.backoffCoefficient || HMSH_DURABLE_EXP_BACKOFF,
-        maximumAttempts: options.config?.maximumAttempts || HMSH_DURABLE_MAX_ATTEMPTS,
-        maximumInterval: ms(options.config?.maximumInterval || HMSH_DURABLE_MAX_INTERVAL) / 1000,
-      }
-      //seed search data if presentthe hook before entering
-      const hotMeshClient = await this.getHotMeshClient(workflowTopic, options.namespace);
-      const msgId = await hotMeshClient.hook(`${hotMeshClient.appId}.flow.signal`, payload, StreamStatus.PENDING, 202);
-      if (options.search?.data) {
-        const searchSessionId = `-search-${HotMesh.guid()}-0`;
-        const search = new Search(options.workflowId, hotMeshClient, searchSessionId);
-        const entries = Object.entries(options.search.data).flat();
-        await search.set(...entries);
-      }
-      return msgId;
-    },
-
-    getHandle: async (taskQueue: string, workflowName: string, workflowId: string, namespace?: string): Promise<WorkflowHandleService> => {
-      const workflowTopic = `${taskQueue}-${workflowName}`;
-      const hotMeshClient = await this.getHotMeshClient(workflowTopic, namespace);
-      return new WorkflowHandleService(hotMeshClient, workflowTopic, workflowId);
-    },
-
-    search: async (taskQueue: string, workflowName: string, namespace: null | string, index: string, ...query: string[]): Promise<string[]> => {
-      const workflowTopic = `${taskQueue}-${workflowName}`;
-      const hotMeshClient = await this.getHotMeshClient(workflowTopic, namespace);
-      try {
-        return await this.search(hotMeshClient, index, query);
-      } catch (error) {
-        hotMeshClient.engine.logger.error('durable-client-search-err', { ...error });
-        throw error;
-      }
-    }
-  }
-
-  async verifyWorkflowActive(hotMesh: HotMesh, appId = APP_ID, count = 0): Promise<boolean> {
-    const app = await hotMesh.engine.store.getApp(appId);
-    const appVersion = app?.version as unknown as number;
-    if(isNaN(appVersion)) {
-      if (count > 10) {
-        throw new Error('Workflow failed to activate');
-      }
-      await sleepFor(HMSH_QUORUM_DELAY_MS * 2);
-      return await this.verifyWorkflowActive(hotMesh, appId, count + 1);
-    }
-    return true;
-  }
-
-  async activateWorkflow(hotMesh: HotMesh, appId = APP_ID, version = APP_VERSION): Promise<void> {
-    const app = await hotMesh.engine.store.getApp(appId);
-    const appVersion = app?.version as unknown as number;
-    if(isNaN(appVersion)) {
-      try {
-        await hotMesh.deploy(getWorkflowYAML(appId, version));
-        await hotMesh.activate(version);
-      } catch (error) {
-        hotMesh.engine.logger.error('durable-client-deploy-activate-err', { ...error });
-        throw error;
-      }
-    } else if(app && !app.active) {
-      try {
-        await hotMesh.activate(version);
-      } catch (error) {
-        hotMesh.engine.logger.error('durable-client-activate-err', { error});
-        throw error;
-      }
-    }
-  }
-
-  static async shutdown(): Promise<void> {
-    for (const [_, hotMeshInstance] of ClientService.instances) {
-      (await hotMeshInstance).stop();
-    }
-  }
-}

package/services/durable/connection.ts

@@ -1,10 +0,0 @@
-import { Connection, ConnectionConfig } from "../../types/durable";
-
-export class ConnectionService {
-  static async connect(config: ConnectionConfig): Promise<Connection> {
-    return {
-      class: config.class,
-      options: { ...config.options },
-     } as Connection;
-  }
-}