@hotmeshio/hotmesh 0.0.1

package/modules/key.ts

@@ -0,0 +1,129 @@
+/**
+ * Keys
+ * 
+ * hmsh ->                                            {hash}    hotmesh config {version: "0.0.1", namespace: "hmsh"}
+ * hmsh:a:<appid> ->                                  {hash}    app profile { "id": "appid", "version": "2", "versions/1": "GMT", "versions/2": "GMT"}
+ * hmsh:<appid>:e:<engineId> ->                       {string}  setnx to ensure only one engine of given id
+ * hmsh:<appid>:w: ->                                 {zset}    work items/tasks an engine must do like garbage collect or hook a set of matching records (hookAll)
+ * hmsh:<appid>:t: ->                                 {zset}    an ordered set of list (work lists) ids
+ * hmsh:<appid>:t:<timeValue?> ->                     {list}    a worklist of `jobId+activityId` items that should be awakened
+ * hmsh:<appid>:q: ->                                 {hash}    quorum-wide messages
+ * hmsh:<appid>:q:<ngnid> ->                          {hash}    engine-targeted messages (targeted quorum-oriented message)
+ * hmsh:<appid>:j:<jobid> ->                          {hash}    job data
+ * hmsh:<appid>:j:<jobid>:<activityid>  ->            {hash}    job activity data (a1)
+ * hmsh:<appid>:s:<jobkey>:<dateTime> ->              {hash}    job stats (general)
+ * hmsh:<appid>:s:<jobkey>:<dateTime>:mdn:<field/path>:<fieldvalue> ->      {zset}    job stats (median)
+ * hmsh:<appid>:s:<jobkey>:<dateTime>:index:<field/path>:<fieldvalue> ->    {list}    job stats (index of jobid[])
+ * hmsh:<appid>:v:<version>:activities ->             {hash}    schemas [cache]
+ * hmsh:<appid>:v:<version>:transitions ->            {hash}    transitions [cache]
+ * hmsh:<appid>:v:<version>:subscriptions ->          {hash}    subscriptions [cache]
+ * hmsh:<appid>:x: ->                                 {xstream} when an engine is sent or reads a buffered task (engines read from their custom topic)
+ * hmsh:<appid>:x:<topic> ->                          {xstream} when a worker is sent or reads a buffered task (workers read from their custom topic)
+ * hmsh:<appid>:hooks ->                              {hash}    hook patterns/rules; set at compile time
+ * hmsh:<appid>:signals ->                            {hash}    dynamic hook signals (hget/hdel) when resolving (always self-clean); added/removed at runtime
+ * hmsh:<appid>:sym:keys: ->                          {hash}    list of symbol ranges and :cursor assigned at version deploy time for job keys
+ * hmsh:<appid>:sym:keys:<activityid|$subscribes> ->  {hash}    list of symbols based upon schema enums (initially) and adaptively optimized (later) during runtime; if '$subscribes' is used as the activityid, it is a top-level `job` symbol set (for job keys)
+ * hmsh:<appid>:sym:vals: ->                          {hash}    list of symbols for job values across all app versions
+ */
+
+//default namespace for hotmesh
+const PSNS = "hmsh";
+
+//these are the entity types that are stored in the key/value store
+enum KeyType {
+  APP,
+  ENGINE_ID,
+  HOOKS,
+  JOB_STATE,
+  JOB_STATS_GENERAL,
+  JOB_STATS_MEDIAN,
+  JOB_STATS_INDEX,
+  HOTMESH,
+  QUORUM,
+  SCHEMAS,
+  SIGNALS,
+  STREAMS,
+  SUBSCRIPTIONS,
+  SUBSCRIPTION_PATTERNS,
+  SYMKEYS,
+  SYMVALS,
+  TIME_RANGE,
+  WORK_ITEMS,
+}
+
+//when minting a key, the following parameters are used to create a unique key per entity
+type KeyStoreParams = {
+  appId?: string;       //app id is a uuid for a hotmesh app
+  engineId?: string;    //unique auto-generated guid for an ephemeral engine instance
+  appVersion?: string;  //(e.g. "1.0.0", "1", "1.0")
+  jobId?: string;       //a customer-defined id for job; must be unique for the entire app
+  activityId?: string;  //activity id is a uuid for a given hotmesh app
+  jobKey?: string;      //a customer-defined label for a job that serves to categorize events 
+  dateTime?: string;    //UTC date time: YYYY-MM-DDTHH:MM (20203-04-12T00:00); serves as a time-series bucket for the job_key
+  facet?: string;       //data path starting at root with values separated by colons (e.g. "object/type:bar")
+  topic?: string;       //topic name (e.g., "foo" or "" for top-level)
+  timeValue?: number;   //time value (rounded to minute) (for delete range)
+};
+
+class KeyService {
+
+  /**
+   * returns a key that can be used to access a value in the key/value store
+   * appropriate for the given key type; the keys have an implicit hierarchy
+   * and are used to organize data in the store in a tree-like structure
+   * via the use of colons as separators. The top-level entity is the hmsh manifest.
+   * This file will reveal the full scope of what is on the server (apps, versions, etc)
+   * @param namespace 
+   * @param keyType 
+   * @param params 
+   * @returns {string}
+   */
+  static mintKey(namespace: string, keyType: KeyType, params: KeyStoreParams): string {
+    switch (keyType) {
+      case KeyType.HOTMESH:
+        return namespace;
+      case KeyType.ENGINE_ID:
+        return `${namespace}:${params.appId}:e:${params.engineId}`;
+      case KeyType.WORK_ITEMS:
+        return `${namespace}:${params.appId}:w:`;
+      case KeyType.TIME_RANGE:
+        return `${namespace}:${params.appId}:t:${params.timeValue || ''}`;
+      case KeyType.APP:
+        return `${namespace}:a:${params.appId || ''}`;
+      case KeyType.QUORUM:
+        return `${namespace}:${params.appId}:q:${params.engineId || ''}`;
+      case KeyType.JOB_STATE:
+        return `${namespace}:${params.appId}:j:${params.jobId}`;
+      case KeyType.JOB_STATS_GENERAL:
+        return `${namespace}:${params.appId}:s:${params.jobKey}:${params.dateTime}`;
+      case KeyType.JOB_STATS_MEDIAN:
+        return `${namespace}:${params.appId}:s:${params.jobKey}:${params.dateTime}:${params.facet}`;
+      case KeyType.JOB_STATS_INDEX:
+        return `${namespace}:${params.appId}:s:${params.jobKey}:${params.dateTime}:${params.facet}`;
+      case KeyType.SCHEMAS:
+        return `${namespace}:${params.appId}:v:${params.appVersion}:schemas`;
+      case KeyType.SUBSCRIPTIONS:
+        return `${namespace}:${params.appId}:v:${params.appVersion}:subscriptions`;
+      case KeyType.SUBSCRIPTION_PATTERNS:
+        return `${namespace}:${params.appId}:v:${params.appVersion}:transitions`;
+      case KeyType.HOOKS:
+        //`hooks` provide the pattern to resolve a value
+        return `${namespace}:${params.appId}:hooks`;
+      case KeyType.SIGNALS:
+        //`signals` provide the registry of resolved values that link back to paused jobs
+        return `${namespace}:${params.appId}:signals`;
+      case KeyType.SYMKEYS:
+        //`symbol keys` provide the registry of replacement values for job keys
+        return `${namespace}:${params.appId}:sym:keys:${params.activityId || ''}`;
+      case KeyType.SYMVALS:
+        //`symbol vals` provide the registry of replacement values for job vals
+        return `${namespace}:${params.appId}:sym:vals:`;
+      case KeyType.STREAMS:
+        return `${namespace}:${params.appId || ''}:x:${params.topic || ''}`;
+      default:
+        throw new Error("Invalid key type.");
+    }
+  }
+}
+
+export { KeyService, KeyType, KeyStoreParams, PSNS };

package/modules/utils.ts

@@ -0,0 +1,170 @@
+import { StoreService } from "../services/store";
+import { AppSubscriptions, AppTransitions, AppVID } from "../types/app";
+import { RedisClient, RedisMulti } from "../types/redis";
+import { StringAnyType } from "../types/serializer";
+import { StreamCode, StreamStatus } from "../types/stream";
+
+export async function sleepFor(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export function identifyRedisType(redisInstance: any): 'redis' | 'ioredis' | null {
+  if (redisInstance.constructor) {
+    if (redisInstance.constructor.name === 'Redis' || redisInstance.constructor.name === 'EventEmitter') {
+      if ('hset' in redisInstance) {
+        return 'ioredis';
+      }
+    } else if (redisInstance.constructor.name === 'RedisClient' || redisInstance.constructor.name === 'Commander') {
+      if ('HSET' in redisInstance) {
+        return 'redis';
+      }
+    }
+  }
+  return null;
+}
+
+export function identifyRedisTypeFromClass(redisClass: any): 'redis' | 'ioredis' | null {
+  if (redisClass && redisClass.name === 'Redis' || redisClass.name === 'EventEmitter') {
+    return 'ioredis';
+  } else if (redisClass && 'createClient' in redisClass) {
+    return 'redis';
+  }
+  return null;
+}
+
+export function matchesStatusCode(code: StreamCode, pattern: string | RegExp): boolean {
+  if (typeof pattern === 'string') {
+    // Convert '*' wildcard to its regex equivalent (\d)
+    const regexPattern = `^${pattern.replace(/\*/g, "\\d")}$`;
+    return new RegExp(regexPattern).test(code.toString());
+  }
+  return pattern.test(code.toString());
+}
+
+export function matchesStatus(status: StreamStatus, targetStatus: StreamStatus): boolean {
+  return status === targetStatus;
+}
+
+export function XSleepFor(ms: number): { promise: Promise<unknown>, timerId: NodeJS.Timeout } {
+  //can be interrupted with `clearTimeout`
+  let timerId: NodeJS.Timeout;
+  let promise = new Promise((resolve) => {
+    timerId = setTimeout(resolve, ms);
+  });
+  return { promise, timerId };
+}
+
+export function findTopKey(obj: AppTransitions, input: string): string | null {
+  for (const [key, value] of Object.entries(obj)) {
+    if (value.hasOwnProperty(input)) {
+      const parentKey = findTopKey(obj, key.replace(/^\./, ''));
+      return (parentKey || key).replace(/^\./, '');
+    }
+  }
+  return null;
+}
+
+export function findSubscriptionForTrigger(obj: AppSubscriptions, value: string): string | null {
+  for (const [key, itemValue] of Object.entries(obj)) {
+    if (itemValue === value) {
+      return key;
+    }
+  }
+  return null;
+}
+
+/**
+ * Get the subscription topic for the flow to which @activityId belongs.
+ * TODO: resolve this value in the compiler...do not call this at runtime
+ */
+export async function getSubscriptionTopic(activityId: string, store: StoreService<RedisClient, RedisMulti>, appVID: AppVID): Promise<string | undefined> {
+  const appTransitions = await store.getTransitions(appVID);
+  const appSubscriptions = await store.getSubscriptions(appVID);
+  const triggerId = findTopKey(appTransitions, activityId);
+  const topic = findSubscriptionForTrigger(appSubscriptions, triggerId);
+  return topic;
+}
+
+/**
+ * returns the 12-digit format of the iso timestamp (e.g, 202101010000)
+ */
+export function getTimeSeries(granularity: string): string {
+  const now = new Date();
+  const granularityUnit = granularity.slice(-1);
+  const granularityValue = parseInt(granularity.slice(0, -1), 10);
+  if (granularityUnit === 'm') {
+    const minute = Math.floor(now.getMinutes() / granularityValue) * granularityValue;
+    now.setUTCMinutes(minute, 0, 0);
+  } else if (granularityUnit === 'h') {
+    now.setUTCMinutes(0, 0, 0);
+  }
+  return now.toISOString().replace(/:\d\d\..+|-|T/g, '').replace(':','');
+}
+
+export function formatISODate(input: Date | string): string {
+  const date = input instanceof Date ? input : new Date(input);
+  return date.toISOString().replace(/[:TZ-]/g, '');
+}
+
+export function getSymKey(number: number): string {
+  const alphabet = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
+  const base = alphabet.length;
+  if (number < 0 || number >= Math.pow(base, 3)) {
+    throw new Error('Number out of range');
+  }
+  let [q1, r1] = divmod(number, base);
+  let [q2, r2] = divmod(q1, base);
+  return alphabet[q2] + alphabet[r1] + alphabet[r2];
+}
+
+export function getSymVal(number: number): string {
+  const alphabet = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
+  const base = alphabet.length;
+  if (number < 0 || number >= Math.pow(base, 2)) {
+    throw new Error('Number out of range');
+  }
+  let [q, r] = divmod(number, base);
+  return alphabet[q] + alphabet[r];
+}
+
+function divmod(m: number, n: number): number[] {
+  return [Math.floor(m / n), m % n];
+}
+
+export function getIndexedHash<T>(hash: T, target: string): [number, T] {
+  const index = hash[target] as number || 0;
+  const newHash = { ...hash };
+  delete newHash[target];
+  return [index, newHash as T];
+}
+
+export function getValueByPath(obj: { [key: string]: any }, path: string): any {
+  const pathParts = path.split('/');
+  let currentValue = obj;
+  for (const part of pathParts) {
+    if (currentValue[part] !== undefined) {
+      currentValue = currentValue[part];
+    } else {
+      return undefined;
+    }
+  }
+  return currentValue;
+}
+
+export function restoreHierarchy(obj: StringAnyType): StringAnyType {
+  const result: StringAnyType = {};
+  for (const key in obj) {
+    if (obj[key] === undefined) continue;
+    const keys = key.split('/');
+    let current = result;
+    for (let i = 0; i < keys.length; i++) {
+      if (i === keys.length - 1) {
+        current[keys[i]] = obj[key];
+      } else {
+        current[keys[i]] = current[keys[i]] || {};
+        current = current[keys[i]];
+      }
+    }
+  }
+  return result;
+}

package/package.json

@@ -0,0 +1,73 @@
+{
+  "name": "@hotmeshio/hotmesh",
+  "version": "0.0.1",
+  "description": "Durable Workflows",
+  "main": "build/index.js",
+  "types": "build/index.d.ts",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/hotmeshio/hotmesh.git"
+  },
+  "homepage": "https://github.com/hotmeshio/hotmesh#readme",
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "clean": "rimraf ./build",
+    "build": "tsc --build tsconfig.json",
+    "clean-build": "npm run clean && npm run build",
+    "lint": "eslint . --ext .ts",
+    "lint:fix": "eslint . --fix --ext .ts",
+    "start": "ts-node src/index.ts",
+    "test": "NODE_ENV=test jest --detectOpenHandles --forceExit --verbose",
+    "test:hmsh": "NODE_ENV=test jest ./tests/functional/index.test.ts --detectOpenHandles --verbose",
+    "test:compile": "NODE_ENV=test jest ./tests/functional/compile/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:connect": "NODE_ENV=test jest ./tests/unit/services/connector/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:connect:redis": "NODE_ENV=test jest ./tests/unit/services/connector/clients/redis.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:connect:ioredis": "NODE_ENV=test jest ./tests/unit/services/connector/clients/ioredis.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:parallel": "NODE_ENV=test jest ./tests/functional/parallel/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:sequence": "NODE_ENV=test jest ./tests/functional/sequence/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:quorum": "NODE_ENV=test jest ./tests/functional/quorum/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:reclaim": "NODE_ENV=test jest ./tests/functional/reclaim/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:redeploy": "NODE_ENV=test jest ./tests/functional/redeploy/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:retry": "NODE_ENV=test jest ./tests/functional/retry/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:status": "NODE_ENV=test jest ./tests/functional/status/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:store:redis": "NODE_ENV=test jest ./tests/functional/store/clients/redis.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:store:ioredis": "NODE_ENV=test jest ./tests/functional/store/clients/ioredis.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:stream:redis": "NODE_ENV=test jest ./tests/functional/stream/clients/redis.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:stream:ioredis": "NODE_ENV=test jest ./tests/functional/stream/clients/ioredis.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:sub:redis": "NODE_ENV=test jest ./tests/functional/sub/clients/redis.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:sub:ioredis": "NODE_ENV=test jest ./tests/functional/sub/clients/ioredis.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:durable": "NODE_ENV=test jest ./tests/durable/*/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:durable:hello": "NODE_ENV=test jest ./tests/durable/helloworld/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:durable:goodbye": "NODE_ENV=test jest ./tests/durable/goodbye/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:durable:loopactivity": "NODE_ENV=test jest ./tests/durable/loopactivity/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:durable:nested": "NODE_ENV=test jest ./tests/durable/nested/index.test.ts --detectOpenHandles --forceExit --verbose"
+  },
+  "keywords": [],
+  "author": "luke.birdeau@gmail.com",
+  "license": "SEE LICENSE IN LICENSE",
+  "dependencies": {
+    "@apidevtools/json-schema-ref-parser": "^10.1.0",
+    "@opentelemetry/api": "^1.4.1",
+    "js-yaml": "^4.1.0",
+    "ms": "^2.1.3",
+    "nanoid": "^3.3.6",
+    "winston": "^3.8.2"
+  },
+  "devDependencies": {
+    "@types/jest": "^29.5.0",
+    "@types/node": "^18.15.11",
+    "@typescript-eslint/eslint-plugin": "^5.56.0",
+    "@typescript-eslint/parser": "^5.56.0",
+    "eslint": "^8.36.0",
+    "ioredis": "^5.3.2",
+    "jest": "^29.5.0",
+    "redis": "^4.6.6",
+    "rimraf": "^4.4.1",
+    "ts-jest": "^29.0.5",
+    "ts-node": "^10.9.1",
+    "ts-node-dev": "^2.0.0",
+    "typescript": "^5.0.4"
+  }
+}