@hotmeshio/hotmesh 0.0.1

package/types/hotmesh.ts

@@ -0,0 +1,102 @@
+import { ILogger } from "../services/logger";
+import { HotMeshService } from "../services/hotmesh";
+import { HookRules } from "./hook";
+import { RedisClass, RedisClient, RedisOptions } from "./redis";
+import { StreamData, StreamDataResponse } from "./stream";
+
+type HotMesh = typeof HotMeshService;
+
+type RedisConfig = {
+  class: RedisClass;
+  options: RedisOptions;
+}
+
+type HotMeshEngine = {
+  store?: RedisClient;  //set by hotmesh using instanced `redis` class
+  stream?: RedisClient; //set by hotmesh using instanced `redis` class
+  sub?: RedisClient;    //set by hotmesh using instanced `redis` class
+  redis?: RedisConfig;
+  reclaimDelay?: number; //milliseconds
+  reclaimCount?: number;
+}
+
+type HotMeshWorker = {
+  topic: string;
+  store?: RedisClient;  //set by hotmesh using instanced `redis` class
+  stream?: RedisClient; //set by hotmesh using instanced `redis` class
+  sub?: RedisClient;    //set by hotmesh using instanced `redis` class
+  redis?: {
+    class: RedisClass;
+    options: RedisOptions;
+  };
+  reclaimDelay?: number; //milliseconds
+  reclaimCount?: number; //max number of times to reclaim a stream
+  callback: (payload: StreamData) => Promise<StreamDataResponse|void>;
+}
+
+type HotMeshConfig = {
+  appId: string;
+  namespace?: string;
+  name?: string;
+  logger?: ILogger;
+  logLevel?: 'silly' | 'debug' | 'info' | 'warn' | 'error' | 'silent';
+  engine?: HotMeshEngine;
+  workers?: HotMeshWorker[];
+}
+
+type HotMeshGraph = {
+  subscribes: string;
+  publishes?: string;
+  expire?: number;
+  output?: {
+    schema: Record<string, any>;
+  };
+  input?: {
+    schema: Record<string, any>;
+  };
+  activities: Record<string, any>;
+  transitions?: Record<string, any>;
+  hooks?: HookRules;
+};
+
+type HotMeshSettings = {
+  namespace: string;
+  version: string;
+};
+
+type HotMeshManifest = {
+  app: {
+    id: string;
+    version: string;
+    settings: Record<string, any>;
+    graphs: HotMeshGraph[];
+  };
+};
+
+type VersionedFields = {
+  [K in `versions/${string}`]: any;
+};
+
+type HotMeshApp = VersionedFields & {
+  id: string;        // customer's chosen app id
+  version: string;   // customer's chosen version scheme (semver, etc)
+  settings?: string; // stringified JSON for app settings
+  active?: boolean;  // is the app active?
+};
+
+type HotMeshApps = {
+  [appId: string]: HotMeshApp;
+};
+
+export {
+  HotMesh,
+  HotMeshEngine,
+  RedisConfig,
+  HotMeshWorker,
+  HotMeshSettings,
+  HotMeshApp,    //a single app in the db
+  HotMeshApps,   //object array of all apps in the db
+  HotMeshConfig, //customer config
+  HotMeshManifest,
+  HotMeshGraph
+};

package/types/index.ts

@@ -0,0 +1,138 @@
+export {
+  ActivityType,
+  ActivityDataType,
+  ActivityContext,
+  ActivityData,
+  ActivityDuplex,
+  ActivityLeg,
+  ActivityMetadata,
+  Consumes,
+  AwaitActivity,
+  BaseActivity,
+  EmitActivity,
+  WorkerActivity,
+  IterateActivity,
+  TriggerActivity,
+  TriggerActivityStats } from './activity';
+export {
+  App,
+  AppVID,
+  AppTransitions,
+  AppSubscriptions
+} from './app';
+export { AsyncSignal } from './async';
+export { CacheMode } from './cache';
+export {
+  CollationFaultType,
+  CollationStage } from './collator';
+export {
+  HookCondition,
+  HookConditions,
+  HookGate,
+  HookInterface,
+  HookRule,
+  HookRules,
+  HookSignal
+} from './hook';
+export {
+  RedisClientType as IORedisClientType,
+  RedisMultiType as IORedisMultiType } from './ioredisclient';
+export { ILogger } from './logger';
+export {
+  JobData,
+  JobsData,
+  JobMetadata,
+  JobOutput,
+  JobState,
+  JobStatus,
+  PartialJobState } from './job';
+export { MappingStatements } from './map';
+export {
+  Pipe,
+  PipeItem,
+  PipeItems } from './pipe';
+export {
+  HotMesh,
+  HotMeshApp,
+  HotMeshApps,
+  HotMeshConfig,
+  HotMeshEngine,
+  RedisConfig,
+  HotMeshGraph,
+  HotMeshManifest,
+  HotMeshSettings,
+  HotMeshWorker } from './hotmesh';
+export {
+  ActivateMessage,
+  JobMessage,
+  JobMessageCallback,
+  PingMessage,
+  PongMessage,
+  QuorumMessage,
+  SubscriptionCallback,
+  ThrottleMessage,
+  WorkMessage } from './quorum';
+export {
+  MultiResponseFlags,
+  RedisClient,
+  RedisMulti } from './redis'; //common redis types
+export {
+  RedisClientType,
+  RedisMultiType } from './redisclient';
+export {
+  JSONSchema,
+  StringAnyType,
+  StringScalarType,
+  StringStringType,
+  SymbolMap,
+  SymbolMaps,
+  SymbolRanges,
+  Symbols,
+  SymbolSets } from './serializer';
+export {
+  AggregatedData,
+  CountByFacet,
+  GetStatsOptions,
+  IdsData,
+  Measure,
+  MeasureIds,
+  MetricTypes,
+  StatType,
+  StatsType,
+  IdsResponse,
+  JobStats,
+  JobStatsInput,
+  JobStatsRange,
+  StatsResponse,
+  Segment,
+  TimeSegment } from './stats';
+export {
+  ReclaimedMessageType,
+  StreamCode,
+  StreamConfig,
+  StreamData,
+  StreamDataType,
+  StreamError,
+  StreamDataResponse,
+  StreamRetryPolicy,
+  StreamRole,
+  StreamStatus } from './stream';
+export {
+  context,
+  Context,
+  Counter,
+  Meter,
+  metrics,
+  propagation,
+  SpanContext,
+  Span,
+  SpanStatus,
+  SpanStatusCode,
+  SpanKind,
+  trace,
+  Tracer,
+  ValueType } from './telemetry';
+export {
+  TransitionMatch,
+  TransitionRule,
+  Transitions } from './transition';

package/types/ioredisclient.ts

@@ -0,0 +1,10 @@
+import { Redis, ChainableCommander, RedisOptions } from 'ioredis';
+
+type RedisClassType = { new (options: RedisOptions): Redis };
+
+export {
+  RedisClassType,
+  Redis as RedisClientType,
+  ChainableCommander as RedisMultiType,
+  RedisOptions as RedisClientOptions,
+}

package/types/job.ts

@@ -0,0 +1,59 @@
+type JobData = Record<string, unknown|Record<string, unknown>>;
+type JobsData = Record<string, unknown>;
+
+type ActivityData = {
+  data: Record<string, unknown>;
+  metadata?: Record<string, unknown>;
+};
+
+type JobMetadata = {
+  key?: string; //job_key
+  jid: string;  //job_id (jid+dad+aid) is composite key for activity
+  dad: string;  //dimensional address for the activity (,0,0,1)
+  aid: string;  //activity_id as in the YAML file
+  pj?: string;  //parent_job_id (pj+pd+pa) is composite key for parent activity
+  pd?: string;  //parent_dimensional_address
+  pa?: string;  //parent_activity_id
+  ngn?: string; //engine guid (one time subscriptions)
+  app: string;  //app_id
+  vrs: string;  //app version
+  tpc: string;  //subscription topic
+  ts: string    //201203120005 (slice of time) //time series
+  jc: string;   //GMT created //job_created
+  ju: string;   //GMT updated //job_updated
+  js: JobStatus;
+  atp: string;  //activity_type
+  stp: string;  //activity_subtype
+  spn: string;  //open telemetry span context
+  trc: string;  //open telemetry trace context
+  err?: string; //stringified job error json: {message: string, code: number, error?}
+  expire?: number; //process data expire policy
+};
+
+type JobStatus = number;       //job_status semaphore 
+
+type JobState = {
+  metadata: JobMetadata;
+  data: JobData;
+  [activityId: symbol]: {
+    input: ActivityData;
+    output: ActivityData;
+    hook: ActivityData;
+    settings: ActivityData;
+    errors: ActivityData;
+  };
+};
+
+//format when publishing job meta/data on the wire when it completes
+type JobOutput = {
+  metadata: JobMetadata;
+  data: JobData;
+};
+
+//jid+dad+aid is a composite guid; signal in and restore the full job context
+type PartialJobState = {
+  metadata: JobMetadata | Pick<JobMetadata, 'jid' | 'dad' | 'aid'>;
+  data: JobData;
+};
+
+export { JobState, JobStatus, JobData, JobsData, JobMetadata, PartialJobState, JobOutput };

package/types/logger.ts

@@ -0,0 +1,6 @@
+export interface ILogger {
+  info(message: string, ...meta: any[]): void;
+  error(message: string, ...meta: any[]): void;
+  warn(message: string, ...meta: any[]): void;
+  debug(message: string, ...meta: any[]): void;
+}

package/types/map.ts

@@ -0,0 +1,5 @@
+type MappingStatements = {
+  [activityId: string]: string[];
+};
+
+export { MappingStatements };

package/types/ms.d.ts

@@ -0,0 +1,7 @@
+declare module 'ms' {
+  function ms(value: string): number;
+  function ms(value: number): string;
+  function ms(value: string | number): string | number;
+
+  export = ms;
+}

package/types/pipe.ts

@@ -0,0 +1,7 @@
+type PipeItem = string | boolean | number | null;
+
+type PipeItems = PipeItem[];
+
+type Pipe = (PipeItem[] | Pipe)[];
+
+export { Pipe, PipeItem, PipeItems };

package/types/quorum.ts

@@ -0,0 +1,59 @@
+import { JobOutput } from "./job";
+
+/**
+ * The types in this file are used to define those messages that are sent
+ * to hotmesh client instances when a new version is about to be activated.
+ * These messages serve to coordinate the cache invalidation and switch-over
+ * to the new version without any downtime and a coordinating parent server.
+ */
+export type QuorumMessage = PingMessage | PongMessage | ActivateMessage | WorkMessage | JobMessage | ThrottleMessage;
+
+//used for coordination like version activation
+export interface PingMessage {
+  type: 'ping';
+  originator: string; //guid
+}
+
+export interface WorkMessage {
+  type: 'work';
+  originator: string; //guid
+}
+
+//used for coordination like version activation
+export interface PongMessage {
+  type: 'pong';
+  originator: string; //clone of originator guid passed in ping
+  guid: string;
+}
+
+export interface ActivateMessage {
+  type: 'activate';
+  cache_mode: 'nocache' | 'cache';
+  until_version: string;
+}
+
+export interface JobMessage {
+  type: 'job';
+  topic: string; //this comes from the 'publishes' field in the YAML
+  job: JobOutput
+}
+
+//delay in ms between fetches from the buffered stream (speed/slow down entire network)
+export interface ThrottleMessage {
+  type: 'throttle';
+  guid?: string; //target the engine quorum
+  topic?: string;  //target a worker quorum
+  throttle: number; //0-n
+}
+
+export interface JobMessageCallback {
+  (topic: string, message: JobOutput): void;
+}
+
+export interface SubscriptionCallback {
+  (topic: string, message: Record<string, any>): void;
+}
+
+export interface QuorumMessageCallback {
+  (topic: string, message: QuorumMessage): void;
+}

package/types/redis.ts

@@ -0,0 +1,27 @@
+import {
+  RedisClassType,
+  RedisClientType as RCT,
+  RedisMultiType as RMT,
+  RedisClientOptions as RCO
+} from './redisclient';
+import {
+  RedisClassType as IORRedisClassType,
+  RedisClientType as IORCT,
+  RedisMultiType as IORMT,
+  RedisClientOptions as IORCO
+} from './ioredisclient';
+
+type RedisClass = RedisClassType | IORRedisClassType;
+type RedisClient = RCT | IORCT;
+type RedisMulti = RMT | IORMT;
+type RedisOptions = RCO | IORCO;
+
+type MultiResponseFlags = (string | number)[]; // e.g., [3, 2, '0']
+
+export {
+  RedisClass,
+  RedisClient,
+  RedisMulti,
+  RedisOptions,
+  MultiResponseFlags,
+}

package/types/redisclient.ts

@@ -0,0 +1,29 @@
+import { createClient, RedisClientOptions } from 'redis';
+
+type RedisClientType = ReturnType<typeof createClient>;
+
+interface RedisMultiType {
+  XADD(key: string, id: string, fields: any): this;
+  XACK(key: string, group: string, id: string): this;
+  XDEL(key: string, id: string): this;
+  HDEL(key: string, itemId: string): this;
+  HGET(key: string, itemId: string): this;
+  HGETALL(key: string): this;
+  HINCRBYFLOAT(key: string, itemId: string, value: number): this;
+  HMGET(key: string, itemIds: string[]): this;
+  HSET(key: string, values: Record<string, string>): this;
+  LRANGE(key: string, start: number, end: number): this;
+  RPUSH(key: string, value: string): this;
+  ZADD(key: string, values: { score: string, value: string }): this;
+  sendCommand(command: string[]): Promise<any>;
+  exec: () => Promise<unknown[]>;
+}
+
+type RedisClassType = { createClient: (options: RedisClientOptions) => any };
+
+export {
+  RedisClassType,
+  RedisClientType,
+  RedisMultiType,
+  RedisClientOptions,
+}

package/types/serializer.ts

@@ -0,0 +1,38 @@
+export interface JSONSchema {
+  type?: string;
+  enum?: string[];
+  examples?: any[];
+  properties?: { [key: string]: JSONSchema };
+  items?: JSONSchema;
+  description?: string;
+  'x-train'?: boolean; //extension property to mark item `values` as not being trainable (ssn, dob, guids are examples of fields that should never have their `values` trained)
+}
+
+export type SymbolRanges = { //keyname is <ns>:<app>:symbols:
+  [key: string]: number; //eg: {"$": 0, "a1": 26, "a2" 39, "$metadata_cursor": 39, "$data_cursor": 2704} (job ($) holds range 0-26; every other activity has a number that increments by 13; up to 200+ unique activities may be modeled; the :cursor fields are used by the sytem to track the next reserved tranche using hincrby
+}
+
+export type Symbols = { //keyname is <ns>:<app>:symbols:<aid> (where aid can be $ for job or a1, a2, etc. for activities)
+  [key: string]: string; //eg: {"operation/name": "26", "a2" 39, ":cursor": 39} (job holds range 0-26; every other activity has a number that increments by 13; up to 200 activity ranges may be listed; one field called $count is used by the sytem to track the next reserved tranche using hincrby; job always seeds with 26
+}
+
+export type SymbolSets = {
+  [key: string]: {
+    [key: string]: string;
+  }
+}
+
+export type StringStringType = {
+  [key: string]: string;
+};
+
+export type StringAnyType = {
+  [key: string]: any;
+};
+
+export type StringScalarType = {
+  [key: string]: boolean | number | string;
+};
+
+export type SymbolMap = Map<string, string>;
+export type SymbolMaps = Map<string, SymbolMap>;

package/types/stats.ts

@@ -0,0 +1,100 @@
+type MetricTypes = 'count' | 'sum' | 'avg' | 'mdn' | 'max' | 'min' | 'index';
+
+interface StatType {
+  target: string;       //e.g, (a target on the input data: `<activity>.input.data`) => {`object/type:widgetA|widgetB:sum`: <sum>}, {`object/type:widgetA|widgetB:count`: <count>}
+  metric: MetricTypes;  //count, avg, etc
+  value: number|string; //a value to increment (sum); value to save to sorted set (mdn) or an id to add to an `index` or just '1' for a count
+}
+
+interface Measure {
+  target: string;
+  type: string;
+  value: number;
+}
+
+interface Segment {
+  time: string;
+  count: number;
+  measures: Measure[];
+}
+
+interface StatsType {
+  general: StatType[];
+  index: StatType[];
+  median: StatType[];
+}
+
+interface JobStats {
+  count?: number;
+  [field: string]: number;
+}
+
+interface JobStatsRange {
+  [key: string]: JobStats
+}
+
+interface JobStatsInput {
+  data: Record<string, unknown>;
+  range?: string;
+  start?: string;
+  end?: string;
+  sparse?: boolean;
+}
+
+interface GetStatsOptions {
+  key: string;
+  granularity: string;
+  range?: string;
+  start?: string;
+  end?: string;
+  sparse?: boolean;
+}
+
+interface StatsResponse {
+  key: string;
+  granularity: string;
+  range: string;
+  end: string | Date;
+  count: number;
+  measures: Measure[];
+  segments?: Segment[]; //`sparse` output does not subdived by datetime segments
+}
+
+interface AggregatedData {
+  [key: string]: number;
+}
+
+interface IdsData {
+  [target: string]: string[];
+}
+
+interface MeasureIds {
+  time: string;
+  target: string;
+  count: number;
+  type: 'ids';
+  ids: string[];
+}
+
+interface TimeSegment {
+  time: string;
+  measures: MeasureIds[];
+}
+
+
+interface CountByFacet {
+  facet: string;
+  count: number;
+}
+
+interface IdsResponse {
+  key: string;
+  facets: string[];
+  granularity: string;
+  range: string;
+  start: string;
+  counts: CountByFacet[];
+  segments: TimeSegment[];
+}
+
+export { StatsType, StatType, MetricTypes, JobStats, JobStatsRange, JobStatsInput, GetStatsOptions, StatsResponse, AggregatedData, Measure, Segment, IdsData, MeasureIds, TimeSegment, IdsResponse, CountByFacet };

package/types/stream.ts

@@ -0,0 +1,75 @@
+export interface StreamRetryPolicy {
+  [key: string]: [number, 'x']; //key is err code, val is the retry profile [(max retry count),(type (x:exponential (default)) (only 10, 100, 1000, 10000 allowed))
+}
+
+export type StreamCode = number; //3-digit status code
+
+export type StreamError = {
+  message: string;
+  code: number;
+  job_id?: string; //used when communicating errors externally
+  stack?: string;  //unhandled errors will have a stack
+  name?: string;   //unhandled errors will have a name
+  error?: Record<string, unknown>; //custom user-defined error details go here
+}
+
+export enum StreamStatus {
+  SUCCESS = 'success',
+  ERROR = 'error',
+  PENDING = 'pending',
+}
+
+export enum StreamDataType {
+  TIMEHOOK = 'timehook',
+  WEBHOOK = 'webhook',
+  AWAIT = 'await',
+  RESULT = 'result', //await result
+  WORKER = 'worker',
+  RESPONSE = 'response', //worker response
+  TRANSITION = 'transition',
+}
+
+export interface StreamData {
+  metadata: {
+    topic?: string;
+    jid?: string; //is optonal if type is WEBHOOK
+    dad?: string; //dimensional address
+    aid: string;
+    trc?: string; //trace id
+    spn?: string; //span id
+    try?: number; //current try count
+  };
+  type?: StreamDataType;
+  data: Record<string, unknown>;
+  policies?: {
+    retry?: StreamRetryPolicy;
+  };
+  status?: StreamStatus; //assume success
+  code?: number;         //assume 200
+}
+
+export interface StreamDataResponse extends StreamData {}
+
+export enum StreamRole {
+  WORKER = 'worker',
+  ENGINE = 'engine',
+  SYSTEM = 'system', //reserved for system use (i.e, if worker or engine fails)
+}
+
+export type ReclaimedMessageType = [
+  messageId: string, //stream id (e.g.,`<timestamp>-<count>`)
+  details: [
+    key: string,     //`key' is always 'message'
+    value: string    //`value` is stringified StreamData (StreamDataType)
+  ]
+][];                 //wrapped in an outer array
+
+export type StreamConfig = {
+  namespace: string;
+  appId: string;
+  guid: string;
+  role: StreamRole;
+  topic?: string;
+  reclaimDelay?: number; //default 60_000
+  reclaimCount?: number; //default 3 (any value greater throws an error)
+}

package/types/telemetry.ts

@@ -0,0 +1,15 @@
+export {
+  context,
+  Context,
+  Counter,
+  Meter,
+  metrics,
+  propagation,
+  SpanContext,
+  Span,
+  SpanStatus,
+  SpanStatusCode,
+  SpanKind,
+  trace,
+  Tracer,
+  ValueType } from '@opentelemetry/api';