@hotmeshio/hotmesh 0.0.1

package/services/telemetry/index.ts

@@ -0,0 +1,267 @@
+import packageJson from '../../package.json';
+import { MapperService } from '../mapper';
+import {
+  ActivityMetadata,
+  ActivityType,
+  Consumes } from '../../types/activity';
+import { JobState } from '../../types/job';
+import {
+  StringAnyType,
+  StringScalarType,
+  StringStringType } from '../../types/serializer';
+import { StreamData, StreamDataType, StreamRole } from '../../types/stream';
+import {
+  Span,
+  SpanContext,
+  SpanKind,
+  trace,
+  Context,
+  context, 
+  SpanStatusCode } from '../../types/telemetry';
+
+class TelemetryService {
+  span: Span;
+  jobSpan: Span;
+  config: ActivityType;
+  traceId: string | null;
+  spanId: string | null;
+  appId: string;
+  metadata: ActivityMetadata;
+  context: JobState;
+  leg = 1;
+
+  constructor(
+    appId: string,
+    config?: ActivityType,
+    metadata?: ActivityMetadata,
+    context?: JobState,
+  ) {
+    this.appId = appId;
+    //these are REQUIRED for job and activity spans
+    this.config = config;
+    this.metadata = metadata;
+    this.context = context;
+  }
+
+  getJobParentSpanId(): string | undefined {
+    return this.context.metadata.spn;
+  }
+
+  getActivityParentSpanId(leg: number): string | undefined {
+    if (leg === 1) {
+      return this.context[this.config.parent].output?.metadata?.l2s;
+    } else {
+      return this.context['$self'].output?.metadata?.l1s;
+    }
+  }
+
+  getTraceId(): string | undefined {
+    return this.context.metadata.trc;
+  }
+
+  startJobSpan(): TelemetryService {
+    const spanName = `JOB/${this.appId}/${this.config.subscribes}/1`;
+    const traceId = this.getTraceId();
+    const spanId = this.getJobParentSpanId();
+    const attributes = this.getSpanAttrs(1);
+    const span: Span = this.startSpan(traceId, spanId, spanName, attributes);
+    this.jobSpan = span;
+    this.setTelemetryContext(span, 1);
+    return this;
+  }
+
+  startActivitySpan(leg = this.leg): TelemetryService {
+    const spanName = `${this.config.type.toUpperCase()}/${this.appId}/${this.metadata.aid}/${leg}`;
+    const traceId = this.getTraceId();
+    const spanId = this.getActivityParentSpanId(leg);
+    const attributes = this.getSpanAttrs(leg);
+    const span: Span = this.startSpan(traceId, spanId, spanName, attributes);
+    this.setTelemetryContext(span, leg);
+    this.span = span;
+    return this;
+  }
+
+  startStreamSpan(data: StreamData, role: StreamRole): TelemetryService {
+    let type: string;
+    if (role === StreamRole.SYSTEM) {
+      type = 'SYSTEM';
+    } else if (role === StreamRole.WORKER) {
+      type = 'EXECUTE';
+    } else if (data.type === StreamDataType.RESULT || data.type === StreamDataType.RESPONSE) {
+      type = 'FANIN';
+    } else {
+      type = 'FANOUT';
+    }
+    const topic = data.metadata.topic ? `/${data.metadata.topic}` : '';
+    const spanName = `${type}/${this.appId}/${data.metadata.aid}${topic}`;
+    const attributes = this.getStreamSpanAttrs(data);
+    const span: Span = this.startSpan(data.metadata.trc, data.metadata.spn, spanName, attributes);
+    this.span = span;
+    return this;
+  }
+
+  startSpan(traceId: string, spanId: string, spanName: string, attributes: StringScalarType): Span {
+    this.traceId = traceId;
+    this.spanId = spanId;
+    const tracer = trace.getTracer(packageJson.name, packageJson.version);
+    let parentContext = this.getParentSpanContext();
+    const span = tracer.startSpan(
+      spanName,
+      { kind: SpanKind.CLIENT, attributes, root: !parentContext },
+      parentContext
+    );
+    return span;
+  }
+
+  mapActivityAttributes(): void {
+    //export user-defined span attributes (app.activity.data.*)
+    if (this.config.telemetry) {
+      const telemetryAtts = new MapperService(this.config.telemetry, this.context).mapRules();
+      const namespacedAtts = {
+        ...Object.keys(telemetryAtts).reduce((result, key) => {
+          if (['string', 'boolean', 'number'].includes(typeof telemetryAtts[key])) {
+            result[`app.activity.data.${key}`] = telemetryAtts[key];
+          }
+          return result;
+        }, {})
+      };
+      this.span.setAttributes(namespacedAtts as StringScalarType);
+    }
+  }
+
+  setActivityAttributes(attributes: StringScalarType): void {
+    this.span.setAttributes(attributes);
+  }
+
+  setStreamAttributes(attributes: StringScalarType): void {
+    this.span.setAttributes(attributes);
+  }
+
+  setJobAttributes(attributes: StringScalarType): void {
+    this.jobSpan.setAttributes(attributes);
+  }
+
+  endJobSpan(): void {
+    this.endSpan(this.jobSpan);
+  }
+
+  endActivitySpan(): void {
+    this.endSpan(this.span);
+  }
+
+  endStreamSpan(): void {
+    this.endSpan(this.span);
+  }
+
+  endSpan(span: Span): void {
+    span && span.end();
+  }
+
+  getParentSpanContext(): undefined | Context {
+    if (this.traceId && this.spanId) {
+      const restoredSpanContext: SpanContext = {
+        traceId: this.traceId,
+        spanId: this.spanId,
+        isRemote: true,
+        traceFlags: 1, // (todo: revisit sampling strategy/config)
+      };
+      const parentContext = trace.setSpanContext(context.active(), restoredSpanContext);
+      return parentContext;
+    }
+  }
+
+  getSpanAttrs(leg: number): StringAnyType {
+    return {
+      ...Object.keys(this.context.metadata).reduce((result, key) => {
+        if (key !== 'trc') {
+          result[`app.job.${key}`] = this.context.metadata[key];
+        }
+        return result;
+      }, {}),
+      ...Object.keys(this.metadata).reduce((result, key) => {
+        result[`app.activity.${key}`] = this.metadata[key];
+        return result;
+      }, {}),
+      'app.activity.leg': leg,
+    };
+  };
+
+  getStreamSpanAttrs(input: StreamData): StringAnyType {
+    return {
+      ...Object.keys(input.metadata).reduce((result, key) => {
+        if (key !== 'trc' && key !== 'spn') {
+          result[`app.stream.${key}`] = input.metadata[key];
+        }
+        return result;
+      }, {})
+    };
+  };
+
+  setTelemetryContext(span: Span, leg: number) {
+    if (!this.context.metadata.trc) {
+      this.context.metadata.trc = span.spanContext().traceId;
+    }
+    if (leg === 1) {
+      if (!this.context['$self'].output.metadata) {
+        this.context['$self'].output.metadata = {};
+      }
+      this.context['$self'].output.metadata.l1s = span.spanContext().spanId;
+    } else {
+      if (!this.context['$self'].output.metadata) {
+        this.context['$self'].output.metadata = {};
+      }
+      this.context['$self'].output.metadata.l2s = span.spanContext().spanId;
+    }
+  }
+
+  setActivityError(message: string) {
+    this.span.setStatus({ code: SpanStatusCode.ERROR, message });
+  }
+
+  setStreamError(message: string) {
+    this.span.setStatus({ code: SpanStatusCode.ERROR, message });
+  }
+
+  /**
+   * Adds the paths (HGET) necessary to restore telemetry state for an activity
+   * @param consumes 
+   * @param config 
+   * @param metadata 
+   * @param leg 
+   */
+  static addTargetTelemetryPaths(consumes: Consumes, config: ActivityType, metadata: ActivityMetadata, leg: number): void {
+    if (leg === 1) {
+      if (!(config.parent in consumes)) {
+        consumes[config.parent] = [];
+      }
+      consumes[config.parent].push(`${config.parent}/output/metadata/l2s`);
+    } else {
+      if (!(metadata.aid in consumes)) {
+        consumes[metadata.aid] = [];
+      }
+      consumes[metadata.aid].push(`${metadata.aid}/output/metadata/l1s`);
+    }
+  }
+
+  static bindJobTelemetryToState(state: StringStringType, config: ActivityType, context:JobState) {
+    if (config.type === 'trigger') {
+      state['metadata/trc'] = context.metadata.trc;
+    }
+  }
+
+  static bindActivityTelemetryToState(state: StringAnyType, config: ActivityType, metadata: ActivityMetadata, context: JobState, leg: number): void {
+    if (config.type === 'trigger') {
+      state[`${metadata.aid}/output/metadata/l1s`] = context['$self'].output.metadata.l1s;
+      state[`${metadata.aid}/output/metadata/l2s`] = context['$self'].output.metadata.l2s;
+    } else if (config.type === 'activity' && leg === 1) {
+      //activities run non-duplexed and only have a single leg
+      state[`${metadata.aid}/output/metadata/l1s`] = context['$self'].output.metadata.l1s;
+      state[`${metadata.aid}/output/metadata/l2s`] = context['$self'].output.metadata.l1s;
+    } else {
+      const target = `l${leg}s`;
+      state[`${metadata.aid}/output/metadata/${target}`] = context['$self'].output.metadata[target];
+    }
+  }
+}
+
+export { TelemetryService };

package/services/worker/index.ts

@@ -0,0 +1,165 @@
+import { KeyType } from "../../modules/key";
+import { ILogger } from "../logger";
+import { StreamSignaler } from "../signaler/stream";
+import { StoreService } from '../store';
+import { RedisStoreService as RedisStore } from '../store/clients/redis';
+import { IORedisStoreService as IORedisStore } from '../store/clients/ioredis';
+import { StreamService } from '../stream';
+import { RedisStreamService as RedisStream } from '../stream/clients/redis';
+import { IORedisStreamService as IORedisStream } from '../stream/clients/ioredis';
+import { SubService } from '../sub';
+import { IORedisSubService as IORedisSub } from '../sub/clients/ioredis';
+import { RedisSubService as RedisSub } from '../sub/clients/redis';
+import { RedisClientType as IORedisClientType } from '../../types/ioredisclient';
+import { HotMeshConfig, HotMeshWorker } from "../../types/hotmesh";
+import {
+  QuorumMessage,
+  SubscriptionCallback } from "../../types/quorum";
+import { RedisClient, RedisMulti } from "../../types/redis";
+import { RedisClientType } from '../../types/redisclient';
+import { StreamRole } from "../../types/stream";
+import { identifyRedisType } from "../../modules/utils";
+import { ConnectorService } from "../connector";
+
+class WorkerService {
+  namespace: string;
+  appId: string;
+  guid: string;
+  topic: string;
+  config: HotMeshConfig;
+  store: StoreService<RedisClient, RedisMulti> | null;
+  stream: StreamService<RedisClient, RedisMulti> | null;
+  subscribe: SubService<RedisClient, RedisMulti> | null;
+  streamSignaler: StreamSignaler | null;
+  logger: ILogger;
+  reporting = false;
+
+  static async init(
+    namespace: string,
+    appId: string,
+    guid: string,
+    config: HotMeshConfig,
+    logger: ILogger
+  ): Promise<WorkerService[]> {
+    const services: WorkerService[] = [];
+    if (Array.isArray(config.workers)) {
+      for (const worker of config.workers) {
+
+        await ConnectorService.initRedisClients(
+          worker.redis?.class,
+          worker.redis?.options,
+          worker,
+        );
+  
+        const service = new WorkerService();
+        service.verifyWorkerFields(worker);
+        service.namespace = namespace;
+        service.appId = appId;
+        service.guid = guid;
+        service.topic = worker.topic;
+        service.config = config;
+        service.logger = logger;
+
+        await service.initStoreChannel(service, worker.store);
+        await service.initSubChannel(service, worker.sub);
+        await service.subscribe.subscribe(KeyType.QUORUM, service.subscriptionHandler(), appId);
+        await service.subscribe.subscribe(KeyType.QUORUM, service.subscriptionHandler(), appId, service.topic);
+        await service.subscribe.subscribe(KeyType.QUORUM, service.subscriptionHandler(), appId, service.guid);
+        await service.initStreamChannel(service, worker.stream);
+        service.streamSignaler = service.initStreamSignaler(worker, logger);
+
+        const key = service.stream.mintKey(KeyType.STREAMS, { appId: service.appId, topic: worker.topic });
+        await service.streamSignaler.consumeMessages(
+          key,
+          'WORKER',
+          service.guid,
+          worker.callback
+        );
+        services.push(service);
+      }
+    }
+    return services;
+  }
+
+  verifyWorkerFields(worker: HotMeshWorker) {
+    if ((!identifyRedisType(worker.store) || 
+      !identifyRedisType(worker.stream)||
+      !identifyRedisType(worker.sub)) ||
+      !(worker.topic && worker.callback)) {
+      throw new Error('worker must include `store`, `stream`, and `sub` fields along with a callback function and topic.');
+    }
+  }
+
+  async initStoreChannel(service: WorkerService, store: RedisClient) {
+    if (identifyRedisType(store) === 'redis') {
+      service.store = new RedisStore(store as RedisClientType);
+    } else {
+      service.store = new IORedisStore(store as IORedisClientType);
+    }
+    await service.store.init(
+      service.namespace,
+      service.appId,
+      service.logger
+    );
+  }
+
+  async initSubChannel(service: WorkerService, sub: RedisClient) {
+    if (identifyRedisType(sub) === 'redis') {
+      service.subscribe = new RedisSub(sub as RedisClientType);
+    } else {
+      service.subscribe = new IORedisSub(sub as IORedisClientType);
+    }
+    await service.subscribe.init(
+      service.namespace,
+      service.appId,
+      service.guid,
+      service.logger
+    );
+  }
+
+  async initStreamChannel(service: WorkerService, stream: RedisClient) {
+    if (identifyRedisType(stream) === 'redis') {
+      service.stream = new RedisStream(stream as RedisClientType);
+    } else {
+      service.stream = new IORedisStream(stream as IORedisClientType);
+    }
+    await service.stream.init(
+      service.namespace,
+      service.appId,
+      service.logger
+    );
+  }
+
+  initStreamSignaler(worker: HotMeshWorker, logger: ILogger): StreamSignaler {
+    return new StreamSignaler(
+      {
+        namespace: this.namespace,
+        appId: this.appId,
+        guid: this.guid,
+        role: StreamRole.WORKER,
+        topic: worker.topic,
+        reclaimDelay: worker.reclaimDelay,
+        reclaimCount: worker.reclaimCount,
+      },
+      this.stream,
+      this.store,
+      logger,
+    );
+  }
+
+  subscriptionHandler(): SubscriptionCallback {
+    const self = this;
+    return async (topic: string, message: QuorumMessage) => {
+      self.logger.debug('worker-event-received', { topic, type: message.type });
+      if (message.type === 'throttle') {
+        self.throttle(message.throttle);
+      }
+    };
+  }
+
+  async throttle(delayInMillis: number) {
+    this.streamSignaler.setThrottle(delayInMillis);
+  }
+}
+
+export { WorkerService };

package/types/activity.ts

@@ -0,0 +1,115 @@
+import { MetricTypes } from "./stats";
+import { StreamRetryPolicy } from "./stream";
+
+type ActivityExecutionType = 'trigger' | 'await' | 'worker' | 'activity' | 'emit' | 'iterate';
+
+type Consumes = Record<string, string[]>;
+
+interface BaseActivity {
+  title?: string;
+  type?: ActivityExecutionType;
+  subtype?: string;
+  input?: Record<string, any>;
+  output?: Record<string, any>;
+  settings?: Record<string, any>;
+  job?: Record<string, any>;
+  hook?: Record<string, any>;
+  telemetry?: Record<string, any>;
+  sleep?: number;                      //@pipe /in seconds
+  expire?: number;                     //-1 forever (15 seconds default); todo: make globally configurable
+  retry?: StreamRetryPolicy
+  collationInt?: number;               //compiler
+  consumes?: Consumes;                 //compiler
+  PRODUCES?: string[];                 //compiler
+  produces?: string[];                 //compiler
+  publishes?: string;                  //compiler 
+  subscribes?: string;                 //compiler
+  trigger?: string;                    //compiler
+  parent?: string;                     //compiler
+}
+
+interface Measure {
+  measure: MetricTypes;
+  target: string;
+}
+
+interface TriggerActivityStats {
+  id?: { [key: string]: unknown } | string;
+  key?: { [key: string]: unknown } | string;
+  measures?: Measure[]; //what to capture
+}
+
+interface TriggerActivity extends BaseActivity {
+  type: 'trigger';
+  stats?: TriggerActivityStats;
+}
+
+interface AwaitActivity extends BaseActivity {
+  type: 'await';
+  eventName: string;
+  timeout: number;
+}
+
+interface WorkerActivity extends BaseActivity {
+  type: 'worker';
+  topic: string;
+  timeout: number;
+}
+
+interface EmitActivity extends BaseActivity {
+  type: 'emit';
+}
+
+interface IterateActivity extends BaseActivity {
+  type: 'iterate';
+}
+
+type ActivityType = BaseActivity | TriggerActivity | AwaitActivity | WorkerActivity | EmitActivity | IterateActivity;
+
+type ActivityData = Record<string, any>;
+type ActivityMetadata = {
+  aid: string;  //activity_id
+  atp: string;  //activity_type
+  stp: string;  //activity_subtype
+  ac: string;   //GMT created //activity_created
+  au: string;   //GMT updated //activity_updated
+  err?: string; //stringified error json: {message: string, code: number, error?}
+  l1s?: string; //open telemetry span context (leg 1)
+  l2s?: string; //open telemetry span context (leg 2)
+  dad?: string; //dimensional address
+  as?: string;  //activity status (e.g., 889000001000001)
+};
+
+type ActivityContext = {
+  data?: ActivityData | null;
+  metadata: ActivityMetadata;
+  hook?: ActivityData
+};
+
+type ActivityDuplex = 1 | 2;
+
+type ActivityDataType = {
+  data?: Record<string, unknown>;
+  metadata?: Record<string, unknown>;
+  hook?: Record<string, unknown>;
+};
+
+type ActivityLeg = 1 | 2;
+
+export {
+  ActivityContext,
+  ActivityData,
+  ActivityDataType,
+  ActivityDuplex,
+  ActivityLeg,
+  ActivityMetadata,
+  ActivityType,
+  Consumes,
+  TriggerActivityStats,
+  AwaitActivity,
+  BaseActivity,
+  EmitActivity,
+  IterateActivity,
+  TriggerActivity,
+  WorkerActivity
+};

package/types/app.ts

@@ -0,0 +1,20 @@
+interface App {
+  name: string;
+  title: string;
+  description: string;
+}
+
+type AppVID = {
+  version: string;
+  id: string;
+};
+
+type AppTransitions = {
+  [key: string]: Record<string, unknown>;
+};
+
+type AppSubscriptions = {
+  [key: string]: string;
+};
+
+export { App, AppVID, AppTransitions, AppSubscriptions };

package/types/async.ts

@@ -0,0 +1,7 @@
+
+type AsyncSignal = {
+  topic: string,
+  jobId: string
+};
+
+export { AsyncSignal };

package/types/cache.ts

@@ -0,0 +1 @@
+export type CacheMode = 'nocache' | 'cache';

package/types/collator.ts

@@ -0,0 +1,9 @@
+export type CollationStage = 'enter' | 'exit' | 'confirm';
+
+export enum CollationFaultType {
+  MISSING = 'missing',      //`as` uninitialized; leg1 entry not allowed
+  DUPLICATE = 'duplicate',  //1st digit < 8
+  INACTIVE = 'inactive',    //3rd digit is 8
+  INVALID = 'invalid',      //unknown value (corrupt for unknown reasons)
+  FORBIDDEN = 'forbidden',  //leg 1 not completed; reentry (leg 2) not allowed
+}

package/types/durable.ts

@@ -0,0 +1,81 @@
+import { RedisClass, RedisOptions } from './redis';
+
+type WorkflowOptions = {
+  taskQueue: string;
+  args: any[];          //input arguments to pass in
+  workflowId: string;   //execution id (the job id)
+  workflowName?: string; //the name of the user's workflow function
+  workflowTrace?: string;
+  workflowSpan?: string;
+}
+
+type ActivityDataType = {
+  activityName: string;
+  arguments: any[];
+  workflowId: string;
+  workflowTopic: string;
+}
+
+type WorkflowDataType = {
+  arguments: any[];
+  workflowId: string;
+  workflowTopic: string;
+}
+
+type ConnectionConfig = {
+  class: RedisClass;
+  options: RedisOptions;
+}
+type Connection =  ConnectionConfig;
+type NativeConnection =  ConnectionConfig;
+
+type ClientConfig = {
+  connection: Connection;
+}
+
+type Registry  = {
+  [key: string]: Function
+};
+
+type WorkerConfig = {
+  connection: Connection;
+  namespace: string; //`appid` in the YAML (e.g, 'default')
+  taskQueue: string; //`subscribes` in the YAML (e.g, 'hello-world')
+  workflowsPath: string; //resolved abs path to dyn import()
+  activities: { [key: string]: Function }; //vanilla activity exports
+}
+
+type ContextType = {
+  workflowId: string
+  workflowTopic: string
+};
+
+type FunctionSignature<T> = T extends (...args: infer A) => infer R ? (...args: A) => R : never;
+type ProxyType<ACT> = {
+  [K in keyof ACT]: FunctionSignature<ACT[K]>;
+};
+
+type ActivityConfig = {
+  startToCloseTimeout: string;
+  retryPolicy: {
+    initialInterval: string;
+    maximumAttempts: number;
+    backoffCoefficient: number;
+    maximumInterval: string;
+  };
+};
+
+export {
+  ActivityConfig,
+  ActivityDataType,
+  ClientConfig,
+  ContextType,
+  ConnectionConfig,
+  Connection,
+  NativeConnection,
+  ProxyType,
+  Registry,
+  WorkerConfig,
+  WorkflowDataType,
+  WorkflowOptions,
+};

package/types/hook.ts

@@ -0,0 +1,32 @@
+
+interface HookCondition {
+  expected: string;
+  actual: string;
+}
+
+enum HookGate {
+  AND = 'and',
+  OR = 'or',
+}
+
+interface HookConditions {
+  gate?: HookGate;
+  match: HookCondition[];
+}
+
+interface HookRule {
+  to: string;
+  conditions: HookConditions;
+}
+
+interface HookRules {
+  [eventName: string]: HookRule[];
+}
+
+type HookSignal = { topic: string, resolved: string, jobId: string};
+
+interface HookInterface {
+  (topic: string, data: { [key: string]: any, id: string }): Promise<void>;
+}
+
+export { HookCondition, HookConditions, HookGate, HookInterface, HookRule, HookRules, HookSignal };