@hotmeshio/hotmesh 0.0.3 → 0.0.5

package/build/esm/types/index.d.ts

@@ -0,0 +1,20 @@
+export { ActivityType, ActivityDataType, ActivityContext, ActivityData, ActivityDuplex, ActivityLeg, ActivityMetadata, Consumes, AwaitActivity, BaseActivity, EmitActivity, WorkerActivity, IterateActivity, TriggerActivity, TriggerActivityStats } from './activity';
+export { App, AppVID, AppTransitions, AppSubscriptions } from './app';
+export { AsyncSignal } from './async';
+export { CacheMode } from './cache';
+export { CollationFaultType, CollationStage } from './collator';
+export { HookCondition, HookConditions, HookGate, HookInterface, HookRule, HookRules, HookSignal } from './hook';
+export { RedisClientType as IORedisClientType, RedisMultiType as IORedisMultiType } from './ioredisclient';
+export { ILogger } from './logger';
+export { JobData, JobsData, JobMetadata, JobOutput, JobState, JobStatus, PartialJobState } from './job';
+export { MappingStatements } from './map';
+export { Pipe, PipeItem, PipeItems } from './pipe';
+export { HotMesh, HotMeshApp, HotMeshApps, HotMeshConfig, HotMeshEngine, RedisConfig, HotMeshGraph, HotMeshManifest, HotMeshSettings, HotMeshWorker } from './hotmesh';
+export { ActivateMessage, JobMessage, JobMessageCallback, PingMessage, PongMessage, QuorumMessage, SubscriptionCallback, ThrottleMessage, WorkMessage } from './quorum';
+export { MultiResponseFlags, RedisClient, RedisMulti } from './redis';
+export { RedisClientType, RedisMultiType } from './redisclient';
+export { JSONSchema, StringAnyType, StringScalarType, StringStringType, SymbolMap, SymbolMaps, SymbolRanges, Symbols, SymbolSets } from './serializer';
+export { AggregatedData, CountByFacet, GetStatsOptions, IdsData, Measure, MeasureIds, MetricTypes, StatType, StatsType, IdsResponse, JobStats, JobStatsInput, JobStatsRange, StatsResponse, Segment, TimeSegment } from './stats';
+export { ReclaimedMessageType, StreamCode, StreamConfig, StreamData, StreamDataType, StreamError, StreamDataResponse, StreamRetryPolicy, StreamRole, StreamStatus } from './stream';
+export { context, Context, Counter, Meter, metrics, propagation, SpanContext, Span, SpanStatus, SpanStatusCode, SpanKind, trace, Tracer, ValueType } from './telemetry';
+export { TransitionMatch, TransitionRule, Transitions } from './transition';

package/build/esm/types/index.js

@@ -0,0 +1,5 @@
+export { CollationFaultType } from './collator';
+export { HookGate } from './hook';
+export { RedisClientType as IORedisClientType } from './ioredisclient';
+export { StreamDataType, StreamRole, StreamStatus } from './stream';
+export { context, metrics, propagation, SpanStatusCode, SpanKind, trace, ValueType } from './telemetry';

package/build/esm/types/ioredisclient.d.ts

@@ -0,0 +1,5 @@
+import { Redis, ChainableCommander, RedisOptions } from 'ioredis';
+type RedisClassType = {
+    new (options: RedisOptions): Redis;
+};
+export { RedisClassType, Redis as RedisClientType, ChainableCommander as RedisMultiType, RedisOptions as RedisClientOptions, };

package/build/esm/types/ioredisclient.js

@@ -0,0 +1,2 @@
+import { Redis } from 'ioredis';
+export { Redis as RedisClientType, };

package/build/esm/types/job.d.ts

@@ -0,0 +1,50 @@
+type JobData = Record<string, unknown | Record<string, unknown>>;
+type JobsData = Record<string, unknown>;
+type ActivityData = {
+    data: Record<string, unknown>;
+    metadata?: Record<string, unknown>;
+};
+type JobMetadata = {
+    key?: string;
+    jid: string;
+    dad: string;
+    aid: string;
+    pj?: string;
+    pd?: string;
+    pa?: string;
+    ngn?: string;
+    app: string;
+    vrs: string;
+    tpc: string;
+    ts: string;
+    jc: string;
+    ju: string;
+    js: JobStatus;
+    atp: string;
+    stp: string;
+    spn: string;
+    trc: string;
+    err?: string;
+    expire?: number;
+};
+type JobStatus = number;
+type JobState = {
+    metadata: JobMetadata;
+    data: JobData;
+    [activityId: symbol]: {
+        input: ActivityData;
+        output: ActivityData;
+        hook: ActivityData;
+        settings: ActivityData;
+        errors: ActivityData;
+    };
+};
+type JobOutput = {
+    metadata: JobMetadata;
+    data: JobData;
+};
+type PartialJobState = {
+    metadata: JobMetadata | Pick<JobMetadata, 'jid' | 'dad' | 'aid'>;
+    data: JobData;
+};
+export { JobState, JobStatus, JobData, JobsData, JobMetadata, PartialJobState, JobOutput };

package/build/esm/types/job.js

@@ -0,0 +1 @@
+export {};

package/build/esm/types/logger.d.ts

@@ -0,0 +1,6 @@
+export interface ILogger {
+    info(message: string, ...meta: any[]): void;
+    error(message: string, ...meta: any[]): void;
+    warn(message: string, ...meta: any[]): void;
+    debug(message: string, ...meta: any[]): void;
+}

package/build/esm/types/logger.js

@@ -0,0 +1 @@
+export {};

package/build/esm/types/map.d.ts

@@ -0,0 +1,4 @@
+type MappingStatements = {
+    [activityId: string]: string[];
+};
+export { MappingStatements };

package/build/esm/types/map.js

@@ -0,0 +1 @@
+export {};

package/build/esm/types/pipe.d.ts

@@ -0,0 +1,4 @@
+type PipeItem = string | boolean | number | null;
+type PipeItems = PipeItem[];
+type Pipe = (PipeItem[] | Pipe)[];
+export { Pipe, PipeItem, PipeItems };

package/build/esm/types/pipe.js

@@ -0,0 +1 @@
+export {};

package/build/esm/types/quorum.d.ts

@@ -0,0 +1,46 @@
+import { JobOutput } from "./job";
+/**
+ * The types in this file are used to define those messages that are sent
+ * to hotmesh client instances when a new version is about to be activated.
+ * These messages serve to coordinate the cache invalidation and switch-over
+ * to the new version without any downtime and a coordinating parent server.
+ */
+export type QuorumMessage = PingMessage | PongMessage | ActivateMessage | WorkMessage | JobMessage | ThrottleMessage;
+export interface PingMessage {
+    type: 'ping';
+    originator: string;
+}
+export interface WorkMessage {
+    type: 'work';
+    originator: string;
+}
+export interface PongMessage {
+    type: 'pong';
+    originator: string;
+    guid: string;
+}
+export interface ActivateMessage {
+    type: 'activate';
+    cache_mode: 'nocache' | 'cache';
+    until_version: string;
+}
+export interface JobMessage {
+    type: 'job';
+    topic: string;
+    job: JobOutput;
+}
+export interface ThrottleMessage {
+    type: 'throttle';
+    guid?: string;
+    topic?: string;
+    throttle: number;
+}
+export interface JobMessageCallback {
+    (topic: string, message: JobOutput): void;
+}
+export interface SubscriptionCallback {
+    (topic: string, message: Record<string, any>): void;
+}
+export interface QuorumMessageCallback {
+    (topic: string, message: QuorumMessage): void;
+}

package/build/esm/types/quorum.js

@@ -0,0 +1 @@
+export {};

package/build/esm/types/redis.d.ts

@@ -0,0 +1,8 @@
+import { RedisClassType, RedisClientType as RCT, RedisMultiType as RMT, RedisClientOptions as RCO } from './redisclient';
+import { RedisClassType as IORRedisClassType, RedisClientType as IORCT, RedisMultiType as IORMT, RedisClientOptions as IORCO } from './ioredisclient';
+type RedisClass = RedisClassType | IORRedisClassType;
+type RedisClient = RCT | IORCT;
+type RedisMulti = RMT | IORMT;
+type RedisOptions = RCO | IORCO;
+type MultiResponseFlags = (string | number)[];
+export { RedisClass, RedisClient, RedisMulti, RedisOptions, MultiResponseFlags, };

package/build/esm/types/redis.js

@@ -0,0 +1 @@
+export {};

package/build/esm/types/redisclient.d.ts

@@ -0,0 +1,25 @@
+import { createClient, RedisClientOptions } from 'redis';
+type RedisClientType = ReturnType<typeof createClient>;
+interface RedisMultiType {
+    XADD(key: string, id: string, fields: any): this;
+    XACK(key: string, group: string, id: string): this;
+    XDEL(key: string, id: string): this;
+    HDEL(key: string, itemId: string): this;
+    HGET(key: string, itemId: string): this;
+    HGETALL(key: string): this;
+    HINCRBYFLOAT(key: string, itemId: string, value: number): this;
+    HMGET(key: string, itemIds: string[]): this;
+    HSET(key: string, values: Record<string, string>): this;
+    LRANGE(key: string, start: number, end: number): this;
+    RPUSH(key: string, value: string): this;
+    ZADD(key: string, values: {
+        score: string;
+        value: string;
+    }): this;
+    sendCommand(command: string[]): Promise<any>;
+    exec: () => Promise<unknown[]>;
+}
+type RedisClassType = {
+    createClient: (options: RedisClientOptions) => any;
+};
+export { RedisClassType, RedisClientType, RedisMultiType, RedisClientOptions, };

package/build/esm/types/redisclient.js

@@ -0,0 +1 @@
+export {};

package/build/esm/types/serializer.d.ts

@@ -0,0 +1,33 @@
+export interface JSONSchema {
+    type?: string;
+    enum?: string[];
+    examples?: any[];
+    properties?: {
+        [key: string]: JSONSchema;
+    };
+    items?: JSONSchema;
+    description?: string;
+    'x-train'?: boolean;
+}
+export type SymbolRanges = {
+    [key: string]: number;
+};
+export type Symbols = {
+    [key: string]: string;
+};
+export type SymbolSets = {
+    [key: string]: {
+        [key: string]: string;
+    };
+};
+export type StringStringType = {
+    [key: string]: string;
+};
+export type StringAnyType = {
+    [key: string]: any;
+};
+export type StringScalarType = {
+    [key: string]: boolean | number | string;
+};
+export type SymbolMap = Map<string, string>;
+export type SymbolMaps = Map<string, SymbolMap>;

package/build/esm/types/serializer.js

@@ -0,0 +1 @@
+export {};

package/build/esm/types/stats.d.ts

@@ -0,0 +1,83 @@
+type MetricTypes = 'count' | 'sum' | 'avg' | 'mdn' | 'max' | 'min' | 'index';
+interface StatType {
+    target: string;
+    metric: MetricTypes;
+    value: number | string;
+}
+interface Measure {
+    target: string;
+    type: string;
+    value: number;
+}
+interface Segment {
+    time: string;
+    count: number;
+    measures: Measure[];
+}
+interface StatsType {
+    general: StatType[];
+    index: StatType[];
+    median: StatType[];
+}
+interface JobStats {
+    count?: number;
+    [field: string]: number;
+}
+interface JobStatsRange {
+    [key: string]: JobStats;
+}
+interface JobStatsInput {
+    data: Record<string, unknown>;
+    range?: string;
+    start?: string;
+    end?: string;
+    sparse?: boolean;
+}
+interface GetStatsOptions {
+    key: string;
+    granularity: string;
+    range?: string;
+    start?: string;
+    end?: string;
+    sparse?: boolean;
+}
+interface StatsResponse {
+    key: string;
+    granularity: string;
+    range: string;
+    end: string | Date;
+    count: number;
+    measures: Measure[];
+    segments?: Segment[];
+}
+interface AggregatedData {
+    [key: string]: number;
+}
+interface IdsData {
+    [target: string]: string[];
+}
+interface MeasureIds {
+    time: string;
+    target: string;
+    count: number;
+    type: 'ids';
+    ids: string[];
+}
+interface TimeSegment {
+    time: string;
+    measures: MeasureIds[];
+}
+interface CountByFacet {
+    facet: string;
+    count: number;
+}
+interface IdsResponse {
+    key: string;
+    facets: string[];
+    granularity: string;
+    range: string;
+    start: string;
+    counts: CountByFacet[];
+    segments: TimeSegment[];
+}
+export { StatsType, StatType, MetricTypes, JobStats, JobStatsRange, JobStatsInput, GetStatsOptions, StatsResponse, AggregatedData, Measure, Segment, IdsData, MeasureIds, TimeSegment, IdsResponse, CountByFacet };

package/build/esm/types/stats.js

@@ -0,0 +1 @@
+export {};

package/build/esm/types/stream.d.ts

@@ -0,0 +1,67 @@
+export interface StreamRetryPolicy {
+    [key: string]: [number, 'x'];
+}
+export type StreamCode = number;
+export type StreamError = {
+    message: string;
+    code: number;
+    job_id?: string;
+    stack?: string;
+    name?: string;
+    error?: Record<string, unknown>;
+};
+export declare enum StreamStatus {
+    SUCCESS = "success",
+    ERROR = "error",
+    PENDING = "pending"
+}
+export declare enum StreamDataType {
+    TIMEHOOK = "timehook",
+    WEBHOOK = "webhook",
+    AWAIT = "await",
+    RESULT = "result",
+    WORKER = "worker",
+    RESPONSE = "response",
+    TRANSITION = "transition"
+}
+export interface StreamData {
+    metadata: {
+        topic?: string;
+        jid?: string;
+        dad?: string;
+        aid: string;
+        trc?: string;
+        spn?: string;
+        try?: number;
+    };
+    type?: StreamDataType;
+    data: Record<string, unknown>;
+    policies?: {
+        retry?: StreamRetryPolicy;
+    };
+    status?: StreamStatus;
+    code?: number;
+}
+export interface StreamDataResponse extends StreamData {
+}
+export declare enum StreamRole {
+    WORKER = "worker",
+    ENGINE = "engine",
+    SYSTEM = "system"
+}
+export type ReclaimedMessageType = [
+    messageId: string,
+    details: [
+        key: string,
+        value: string
+    ]
+][];
+export type StreamConfig = {
+    namespace: string;
+    appId: string;
+    guid: string;
+    role: StreamRole;
+    topic?: string;
+    reclaimDelay?: number;
+    reclaimCount?: number;
+};

package/build/esm/types/stream.js

@@ -0,0 +1,22 @@
+export var StreamStatus;
+(function (StreamStatus) {
+    StreamStatus["SUCCESS"] = "success";
+    StreamStatus["ERROR"] = "error";
+    StreamStatus["PENDING"] = "pending";
+})(StreamStatus || (StreamStatus = {}));
+export var StreamDataType;
+(function (StreamDataType) {
+    StreamDataType["TIMEHOOK"] = "timehook";
+    StreamDataType["WEBHOOK"] = "webhook";
+    StreamDataType["AWAIT"] = "await";
+    StreamDataType["RESULT"] = "result";
+    StreamDataType["WORKER"] = "worker";
+    StreamDataType["RESPONSE"] = "response";
+    StreamDataType["TRANSITION"] = "transition";
+})(StreamDataType || (StreamDataType = {}));
+export var StreamRole;
+(function (StreamRole) {
+    StreamRole["WORKER"] = "worker";
+    StreamRole["ENGINE"] = "engine";
+    StreamRole["SYSTEM"] = "system";
+})(StreamRole || (StreamRole = {}));

package/build/esm/types/telemetry.d.ts

@@ -0,0 +1 @@
+export { context, Context, Counter, Meter, metrics, propagation, SpanContext, Span, SpanStatus, SpanStatusCode, SpanKind, trace, Tracer, ValueType } from '@opentelemetry/api';

package/build/esm/types/telemetry.js

@@ -0,0 +1 @@
+export { context, metrics, propagation, SpanStatusCode, SpanKind, trace, ValueType } from '@opentelemetry/api';

package/build/esm/types/transition.d.ts

@@ -0,0 +1,17 @@
+import { Pipe } from "./pipe";
+export type TransitionMatch = {
+    expected: boolean | string | number | null;
+    actual: boolean | string | number | null | {
+        '@pipe': Pipe;
+    };
+};
+export type TransitionRule = {
+    gate?: 'and' | 'or';
+    code?: string;
+    match: Array<TransitionMatch>;
+};
+export type Transitions = {
+    [key: string]: {
+        [key: string]: TransitionRule;
+    };
+};

package/build/esm/types/transition.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,9 +1,14 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.0.3",
+  "version": "0.0.5",
   "description": "Durable Workflows",
-  "main": "build/index.js",
-  "types": "build/index.d.ts",
+  "main": "build/cjs/index.js",
+  "module": "build/esm/index.js",
+  "types": "build/cjs/index.d.ts",
+  "exports": {
+    "import": "./build/esm/index.js",
+    "require": "./build/cjs/index.js"
+  },
   "repository": {
     "type": "git",
     "url": "https://github.com/hotmeshio/sdk-typescript.git"
@@ -14,7 +19,9 @@
   },
   "scripts": {
     "clean": "rimraf ./build",
-    "build": "tsc --build tsconfig.json",
+    "build:cjs": "tsc --build tsconfig.cjs.json",
+    "build:esm": "tsc --build tsconfig.esm.json",
+    "build": "npm run build:cjs && npm run build:esm",
     "clean-build": "npm run clean && npm run build",
     "lint": "eslint . --ext .ts",
     "lint:fix": "eslint . --fix --ext .ts",
@@ -52,7 +59,7 @@
     "@opentelemetry/api": "^1.4.1",
     "js-yaml": "^4.1.0",
     "ms": "^2.1.3",
-    "nanoid": "^3.3.6",
+    "uuid": "^9.0.1",
     "winston": "^3.8.2"
   },
   "devDependencies": {

package/services/activities/activity.ts

@@ -142,7 +142,8 @@ class Activity {
       const durationInSeconds = Pipe.resolve(this.config.sleep, this.context);
       const jobId = this.context.metadata.jid;
       const activityId = this.metadata.aid;
-      await this.engine.task.registerTimeHook(jobId, activityId, 'sleep', durationInSeconds);
+      const dId = this.metadata.dad;
+      await this.engine.task.registerTimeHook(jobId, `${activityId}${dId||''}`, 'sleep', durationInSeconds);
       return jobId;
     }
   }
@@ -206,6 +207,7 @@ class Activity {
       telemetry.setActivityAttributes(attrs);
       return jobStatus as number;
     } catch (error) {
+      console.error('this error?', error);
       this.logger.error('engine-process-hook-event-error', error);
       telemetry.setActivityError(error.message);
       throw error;
@@ -278,12 +280,11 @@ class Activity {
   }
 
   bindDimensionalAddress(state: StringAnyType) {
-    const { aid, dad } = this.metadata;
-    state[`${aid}/output/metadata/dad`] = dad;
+    const dad = this.resolveDad();
+    state[`${this.metadata.aid}/output/metadata/dad`] = dad;
   }
 
   async setState(multi?: RedisMulti): Promise<string> {
-    const { id: appId } = await this.engine.getVID();
     const jobId = this.context.metadata.jid;
     this.bindJobMetadata();
     this.bindActivityMetadata();
@@ -298,7 +299,8 @@ class Activity {
       this.metadata.aid,
       ...presets
     ];
-    return await this.store.setState(state, this.getJobStatus(), jobId, appId, symbolNames, multi);
+    const dIds = CollatorService.getDimensionsById([...this.config.ancestors, this.metadata.aid], this.resolveDad());
+    return await this.store.setState(state, this.getJobStatus(), jobId, symbolNames, dIds, multi);
   }
 
   bindJobMetadata(): void {
@@ -314,6 +316,7 @@ class Activity {
     if (this.status === StreamStatus.ERROR) {
       self.output.metadata.err = JSON.stringify(this.data);
     }
+    //todo: verify leg2 never overwrites leg1 `ac`
     self.output.metadata.ac = 
       self.output.metadata.au = formatISODate(new Date());
     self.output.metadata.atp = this.config.type;
@@ -386,10 +389,11 @@ class Activity {
       }
     }
     TelemetryService.addTargetTelemetryPaths(consumes, this.config, this.metadata, this.leg);
-    const { dad, jid } = this.context.metadata;
+    let { dad, jid } = this.context.metadata;
     jobId = jobId || jid;
     //`state` is a flat hash
-    const [state, status] = await this.store.getState(jobId, consumes);
+    const dIds = CollatorService.getDimensionsById([...this.config.ancestors, this.metadata.aid], dad);
+    const [state, status] = await this.store.getState(jobId, consumes, dIds);
     //`context` is a tree
     this.context = restoreHierarchy(state) as JobState;
     this.initDimensionalAddress(dad);
@@ -429,11 +433,26 @@ class Activity {
     this.context[this.metadata.aid][type].data = this.data;
   }
 
+  resolveDad(): string {
+    let dad = this.metadata.dad;
+    if (this.adjacentIndex > 0) {
+      //if adjacent index > 0 the activity is cycling; replace last index with cycle index
+      dad = `${dad.substring(0, dad.lastIndexOf(','))},${this.adjacentIndex}`
+    }
+    return dad;
+  }
+
+  resolveAdjacentDad(): string {
+    //concat self and child dimension (all children (leg 1) begin life at 0)
+    return `${this.resolveDad()}${DimensionService.getSeed(0)}`;
+  };
+
   async filterAdjacent(): Promise<StreamData[]> {
     const adjacencyList: StreamData[] = [];
     const transitions = await this.store.getTransitions(await this.engine.getVID());
     const transition = transitions[`.${this.metadata.aid}`];
-    const adjacentSuffix = DimensionService.getSeed(this.adjacentIndex);
+    //resolve the dimensional address for adjacent children
+    const adjacentDad = this.resolveAdjacentDad();
     if (transition) {
       for (const toActivityId in transition) {
         const transitionRule: boolean | TransitionRule = transition[toActivityId];
@@ -441,7 +460,7 @@ class Activity {
           adjacencyList.push({
             metadata: {
               jid: this.context.metadata.jid,
-              dad: `${this.metadata.dad}${adjacentSuffix}`,
+              dad: adjacentDad,
               aid: toActivityId,
               spn: this.context['$self'].output.metadata?.l2s,
               trc: this.context.metadata.trc,

package/services/activities/trigger.ts

@@ -1,4 +1,4 @@
-import { nanoid } from 'nanoid';
+import { v4 as uuidv4 } from 'uuid';
 import { DuplicateJobError } from '../../modules/errors';
 import { formatISODate, getTimeSeries } from '../../modules/utils';
 import { Activity } from './activity';
@@ -161,7 +161,7 @@ class Trigger extends Activity {
 
   resolveJobId(context: Partial<JobState>): string {
     const jobId = this.config.stats?.id;
-    return jobId ? Pipe.resolve(jobId, context) : nanoid();
+    return jobId ? Pipe.resolve(jobId, context) : uuidv4();
   }
 
   resolveJobKey(context: Partial<JobState>): string {

package/services/activities/worker.ts

@@ -62,6 +62,7 @@ class Worker extends Activity {
       if (error instanceof GetStateError) {
         this.logger.error('worker-get-state-error', error);
       } else {
+        console.error(error);
         this.logger.error('worker-process-error', error);
       }
       telemetry.setActivityError(error.message);