@hotmeshio/hotmesh 0.0.3 → 0.0.5

package/build/esm/services/telemetry/index.js

@@ -0,0 +1,217 @@
+import packageJson from '../../package.json';
+import { MapperService } from '../mapper';
+import { StreamDataType, StreamRole } from '../../types/stream';
+import { SpanKind, trace, context, SpanStatusCode } from '../../types/telemetry';
+class TelemetryService {
+    constructor(appId, config, metadata, context) {
+        this.leg = 1;
+        this.appId = appId;
+        //these are REQUIRED for job and activity spans
+        this.config = config;
+        this.metadata = metadata;
+        this.context = context;
+    }
+    getJobParentSpanId() {
+        return this.context.metadata.spn;
+    }
+    getActivityParentSpanId(leg) {
+        if (leg === 1) {
+            return this.context[this.config.parent].output?.metadata?.l2s;
+        }
+        else {
+            return this.context['$self'].output?.metadata?.l1s;
+        }
+    }
+    getTraceId() {
+        return this.context.metadata.trc;
+    }
+    startJobSpan() {
+        const spanName = `JOB/${this.appId}/${this.config.subscribes}/1`;
+        const traceId = this.getTraceId();
+        const spanId = this.getJobParentSpanId();
+        const attributes = this.getSpanAttrs(1);
+        const span = this.startSpan(traceId, spanId, spanName, attributes);
+        this.jobSpan = span;
+        this.setTelemetryContext(span, 1);
+        return this;
+    }
+    startActivitySpan(leg = this.leg) {
+        const spanName = `${this.config.type.toUpperCase()}/${this.appId}/${this.metadata.aid}/${leg}`;
+        const traceId = this.getTraceId();
+        const spanId = this.getActivityParentSpanId(leg);
+        const attributes = this.getSpanAttrs(leg);
+        const span = this.startSpan(traceId, spanId, spanName, attributes);
+        this.setTelemetryContext(span, leg);
+        this.span = span;
+        return this;
+    }
+    startStreamSpan(data, role) {
+        let type;
+        if (role === StreamRole.SYSTEM) {
+            type = 'SYSTEM';
+        }
+        else if (role === StreamRole.WORKER) {
+            type = 'EXECUTE';
+        }
+        else if (data.type === StreamDataType.RESULT || data.type === StreamDataType.RESPONSE) {
+            type = 'FANIN';
+        }
+        else {
+            type = 'FANOUT';
+        }
+        const topic = data.metadata.topic ? `/${data.metadata.topic}` : '';
+        const spanName = `${type}/${this.appId}/${data.metadata.aid}${topic}`;
+        const attributes = this.getStreamSpanAttrs(data);
+        const span = this.startSpan(data.metadata.trc, data.metadata.spn, spanName, attributes);
+        this.span = span;
+        return this;
+    }
+    startSpan(traceId, spanId, spanName, attributes) {
+        this.traceId = traceId;
+        this.spanId = spanId;
+        const tracer = trace.getTracer(packageJson.name, packageJson.version);
+        let parentContext = this.getParentSpanContext();
+        const span = tracer.startSpan(spanName, { kind: SpanKind.CLIENT, attributes, root: !parentContext }, parentContext);
+        return span;
+    }
+    mapActivityAttributes() {
+        //export user-defined span attributes (app.activity.data.*)
+        if (this.config.telemetry) {
+            const telemetryAtts = new MapperService(this.config.telemetry, this.context).mapRules();
+            const namespacedAtts = {
+                ...Object.keys(telemetryAtts).reduce((result, key) => {
+                    if (['string', 'boolean', 'number'].includes(typeof telemetryAtts[key])) {
+                        result[`app.activity.data.${key}`] = telemetryAtts[key];
+                    }
+                    return result;
+                }, {})
+            };
+            this.span.setAttributes(namespacedAtts);
+        }
+    }
+    setActivityAttributes(attributes) {
+        this.span.setAttributes(attributes);
+    }
+    setStreamAttributes(attributes) {
+        this.span.setAttributes(attributes);
+    }
+    setJobAttributes(attributes) {
+        this.jobSpan.setAttributes(attributes);
+    }
+    endJobSpan() {
+        this.endSpan(this.jobSpan);
+    }
+    endActivitySpan() {
+        this.endSpan(this.span);
+    }
+    endStreamSpan() {
+        this.endSpan(this.span);
+    }
+    endSpan(span) {
+        span && span.end();
+    }
+    getParentSpanContext() {
+        if (this.traceId && this.spanId) {
+            const restoredSpanContext = {
+                traceId: this.traceId,
+                spanId: this.spanId,
+                isRemote: true,
+                traceFlags: 1, // (todo: revisit sampling strategy/config)
+            };
+            const parentContext = trace.setSpanContext(context.active(), restoredSpanContext);
+            return parentContext;
+        }
+    }
+    getSpanAttrs(leg) {
+        return {
+            ...Object.keys(this.context.metadata).reduce((result, key) => {
+                if (key !== 'trc') {
+                    result[`app.job.${key}`] = this.context.metadata[key];
+                }
+                return result;
+            }, {}),
+            ...Object.keys(this.metadata).reduce((result, key) => {
+                result[`app.activity.${key}`] = this.metadata[key];
+                return result;
+            }, {}),
+            'app.activity.leg': leg,
+        };
+    }
+    ;
+    getStreamSpanAttrs(input) {
+        return {
+            ...Object.keys(input.metadata).reduce((result, key) => {
+                if (key !== 'trc' && key !== 'spn') {
+                    result[`app.stream.${key}`] = input.metadata[key];
+                }
+                return result;
+            }, {})
+        };
+    }
+    ;
+    setTelemetryContext(span, leg) {
+        if (!this.context.metadata.trc) {
+            this.context.metadata.trc = span.spanContext().traceId;
+        }
+        if (leg === 1) {
+            if (!this.context['$self'].output.metadata) {
+                this.context['$self'].output.metadata = {};
+            }
+            this.context['$self'].output.metadata.l1s = span.spanContext().spanId;
+        }
+        else {
+            if (!this.context['$self'].output.metadata) {
+                this.context['$self'].output.metadata = {};
+            }
+            this.context['$self'].output.metadata.l2s = span.spanContext().spanId;
+        }
+    }
+    setActivityError(message) {
+        this.span.setStatus({ code: SpanStatusCode.ERROR, message });
+    }
+    setStreamError(message) {
+        this.span.setStatus({ code: SpanStatusCode.ERROR, message });
+    }
+    /**
+     * Adds the paths (HGET) necessary to restore telemetry state for an activity
+     * @param consumes
+     * @param config
+     * @param metadata
+     * @param leg
+     */
+    static addTargetTelemetryPaths(consumes, config, metadata, leg) {
+        if (leg === 1) {
+            if (!(config.parent in consumes)) {
+                consumes[config.parent] = [];
+            }
+            consumes[config.parent].push(`${config.parent}/output/metadata/l2s`);
+        }
+        else {
+            if (!(metadata.aid in consumes)) {
+                consumes[metadata.aid] = [];
+            }
+            consumes[metadata.aid].push(`${metadata.aid}/output/metadata/l1s`);
+        }
+    }
+    static bindJobTelemetryToState(state, config, context) {
+        if (config.type === 'trigger') {
+            state['metadata/trc'] = context.metadata.trc;
+        }
+    }
+    static bindActivityTelemetryToState(state, config, metadata, context, leg) {
+        if (config.type === 'trigger') {
+            state[`${metadata.aid}/output/metadata/l1s`] = context['$self'].output.metadata.l1s;
+            state[`${metadata.aid}/output/metadata/l2s`] = context['$self'].output.metadata.l2s;
+        }
+        else if (config.type === 'activity' && leg === 1) {
+            //activities run non-duplexed and only have a single leg
+            state[`${metadata.aid}/output/metadata/l1s`] = context['$self'].output.metadata.l1s;
+            state[`${metadata.aid}/output/metadata/l2s`] = context['$self'].output.metadata.l1s;
+        }
+        else {
+            const target = `l${leg}s`;
+            state[`${metadata.aid}/output/metadata/${target}`] = context['$self'].output.metadata[target];
+        }
+    }
+}
+export { TelemetryService };

package/build/esm/services/worker/index.d.ts

@@ -0,0 +1,30 @@
+import { ILogger } from "../logger";
+import { StreamSignaler } from "../signaler/stream";
+import { StoreService } from '../store';
+import { StreamService } from '../stream';
+import { SubService } from '../sub';
+import { HotMeshConfig, HotMeshWorker } from "../../types/hotmesh";
+import { SubscriptionCallback } from "../../types/quorum";
+import { RedisClient, RedisMulti } from "../../types/redis";
+declare class WorkerService {
+    namespace: string;
+    appId: string;
+    guid: string;
+    topic: string;
+    config: HotMeshConfig;
+    store: StoreService<RedisClient, RedisMulti> | null;
+    stream: StreamService<RedisClient, RedisMulti> | null;
+    subscribe: SubService<RedisClient, RedisMulti> | null;
+    streamSignaler: StreamSignaler | null;
+    logger: ILogger;
+    reporting: boolean;
+    static init(namespace: string, appId: string, guid: string, config: HotMeshConfig, logger: ILogger): Promise<WorkerService[]>;
+    verifyWorkerFields(worker: HotMeshWorker): void;
+    initStoreChannel(service: WorkerService, store: RedisClient): Promise<void>;
+    initSubChannel(service: WorkerService, sub: RedisClient): Promise<void>;
+    initStreamChannel(service: WorkerService, stream: RedisClient): Promise<void>;
+    initStreamSignaler(worker: HotMeshWorker, logger: ILogger): StreamSignaler;
+    subscriptionHandler(): SubscriptionCallback;
+    throttle(delayInMillis: number): Promise<void>;
+}
+export { WorkerService };

package/build/esm/services/worker/index.js

@@ -0,0 +1,102 @@
+import { KeyType } from "../../modules/key";
+import { StreamSignaler } from "../signaler/stream";
+import { RedisStoreService as RedisStore } from '../store/clients/redis';
+import { IORedisStoreService as IORedisStore } from '../store/clients/ioredis';
+import { RedisStreamService as RedisStream } from '../stream/clients/redis';
+import { IORedisStreamService as IORedisStream } from '../stream/clients/ioredis';
+import { IORedisSubService as IORedisSub } from '../sub/clients/ioredis';
+import { RedisSubService as RedisSub } from '../sub/clients/redis';
+import { StreamRole } from "../../types/stream";
+import { identifyRedisType } from "../../modules/utils";
+import { ConnectorService } from "../connector";
+class WorkerService {
+    constructor() {
+        this.reporting = false;
+    }
+    static async init(namespace, appId, guid, config, logger) {
+        const services = [];
+        if (Array.isArray(config.workers)) {
+            for (const worker of config.workers) {
+                await ConnectorService.initRedisClients(worker.redis?.class, worker.redis?.options, worker);
+                const service = new WorkerService();
+                service.verifyWorkerFields(worker);
+                service.namespace = namespace;
+                service.appId = appId;
+                service.guid = guid;
+                service.topic = worker.topic;
+                service.config = config;
+                service.logger = logger;
+                await service.initStoreChannel(service, worker.store);
+                await service.initSubChannel(service, worker.sub);
+                await service.subscribe.subscribe(KeyType.QUORUM, service.subscriptionHandler(), appId);
+                await service.subscribe.subscribe(KeyType.QUORUM, service.subscriptionHandler(), appId, service.topic);
+                await service.subscribe.subscribe(KeyType.QUORUM, service.subscriptionHandler(), appId, service.guid);
+                await service.initStreamChannel(service, worker.stream);
+                service.streamSignaler = service.initStreamSignaler(worker, logger);
+                const key = service.stream.mintKey(KeyType.STREAMS, { appId: service.appId, topic: worker.topic });
+                await service.streamSignaler.consumeMessages(key, 'WORKER', service.guid, worker.callback);
+                services.push(service);
+            }
+        }
+        return services;
+    }
+    verifyWorkerFields(worker) {
+        if ((!identifyRedisType(worker.store) ||
+            !identifyRedisType(worker.stream) ||
+            !identifyRedisType(worker.sub)) ||
+            !(worker.topic && worker.callback)) {
+            throw new Error('worker must include `store`, `stream`, and `sub` fields along with a callback function and topic.');
+        }
+    }
+    async initStoreChannel(service, store) {
+        if (identifyRedisType(store) === 'redis') {
+            service.store = new RedisStore(store);
+        }
+        else {
+            service.store = new IORedisStore(store);
+        }
+        await service.store.init(service.namespace, service.appId, service.logger);
+    }
+    async initSubChannel(service, sub) {
+        if (identifyRedisType(sub) === 'redis') {
+            service.subscribe = new RedisSub(sub);
+        }
+        else {
+            service.subscribe = new IORedisSub(sub);
+        }
+        await service.subscribe.init(service.namespace, service.appId, service.guid, service.logger);
+    }
+    async initStreamChannel(service, stream) {
+        if (identifyRedisType(stream) === 'redis') {
+            service.stream = new RedisStream(stream);
+        }
+        else {
+            service.stream = new IORedisStream(stream);
+        }
+        await service.stream.init(service.namespace, service.appId, service.logger);
+    }
+    initStreamSignaler(worker, logger) {
+        return new StreamSignaler({
+            namespace: this.namespace,
+            appId: this.appId,
+            guid: this.guid,
+            role: StreamRole.WORKER,
+            topic: worker.topic,
+            reclaimDelay: worker.reclaimDelay,
+            reclaimCount: worker.reclaimCount,
+        }, this.stream, this.store, logger);
+    }
+    subscriptionHandler() {
+        const self = this;
+        return async (topic, message) => {
+            self.logger.debug('worker-event-received', { topic, type: message.type });
+            if (message.type === 'throttle') {
+                self.throttle(message.throttle);
+            }
+        };
+    }
+    async throttle(delayInMillis) {
+        this.streamSignaler.setThrottle(delayInMillis);
+    }
+}
+export { WorkerService };

package/build/esm/types/activity.d.ts

@@ -0,0 +1,87 @@
+import { MetricTypes } from "./stats";
+import { StreamRetryPolicy } from "./stream";
+type ActivityExecutionType = 'trigger' | 'await' | 'worker' | 'activity' | 'emit' | 'iterate';
+type Consumes = Record<string, string[]>;
+interface BaseActivity {
+    title?: string;
+    type?: ActivityExecutionType;
+    subtype?: string;
+    input?: Record<string, any>;
+    output?: Record<string, any>;
+    settings?: Record<string, any>;
+    job?: Record<string, any>;
+    hook?: Record<string, any>;
+    telemetry?: Record<string, any>;
+    sleep?: number;
+    expire?: number;
+    retry?: StreamRetryPolicy;
+    collationInt?: number;
+    consumes?: Consumes;
+    PRODUCES?: string[];
+    produces?: string[];
+    publishes?: string;
+    subscribes?: string;
+    trigger?: string;
+    parent?: string;
+    ancestors?: string[];
+}
+interface Measure {
+    measure: MetricTypes;
+    target: string;
+}
+interface TriggerActivityStats {
+    id?: {
+        [key: string]: unknown;
+    } | string;
+    key?: {
+        [key: string]: unknown;
+    } | string;
+    measures?: Measure[];
+}
+interface TriggerActivity extends BaseActivity {
+    type: 'trigger';
+    stats?: TriggerActivityStats;
+}
+interface AwaitActivity extends BaseActivity {
+    type: 'await';
+    eventName: string;
+    timeout: number;
+}
+interface WorkerActivity extends BaseActivity {
+    type: 'worker';
+    topic: string;
+    timeout: number;
+}
+interface EmitActivity extends BaseActivity {
+    type: 'emit';
+}
+interface IterateActivity extends BaseActivity {
+    type: 'iterate';
+}
+type ActivityType = BaseActivity | TriggerActivity | AwaitActivity | WorkerActivity | EmitActivity | IterateActivity;
+type ActivityData = Record<string, any>;
+type ActivityMetadata = {
+    aid: string;
+    atp: string;
+    stp: string;
+    ac: string;
+    au: string;
+    err?: string;
+    l1s?: string;
+    l2s?: string;
+    dad?: string;
+    as?: string;
+};
+type ActivityContext = {
+    data?: ActivityData | null;
+    metadata: ActivityMetadata;
+    hook?: ActivityData;
+};
+type ActivityDuplex = 1 | 2;
+type ActivityDataType = {
+    data?: Record<string, unknown>;
+    metadata?: Record<string, unknown>;
+    hook?: Record<string, unknown>;
+};
+type ActivityLeg = 1 | 2;
+export { ActivityContext, ActivityData, ActivityDataType, ActivityDuplex, ActivityLeg, ActivityMetadata, ActivityType, Consumes, TriggerActivityStats, AwaitActivity, BaseActivity, EmitActivity, IterateActivity, TriggerActivity, WorkerActivity };

package/build/esm/types/activity.js

@@ -0,0 +1 @@
+export {};

package/build/esm/types/app.d.ts

@@ -0,0 +1,16 @@
+interface App {
+    name: string;
+    title: string;
+    description: string;
+}
+type AppVID = {
+    version: string;
+    id: string;
+};
+type AppTransitions = {
+    [key: string]: Record<string, unknown>;
+};
+type AppSubscriptions = {
+    [key: string]: string;
+};
+export { App, AppVID, AppTransitions, AppSubscriptions };

package/build/esm/types/app.js

@@ -0,0 +1 @@
+export {};

package/build/esm/types/async.d.ts

@@ -0,0 +1,5 @@
+type AsyncSignal = {
+    topic: string;
+    jobId: string;
+};
+export { AsyncSignal };

package/build/esm/types/async.js

@@ -0,0 +1 @@
+export {};

package/build/esm/types/cache.d.ts

@@ -0,0 +1 @@
+export type CacheMode = 'nocache' | 'cache';

package/build/esm/types/cache.js

@@ -0,0 +1 @@
+export {};

package/build/esm/types/collator.d.ts

@@ -0,0 +1,8 @@
+export type CollationStage = 'enter' | 'exit' | 'confirm';
+export declare enum CollationFaultType {
+    MISSING = "missing",
+    DUPLICATE = "duplicate",
+    INACTIVE = "inactive",
+    INVALID = "invalid",
+    FORBIDDEN = "forbidden"
+}

package/build/esm/types/collator.js

@@ -0,0 +1,8 @@
+export var CollationFaultType;
+(function (CollationFaultType) {
+    CollationFaultType["MISSING"] = "missing";
+    CollationFaultType["DUPLICATE"] = "duplicate";
+    CollationFaultType["INACTIVE"] = "inactive";
+    CollationFaultType["INVALID"] = "invalid";
+    CollationFaultType["FORBIDDEN"] = "forbidden";
+})(CollationFaultType || (CollationFaultType = {}));

package/build/esm/types/durable.d.ts

@@ -0,0 +1,59 @@
+import { RedisClass, RedisOptions } from './redis';
+type WorkflowOptions = {
+    taskQueue: string;
+    args: any[];
+    workflowId: string;
+    workflowName?: string;
+    workflowTrace?: string;
+    workflowSpan?: string;
+};
+type ActivityDataType = {
+    activityName: string;
+    arguments: any[];
+    workflowId: string;
+    workflowTopic: string;
+};
+type WorkflowDataType = {
+    arguments: any[];
+    workflowId: string;
+    workflowTopic: string;
+};
+type ConnectionConfig = {
+    class: RedisClass;
+    options: RedisOptions;
+};
+type Connection = ConnectionConfig;
+type NativeConnection = ConnectionConfig;
+type ClientConfig = {
+    connection: Connection;
+};
+type Registry = {
+    [key: string]: Function;
+};
+type WorkerConfig = {
+    connection: Connection;
+    namespace: string;
+    taskQueue: string;
+    workflowsPath: string;
+    activities: {
+        [key: string]: Function;
+    };
+};
+type ContextType = {
+    workflowId: string;
+    workflowTopic: string;
+};
+type FunctionSignature<T> = T extends (...args: infer A) => infer R ? (...args: A) => R : never;
+type ProxyType<ACT> = {
+    [K in keyof ACT]: FunctionSignature<ACT[K]>;
+};
+type ActivityConfig = {
+    startToCloseTimeout: string;
+    retryPolicy: {
+        initialInterval: string;
+        maximumAttempts: number;
+        backoffCoefficient: number;
+        maximumInterval: string;
+    };
+};
+export { ActivityConfig, ActivityDataType, ClientConfig, ContextType, ConnectionConfig, Connection, NativeConnection, ProxyType, Registry, WorkerConfig, WorkflowDataType, WorkflowOptions, };

package/build/esm/types/durable.js

@@ -0,0 +1 @@
+export {};

package/build/esm/types/hook.d.ts

@@ -0,0 +1,31 @@
+interface HookCondition {
+    expected: string;
+    actual: string;
+}
+declare enum HookGate {
+    AND = "and",
+    OR = "or"
+}
+interface HookConditions {
+    gate?: HookGate;
+    match: HookCondition[];
+}
+interface HookRule {
+    to: string;
+    conditions: HookConditions;
+}
+interface HookRules {
+    [eventName: string]: HookRule[];
+}
+type HookSignal = {
+    topic: string;
+    resolved: string;
+    jobId: string;
+};
+interface HookInterface {
+    (topic: string, data: {
+        [key: string]: any;
+        id: string;
+    }): Promise<void>;
+}
+export { HookCondition, HookConditions, HookGate, HookInterface, HookRule, HookRules, HookSignal };

package/build/esm/types/hook.js

@@ -0,0 +1,6 @@
+var HookGate;
+(function (HookGate) {
+    HookGate["AND"] = "and";
+    HookGate["OR"] = "or";
+})(HookGate || (HookGate = {}));
+export { HookGate };

package/build/esm/types/hotmesh.d.ts

@@ -0,0 +1,82 @@
+import { ILogger } from '../services/logger';
+import { HotMeshService } from '../services/hotmesh';
+import { HookRules } from './hook';
+import { RedisClass, RedisClient, RedisOptions } from './redis';
+import { StreamData, StreamDataResponse } from './stream';
+type HotMesh = typeof HotMeshService;
+type RedisConfig = {
+    class: RedisClass;
+    options: RedisOptions;
+};
+type HotMeshEngine = {
+    store?: RedisClient;
+    stream?: RedisClient;
+    sub?: RedisClient;
+    redis?: RedisConfig;
+    reclaimDelay?: number;
+    reclaimCount?: number;
+};
+type HotMeshWorker = {
+    topic: string;
+    store?: RedisClient;
+    stream?: RedisClient;
+    sub?: RedisClient;
+    redis?: {
+        class: RedisClass;
+        options: RedisOptions;
+    };
+    reclaimDelay?: number;
+    reclaimCount?: number;
+    callback: (payload: StreamData) => Promise<StreamDataResponse | void>;
+};
+type HotMeshConfig = {
+    appId: string;
+    namespace?: string;
+    name?: string;
+    logger?: ILogger;
+    logLevel?: 'silly' | 'debug' | 'info' | 'warn' | 'error' | 'silent';
+    engine?: HotMeshEngine;
+    workers?: HotMeshWorker[];
+};
+type HotMeshGraph = {
+    subscribes: string;
+    publishes?: string;
+    expire?: number;
+    output?: {
+        schema: Record<string, any>;
+    };
+    input?: {
+        schema: Record<string, any>;
+    };
+    activities: Record<string, any>;
+    transitions?: Record<string, any>;
+    hooks?: HookRules;
+};
+type HotMeshSettings = {
+    namespace: string;
+    version: string;
+};
+type HotMeshManifest = {
+    app: {
+        id: string;
+        version: string;
+        settings: Record<string, any>;
+        graphs: HotMeshGraph[];
+    };
+};
+type VersionedFields = {
+    [K in `versions/${string}`]: any;
+};
+type HotMeshApp = VersionedFields & {
+    id: string;
+    version: string;
+    settings?: string;
+    active?: boolean;
+};
+type HotMeshApps = {
+    [appId: string]: HotMeshApp;
+};
+export { HotMesh, HotMeshEngine, RedisConfig, HotMeshWorker, HotMeshSettings, HotMeshApp, //a single app in the db
+HotMeshApps, //object array of all apps in the db
+HotMeshConfig, //customer config
+HotMeshManifest, HotMeshGraph };

package/build/esm/types/hotmesh.js

@@ -0,0 +1 @@
+export {};