@hotmeshio/hotmesh 0.0.1

package/build/services/activities/trigger.js

@@ -0,0 +1,161 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Trigger = void 0;
+const nanoid_1 = require("nanoid");
+const errors_1 = require("../../modules/errors");
+const utils_1 = require("../../modules/utils");
+const activity_1 = require("./activity");
+const collator_1 = require("../collator");
+const dimension_1 = require("../dimension");
+const pipe_1 = require("../pipe");
+const reporter_1 = require("../reporter");
+const serializer_1 = require("../serializer");
+const telemetry_1 = require("../telemetry");
+class Trigger extends activity_1.Activity {
+    constructor(config, data, metadata, hook, engine, context) {
+        super(config, data, metadata, hook, engine, context);
+    }
+    async process() {
+        this.logger.debug('trigger-process', { subscribes: this.config.subscribes });
+        let telemetry;
+        try {
+            this.setLeg(2);
+            await this.getState();
+            telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
+            telemetry.startJobSpan();
+            telemetry.startActivitySpan(this.leg);
+            this.mapJobData();
+            await this.setStateNX();
+            this.adjacencyList = await this.filterAdjacent();
+            await this.setStatus(this.adjacencyList.length);
+            const multi = this.store.getMulti();
+            await this.setState(multi);
+            await this.setStats(multi);
+            await multi.exec();
+            telemetry.mapActivityAttributes();
+            const jobStatus = Number(this.context.metadata.js);
+            telemetry.setJobAttributes({ 'app.job.jss': jobStatus });
+            const attrs = { 'app.job.jss': jobStatus };
+            const messageIds = await this.transition(this.adjacencyList, jobStatus);
+            if (messageIds.length) {
+                attrs['app.activity.mids'] = messageIds.join(',');
+            }
+            telemetry.setActivityAttributes(attrs);
+            return this.context.metadata.jid;
+        }
+        catch (error) {
+            if (error instanceof errors_1.DuplicateJobError) {
+                this.logger.error('duplicate-job-error', error);
+            }
+            else {
+                this.logger.error('trigger-process-error', error);
+            }
+            telemetry.setActivityError(error.message);
+            throw error;
+        }
+        finally {
+            telemetry.endJobSpan();
+            telemetry.endActivitySpan();
+            this.logger.debug('trigger-process-end', { subscribes: this.config.subscribes, jid: this.context.metadata.jid });
+        }
+    }
+    async setStatus(amount) {
+        this.context.metadata.js = amount;
+    }
+    createInputContext() {
+        const input = {
+            [this.metadata.aid]: {
+                input: { data: this.data }
+            },
+            '$self': {
+                input: { data: this.data },
+                output: { data: this.data }
+            },
+        };
+        return input;
+    }
+    async getState() {
+        const inputContext = this.createInputContext();
+        const jobId = this.resolveJobId(inputContext);
+        const jobKey = this.resolveJobKey(inputContext);
+        const utc = (0, utils_1.formatISODate)(new Date());
+        const { id, version } = await this.engine.getVID();
+        this.initDimensionalAddress(dimension_1.DimensionService.getSeed());
+        const activityMetadata = {
+            ...this.metadata,
+            jid: jobId,
+            key: jobKey,
+            as: collator_1.CollatorService.getTriggerSeed(),
+        };
+        this.context = {
+            metadata: {
+                ...this.metadata,
+                ngn: this.context.metadata.ngn,
+                pj: this.context.metadata.pj,
+                pd: this.context.metadata.pd,
+                pa: this.context.metadata.pa,
+                app: id,
+                vrs: version,
+                tpc: this.config.subscribes,
+                trc: this.context.metadata.trc,
+                spn: this.context.metadata.spn,
+                jid: jobId,
+                dad: dimension_1.DimensionService.getSeed(),
+                key: jobKey,
+                jc: utc,
+                ju: utc,
+                ts: (0, utils_1.getTimeSeries)(this.resolveGranularity()),
+                js: 0,
+            },
+            data: {},
+            [this.metadata.aid]: {
+                input: {
+                    data: this.data,
+                    metadata: activityMetadata
+                },
+                output: {
+                    data: this.data,
+                    metadata: activityMetadata
+                },
+                settings: { data: {} },
+                errors: { data: {} },
+            },
+        };
+        this.context['$self'] = this.context[this.metadata.aid];
+    }
+    bindJobMetadataPaths() {
+        return serializer_1.MDATA_SYMBOLS.JOB.KEYS.map((key) => `metadata/${key}`);
+    }
+    bindActivityMetadataPaths() {
+        return serializer_1.MDATA_SYMBOLS.ACTIVITY.KEYS.map((key) => `output/metadata/${key}`);
+    }
+    resolveGranularity() {
+        return reporter_1.ReporterService.DEFAULT_GRANULARITY;
+    }
+    getJobStatus() {
+        return this.context.metadata.js;
+    }
+    resolveJobId(context) {
+        const jobId = this.config.stats?.id;
+        return jobId ? pipe_1.Pipe.resolve(jobId, context) : (0, nanoid_1.nanoid)();
+    }
+    resolveJobKey(context) {
+        const jobKey = this.config.stats?.key;
+        return jobKey ? pipe_1.Pipe.resolve(jobKey, context) : '';
+    }
+    async setStateNX() {
+        const jobId = this.context.metadata.jid;
+        if (!await this.store.setStateNX(jobId, this.engine.appId)) {
+            throw new errors_1.DuplicateJobError(jobId);
+        }
+    }
+    async setStats(multi) {
+        const md = this.context.metadata;
+        if (this.config.stats?.measures) {
+            const config = await this.engine.getVID();
+            const reporter = new reporter_1.ReporterService(config, this.store, this.logger);
+            await this.store.setStats(md.key, md.jid, md.ts, reporter.resolveTriggerStatistics(this.config, this.context), config, multi);
+        }
+    }
+}
+exports.Trigger = Trigger;

package/build/services/activities/worker.d.ts

@@ -0,0 +1,17 @@
+import { Activity } from './activity';
+import { EngineService } from '../engine';
+import { ActivityData, ActivityMetadata, ActivityType, WorkerActivity } from '../../types/activity';
+import { JobState } from '../../types/job';
+import { StreamCode, StreamStatus } from '../../types/stream';
+import { TelemetryService } from '../telemetry';
+declare class Worker extends Activity {
+    config: WorkerActivity;
+    constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);
+    process(): Promise<string>;
+    execActivity(): Promise<string>;
+    processEvent(status?: StreamStatus, code?: StreamCode): Promise<void>;
+    processPending(telemetry: TelemetryService): Promise<void>;
+    processSuccess(telemetry: TelemetryService): Promise<void>;
+    processError(telemetry: TelemetryService): Promise<void>;
+}
+export { Worker };

package/build/services/activities/worker.js

@@ -0,0 +1,164 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Worker = void 0;
+const errors_1 = require("../../modules/errors");
+const activity_1 = require("./activity");
+const collator_1 = require("../collator");
+const stream_1 = require("../../types/stream");
+const telemetry_1 = require("../telemetry");
+class Worker extends activity_1.Activity {
+    constructor(config, data, metadata, hook, engine, context) {
+        super(config, data, metadata, hook, engine, context);
+    }
+    //********  INITIAL ENTRY POINT (A)  ********//
+    async process() {
+        this.logger.debug('worker-process', { jid: this.context.metadata.jid, aid: this.metadata.aid });
+        let telemetry;
+        try {
+            this.setLeg(1);
+            await collator_1.CollatorService.notarizeEntry(this);
+            await this.getState();
+            telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
+            telemetry.startActivitySpan(this.leg);
+            this.mapInputData();
+            const multi = this.store.getMulti();
+            //await this.registerTimeout();
+            await collator_1.CollatorService.authorizeReentry(this, multi);
+            await this.setState(multi);
+            await this.setStatus(0, multi);
+            const multiResponse = await multi.exec();
+            telemetry.mapActivityAttributes();
+            const jobStatus = this.resolveStatus(multiResponse);
+            const messageId = await this.execActivity();
+            telemetry.setActivityAttributes({
+                'app.activity.mid': messageId,
+                'app.job.jss': jobStatus
+            });
+            //TODO: UPDATE ACTIVITY STATE (LEG 1 EXIT)
+            return this.context.metadata.aid;
+        }
+        catch (error) {
+            if (error instanceof errors_1.GetStateError) {
+                this.logger.error('worker-get-state-error', error);
+            }
+            else {
+                this.logger.error('worker-process-error', error);
+            }
+            telemetry.setActivityError(error.message);
+            throw error;
+        }
+        finally {
+            telemetry.endActivitySpan();
+            this.logger.debug('worker-process-end', { jid: this.context.metadata.jid, aid: this.metadata.aid });
+        }
+    }
+    async execActivity() {
+        const streamData = {
+            metadata: {
+                jid: this.context.metadata.jid,
+                dad: this.metadata.dad,
+                aid: this.metadata.aid,
+                topic: this.config.subtype,
+                spn: this.context['$self'].output.metadata.l1s,
+                trc: this.context.metadata.trc,
+            },
+            data: this.context.data
+        };
+        if (this.config.retry) {
+            streamData.policies = {
+                retry: this.config.retry
+            };
+        }
+        return (await this.engine.streamSignaler?.publishMessage(this.config.subtype, streamData));
+    }
+    //********  SIGNAL RE-ENTRY POINT (DUPLEX LEG 2 of 2)  ********//
+    async processEvent(status = stream_1.StreamStatus.SUCCESS, code = 200) {
+        this.setLeg(2);
+        const jid = this.context.metadata.jid;
+        const aid = this.metadata.aid;
+        this.status = status;
+        this.code = code;
+        this.logger.debug('worker-process-event', { topic: this.config.subtype, jid, aid, status, code });
+        let telemetry;
+        try {
+            await this.getState();
+            const aState = await collator_1.CollatorService.notarizeReentry(this);
+            this.adjacentIndex = collator_1.CollatorService.getDimensionalIndex(aState);
+            telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
+            let isComplete = collator_1.CollatorService.isActivityComplete(this.context.metadata.js);
+            if (isComplete) {
+                this.logger.warn('worker-process-event-duplicate', { jid, aid });
+                this.logger.debug('worker-process-event-duplicate-resolution', { resolution: 'Increase HotMesh config `reclaimDelay` timeout.' });
+                return;
+            }
+            telemetry.startActivitySpan(this.leg);
+            if (status === stream_1.StreamStatus.PENDING) {
+                await this.processPending(telemetry);
+            }
+            else if (status === stream_1.StreamStatus.SUCCESS) {
+                await this.processSuccess(telemetry);
+            }
+            else {
+                await this.processError(telemetry);
+            }
+        }
+        catch (error) {
+            this.logger.error('worker-process-event-error', error);
+            telemetry.setActivityError(error.message);
+            throw error;
+        }
+        finally {
+            telemetry.endActivitySpan();
+            this.logger.debug('worker-process-event-end', { jid, aid });
+        }
+    }
+    async processPending(telemetry) {
+        this.bindActivityData('output');
+        this.adjacencyList = await this.filterAdjacent();
+        this.mapJobData();
+        const multi = this.store.getMulti();
+        await this.setState(multi);
+        await collator_1.CollatorService.notarizeContinuation(this, multi);
+        await this.setStatus(0, multi);
+        const multiResponse = await multi.exec();
+        telemetry.mapActivityAttributes();
+        const jobStatus = this.resolveStatus(multiResponse);
+        telemetry.setActivityAttributes({ 'app.job.jss': jobStatus });
+    }
+    async processSuccess(telemetry) {
+        this.bindActivityData('output');
+        this.adjacencyList = await this.filterAdjacent();
+        this.mapJobData();
+        const multi = this.store.getMulti();
+        await this.setState(multi);
+        await collator_1.CollatorService.notarizeCompletion(this, multi);
+        await this.setStatus(this.adjacencyList.length - 1, multi);
+        const multiResponse = await multi.exec();
+        telemetry.mapActivityAttributes();
+        const jobStatus = this.resolveStatus(multiResponse);
+        const attrs = { 'app.job.jss': jobStatus };
+        const messageIds = await this.transition(this.adjacencyList, jobStatus);
+        if (messageIds.length) {
+            attrs['app.activity.mids'] = messageIds.join(',');
+        }
+        telemetry.setActivityAttributes(attrs);
+    }
+    async processError(telemetry) {
+        this.bindActivityError(this.data);
+        this.adjacencyList = await this.filterAdjacent();
+        const multi = this.store.getMulti();
+        await this.setState(multi);
+        await collator_1.CollatorService.notarizeCompletion(this, multi);
+        await this.setStatus(this.adjacencyList.length - 1, multi);
+        const multiResponse = await multi.exec();
+        telemetry.mapActivityAttributes();
+        const jobStatus = this.resolveStatus(multiResponse);
+        const attrs = { 'app.job.jss': jobStatus };
+        const messageIds = await this.transition(this.adjacencyList, jobStatus);
+        if (messageIds.length) {
+            attrs['app.activity.mids'] = messageIds.join(',');
+        }
+        telemetry.setActivityAttributes(attrs);
+    }
+}
+exports.Worker = Worker;

package/build/services/collator/index.d.ts

@@ -0,0 +1,54 @@
+import { RedisMulti } from "../../types/redis";
+import { CollationStage } from "../../types/collator";
+import { ActivityDuplex } from "../../types/activity";
+import { HotMeshGraph } from "../../types/hotmesh";
+import { Activity } from "../activities/activity";
+declare class CollatorService {
+    static targetLength: number;
+    static notarizeEntry(activity: Activity, multi?: RedisMulti): Promise<number>;
+    static authorizeReentry(activity: Activity, multi?: RedisMulti): Promise<number>;
+    static notarizeEarlyCompletion(activity: Activity, multi?: RedisMulti): Promise<number>;
+    static notarizeReentry(activity: Activity, multi?: RedisMulti): Promise<number>;
+    static notarizeContinuation(activity: Activity, multi?: RedisMulti): Promise<number>;
+    static notarizeCompletion(activity: Activity, multi?: RedisMulti): Promise<number>;
+    static getDigitAtIndex(num: number, targetDigitIndex: number): number | null;
+    static getDimensionalIndex(num: number): number | null;
+    static isDuplicate(num: number, targetDigitIndex: number): boolean;
+    static isInactive(num: number): boolean;
+    static isPrimed(amount: number, leg: ActivityDuplex): boolean;
+    static verifyInteger(amount: number, leg: ActivityDuplex, stage: CollationStage): void;
+    /**
+     * All non-trigger activities are assigned a status seed by their parent
+     */
+    static getSeed(): string;
+    /**
+     * All trigger activities are assigned a status seed in a completed state
+     */
+    static getTriggerSeed(): string;
+    /**
+     * entry point for compiler-type activities. This is called by the compiler
+     * to bind the sorted activity IDs to the trigger activity. These are then used
+     * at runtime by the activities to track job/activity status.
+     * @param graphs
+     */
+    static compile(graphs: HotMeshGraph[]): void;
+    /**
+     * binds the ancestor array to each activity.
+     * Used in conjunction with the dimensional
+     *  address (dad). If dad is `,0,1,0,0` and the
+     * ancestor array is `['t1', 'a1', 'a2']` for
+     * activity 'a3', then the SAVED DAD
+     * will always have the trailing
+     * 0's removed. This ensures that the addressing
+     * remains consistent even if the graph changes.
+     *   id    DAD           SAVED DAD
+     * * t1 => ,0        =>  [empty]
+     * * a1 => ,0,1      =>  ,0,1
+     * * a2 => ,0,1,0    =>  ,0,1
+     * * a3 => ,0,1,0,0  =>  ,0,1
+     *
+     */
+    static bindAncestorArray(graphs: HotMeshGraph[]): void;
+    static isActivityComplete(status: number): boolean;
+}
+export { CollatorService };

package/build/services/collator/index.js

@@ -0,0 +1,171 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CollatorService = void 0;
+const errors_1 = require("../../modules/errors");
+const collator_1 = require("../../types/collator");
+class CollatorService {
+    static async notarizeEntry(activity, multi) {
+        //decrement by -100_000_000_000_000
+        const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, -100000000000000, multi);
+        this.verifyInteger(amount, 1, 'enter');
+        return amount;
+    }
+    ;
+    static async authorizeReentry(activity, multi) {
+        //set second digit to 8, allowing for re-entry
+        //decrement by -10_000_000_000_000
+        const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, -10000000000000, multi);
+        //this.verifyInteger(amount, 1, 'exit');
+        return amount;
+    }
+    static async notarizeEarlyCompletion(activity, multi) {
+        //initialize both `possible` (1m) and `actualized` (1) zero dimension, while decrementing the 2nd and 3rd digits to deactivate the activity
+        return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1000001 - 11000000000000, multi);
+    }
+    ;
+    static async notarizeReentry(activity, multi) {
+        //increment by 1_000_000 (indicates re-entry and is used to drive the 'dimensional address' for adjacent activities (minus 1))
+        const amount = await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1000000, multi);
+        this.verifyInteger(amount, 2, 'enter');
+        return amount;
+    }
+    ;
+    static async notarizeContinuation(activity, multi) {
+        //keep open; actualize the leg2 dimension (+1)
+        return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1, multi);
+    }
+    ;
+    static async notarizeCompletion(activity, multi) {
+        //close out; actualize leg2 dimension (+1) and decrement the 3rd digit (-1_000_000_000_000)
+        return await activity.store.collate(activity.context.metadata.jid, activity.metadata.aid, 1 - 1000000000000, multi);
+    }
+    ;
+    static getDigitAtIndex(num, targetDigitIndex) {
+        const numStr = num.toString();
+        if (targetDigitIndex < 0 || targetDigitIndex >= numStr.length) {
+            return null;
+        }
+        const digit = parseInt(numStr[targetDigitIndex], 10);
+        return digit;
+    }
+    static getDimensionalIndex(num) {
+        const numStr = num.toString();
+        if (numStr.length < 9) {
+            return null;
+        }
+        const extractedStr = numStr.substring(3, 9);
+        const extractedInt = parseInt(extractedStr, 10);
+        return extractedInt - 1;
+    }
+    static isDuplicate(num, targetDigitIndex) {
+        return this.getDigitAtIndex(num, targetDigitIndex) < 8;
+    }
+    static isInactive(num) {
+        return this.getDigitAtIndex(num, 2) < 9;
+    }
+    static isPrimed(amount, leg) {
+        //activity entry is not allowed if paths not properly pre-set
+        if (leg == 1) {
+            return amount != -100000000000000;
+        }
+        else {
+            return this.getDigitAtIndex(amount, 0) < 9 &&
+                this.getDigitAtIndex(amount, 1) < 9;
+        }
+    }
+    static verifyInteger(amount, leg, stage) {
+        let faultType;
+        if (leg === 1 && stage === 'enter') {
+            if (!this.isPrimed(amount, 1)) {
+                faultType = collator_1.CollationFaultType.MISSING;
+            }
+            else if (this.isDuplicate(amount, 0)) {
+                faultType = collator_1.CollationFaultType.DUPLICATE;
+            }
+            else if (amount != 899000000000000) {
+                faultType = collator_1.CollationFaultType.INVALID;
+            }
+        }
+        else if (leg === 1 && stage === 'exit') {
+            if (amount === -10000000000000) {
+                faultType = collator_1.CollationFaultType.MISSING;
+            }
+            else if (this.isDuplicate(amount, 1)) {
+                faultType = collator_1.CollationFaultType.DUPLICATE;
+            }
+        }
+        else if (leg === 2 && stage === 'enter') {
+            if (!this.isPrimed(amount, 2)) {
+                faultType = collator_1.CollationFaultType.FORBIDDEN;
+            }
+            else if (this.isInactive(amount)) {
+                faultType = collator_1.CollationFaultType.INACTIVE;
+            }
+        }
+        if (faultType) {
+            throw new errors_1.CollationError(amount, leg, stage, faultType);
+        }
+    }
+    /**
+     * All non-trigger activities are assigned a status seed by their parent
+     */
+    static getSeed() {
+        return '999000000000000';
+    }
+    /**
+     * All trigger activities are assigned a status seed in a completed state
+     */
+    static getTriggerSeed() {
+        return '888000001000001';
+    }
+    /**
+     * entry point for compiler-type activities. This is called by the compiler
+     * to bind the sorted activity IDs to the trigger activity. These are then used
+     * at runtime by the activities to track job/activity status.
+     * @param graphs
+     */
+    static compile(graphs) {
+        CollatorService.bindAncestorArray(graphs);
+    }
+    /**
+     * binds the ancestor array to each activity.
+     * Used in conjunction with the dimensional
+     *  address (dad). If dad is `,0,1,0,0` and the
+     * ancestor array is `['t1', 'a1', 'a2']` for
+     * activity 'a3', then the SAVED DAD
+     * will always have the trailing
+     * 0's removed. This ensures that the addressing
+     * remains consistent even if the graph changes.
+     *   id    DAD           SAVED DAD
+     * * t1 => ,0        =>  [empty]
+     * * a1 => ,0,1      =>  ,0,1
+     * * a2 => ,0,1,0    =>  ,0,1
+     * * a3 => ,0,1,0,0  =>  ,0,1
+     *
+     */
+    static bindAncestorArray(graphs) {
+        graphs.forEach((graph) => {
+            const ancestors = {};
+            const startingNode = Object.keys(graph.activities).find((activity) => graph.activities[activity].type === 'trigger');
+            if (!startingNode) {
+                throw new Error('collator-trigger-activity-not-found');
+            }
+            const dfs = (node, parentList) => {
+                ancestors[node] = parentList;
+                graph.activities[node]['ancestors'] = parentList;
+                const transitions = graph.transitions?.[node] || [];
+                transitions.forEach((transition) => {
+                    dfs(transition.to, [...parentList, node]);
+                });
+            };
+            // Start the DFS traversal
+            dfs(startingNode, []);
+        });
+    }
+    static isActivityComplete(status) {
+        return (status - 0) <= 0;
+    }
+}
+exports.CollatorService = CollatorService;
+//max int digit count that supports `hincrby`
+CollatorService.targetLength = 15;

package/build/services/compiler/deployer.d.ts

@@ -0,0 +1,35 @@
+import { StoreService } from '../store';
+import { HotMeshGraph, HotMeshManifest } from "../../types/hotmesh";
+import { RedisClient, RedisMulti } from "../../types/redis";
+import { Symbols } from "../../types/serializer";
+declare class Deployer {
+    manifest: HotMeshManifest | null;
+    store: StoreService<RedisClient, RedisMulti> | null;
+    constructor(manifest: HotMeshManifest);
+    deploy(store: StoreService<RedisClient, RedisMulti>): Promise<void>;
+    getVID(): {
+        id: string;
+        version: string;
+    };
+    generateSymKeys(): Promise<void>;
+    bindSymbols(startIndex: number, maxIndex: number, existingSymbols: Symbols, prefix: string, produces: string[]): Symbols;
+    copyJobSchemas(): void;
+    bindBackRefs(): void;
+    convertTopicsToTypes(): void;
+    bindParents(): Promise<void>;
+    collectValues(schema: Record<string, any>, values: Set<string>): void;
+    traverse(obj: any, values: Set<string>): void;
+    generateSymVals(): Promise<void>;
+    resolveJobMapsPaths(): void;
+    resolveMappingDependencies(): void;
+    groupMappingRules(rules: string[]): Record<string, string[]>;
+    resolveMappableValue(mappable: string): [string, string];
+    deployActivitySchemas(): Promise<void>;
+    deploySubscriptions(): Promise<void>;
+    findTrigger(graph: HotMeshGraph): [string, Record<string, any>] | null;
+    deployTransitions(): Promise<void>;
+    deployHookPatterns(): Promise<void>;
+    deployConsumerGroups(): Promise<void>;
+    deployConsumerGroup(stream: string, group: string): Promise<void>;
+}
+export { Deployer };