@hotmeshio/hotmesh 0.4.0 → 0.4.2

package/build/services/activities/trigger.d.ts

@@ -0,0 +1,37 @@
+import { EngineService } from '../engine';
+import { ActivityData, ActivityMetadata, ActivityType, TriggerActivity } from '../../types/activity';
+import { JobState, ExtensionType, JobStatus } from '../../types/job';
+import { ProviderTransaction } from '../../types/provider';
+import { StringScalarType } from '../../types/serializer';
+import { Activity } from './activity';
+declare class Trigger extends Activity {
+    config: TriggerActivity;
+    constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);
+    process(options?: ExtensionType): Promise<string>;
+    transitionAndLogAdjacent(options: ExtensionType, jobStatus: JobStatus, attrs: StringScalarType): Promise<void>;
+    /**
+     * `pending` flows will not transition from the trigger to adjacent children until resumed
+     */
+    initStatus(options: ExtensionType, count: number): number;
+    setExpired(seconds: number, transaction: ProviderTransaction): Promise<void>;
+    safeKey(key: string): string;
+    bindSearchData(options?: ExtensionType): void;
+    bindMarkerData(options?: ExtensionType): void;
+    setStatus(amount: number): Promise<void>;
+    /**
+     * if the parent (spawner) chose not to await, emit the job_id
+     * as the data payload { job_id }
+     */
+    execAdjacentParent(): Promise<void>;
+    createInputContext(): Partial<JobState>;
+    getState(): Promise<void>;
+    bindJobMetadataPaths(): string[];
+    bindActivityMetadataPaths(): string[];
+    resolveGranularity(): string;
+    getJobStatus(): number;
+    resolveJobId(context: Partial<JobState>): string;
+    resolveJobKey(context: Partial<JobState>): string;
+    setStateNX(status?: number): Promise<void>;
+    setStats(transaction?: ProviderTransaction): Promise<void>;
+}
+export { Trigger };

package/build/services/activities/trigger.js

@@ -0,0 +1,246 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Trigger = void 0;
+const errors_1 = require("../../modules/errors");
+const utils_1 = require("../../modules/utils");
+const collator_1 = require("../collator");
+const pipe_1 = require("../pipe");
+const reporter_1 = require("../reporter");
+const serializer_1 = require("../serializer");
+const telemetry_1 = require("../telemetry");
+const activity_1 = require("./activity");
+class Trigger extends activity_1.Activity {
+    constructor(config, data, metadata, hook, engine, context) {
+        super(config, data, metadata, hook, engine, context);
+    }
+    async process(options) {
+        this.logger.debug('trigger-process', {
+            subscribes: this.config.subscribes,
+        });
+        let telemetry;
+        try {
+            this.setLeg(2);
+            await this.getState();
+            telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
+            telemetry.startJobSpan();
+            telemetry.startActivitySpan(this.leg);
+            this.mapJobData();
+            this.adjacencyList = await this.filterAdjacent();
+            const initialStatus = this.initStatus(options, this.adjacencyList.length);
+            await this.setStateNX(initialStatus);
+            await this.setStatus(initialStatus);
+            this.bindSearchData(options);
+            this.bindMarkerData(options);
+            const transaction = this.store.transact();
+            await this.setState(transaction);
+            await this.setStats(transaction);
+            if (options?.pending) {
+                await this.setExpired(options?.pending, transaction);
+            }
+            await collator_1.CollatorService.notarizeInception(this, this.context.metadata.guid, transaction);
+            await transaction.exec();
+            this.execAdjacentParent();
+            telemetry.mapActivityAttributes();
+            const jobStatus = Number(this.context.metadata.js);
+            telemetry.setJobAttributes({ 'app.job.jss': jobStatus });
+            const attrs = { 'app.job.jss': jobStatus };
+            await this.transitionAndLogAdjacent(options, jobStatus, attrs);
+            telemetry.setActivityAttributes(attrs);
+            return this.context.metadata.jid;
+        }
+        catch (error) {
+            telemetry?.setActivityError(error.message);
+            if (error instanceof errors_1.DuplicateJobError) {
+                //todo: verify baseline in x-AZ rollover
+                await (0, utils_1.sleepFor)(1000);
+                const isOverage = await collator_1.CollatorService.isInceptionOverage(this, this.context.metadata.guid);
+                if (isOverage) {
+                    this.logger.info('trigger-collation-overage', {
+                        job_id: error.jobId,
+                        guid: this.context.metadata.guid,
+                    });
+                    return;
+                }
+                this.logger.error('duplicate-job-error', {
+                    job_id: error.jobId,
+                    guid: this.context.metadata.guid,
+                });
+            }
+            else {
+                this.logger.error('trigger-process-error', { error });
+            }
+            throw error;
+        }
+        finally {
+            telemetry?.endJobSpan();
+            telemetry?.endActivitySpan();
+            this.logger.debug('trigger-process-end', {
+                subscribes: this.config.subscribes,
+                jid: this.context.metadata.jid,
+                gid: this.context.metadata.gid,
+            });
+        }
+    }
+    async transitionAndLogAdjacent(options = {}, jobStatus, attrs) {
+        //todo: enable resume from pending state
+        if (isNaN(options.pending)) {
+            const messageIds = await this.transition(this.adjacencyList, jobStatus);
+            if (messageIds.length) {
+                attrs['app.activity.mids'] = messageIds.join(',');
+            }
+        }
+    }
+    /**
+     * `pending` flows will not transition from the trigger to adjacent children until resumed
+     */
+    initStatus(options = {}, count) {
+        if (options.pending) {
+            return -1;
+        }
+        return count;
+    }
+    async setExpired(seconds, transaction) {
+        await this.store.expireJob(this.context.metadata.jid, seconds, transaction);
+    }
+    safeKey(key) {
+        return `_${key}`;
+    }
+    bindSearchData(options) {
+        if (options?.search) {
+            Object.keys(options.search).forEach((key) => {
+                this.context.data[this.safeKey(key)] = options.search[key].toString();
+            });
+        }
+    }
+    bindMarkerData(options) {
+        if (options?.marker) {
+            Object.keys(options.marker).forEach((key) => {
+                if (key.startsWith('-')) {
+                    this.context.data[key] = options.marker[key].toString();
+                }
+            });
+        }
+    }
+    async setStatus(amount) {
+        this.context.metadata.js = amount;
+    }
+    /**
+     * if the parent (spawner) chose not to await, emit the job_id
+     * as the data payload { job_id }
+     */
+    async execAdjacentParent() {
+        if (this.context.metadata.px) {
+            const timestamp = (0, utils_1.formatISODate)(new Date());
+            const jobStartedConfirmationMessage = {
+                metadata: this.context.metadata,
+                data: {
+                    job_id: this.context.metadata.jid,
+                    jc: timestamp,
+                    ju: timestamp,
+                },
+            };
+            await this.engine.execAdjacentParent(this.context, jobStartedConfirmationMessage);
+        }
+    }
+    createInputContext() {
+        const input = {
+            [this.metadata.aid]: {
+                input: { data: this.data },
+            },
+            $self: {
+                input: { data: this.data },
+                output: { data: this.data },
+            },
+        };
+        return input;
+    }
+    async getState() {
+        const inputContext = this.createInputContext();
+        const jobId = this.resolveJobId(inputContext);
+        const jobKey = this.resolveJobKey(inputContext);
+        const utc = (0, utils_1.formatISODate)(new Date());
+        const { id, version } = await this.engine.getVID();
+        this.initDimensionalAddress(collator_1.CollatorService.getDimensionalSeed());
+        const activityMetadata = {
+            ...this.metadata,
+            jid: jobId,
+            key: jobKey,
+            as: collator_1.CollatorService.getTriggerSeed(),
+        };
+        this.context = {
+            metadata: {
+                ...this.metadata,
+                gid: (0, utils_1.guid)(),
+                ngn: this.context.metadata.ngn,
+                pj: this.context.metadata.pj,
+                pg: this.context.metadata.pg,
+                pd: this.context.metadata.pd,
+                pa: this.context.metadata.pa,
+                px: this.context.metadata.px,
+                app: id,
+                vrs: version,
+                tpc: this.config.subscribes,
+                trc: this.context.metadata.trc,
+                spn: this.context.metadata.spn,
+                guid: this.context.metadata.guid,
+                jid: jobId,
+                dad: collator_1.CollatorService.getDimensionalSeed(),
+                key: jobKey,
+                jc: utc,
+                ju: utc,
+                ts: (0, utils_1.getTimeSeries)(this.resolveGranularity()),
+                js: 0,
+            },
+            data: {},
+            [this.metadata.aid]: {
+                input: {
+                    data: this.data,
+                    metadata: activityMetadata,
+                },
+                output: {
+                    data: this.data,
+                    metadata: activityMetadata,
+                },
+                settings: { data: {} },
+                errors: { data: {} },
+            },
+        };
+        this.context['$self'] = this.context[this.metadata.aid];
+        this.context['$job'] = this.context; //NEVER call STRINGIFY! (circular)
+    }
+    bindJobMetadataPaths() {
+        return serializer_1.MDATA_SYMBOLS.JOB.KEYS.map((key) => `metadata/${key}`);
+    }
+    bindActivityMetadataPaths() {
+        return serializer_1.MDATA_SYMBOLS.ACTIVITY.KEYS.map((key) => `output/metadata/${key}`);
+    }
+    resolveGranularity() {
+        return (this.config.stats?.granularity || reporter_1.ReporterService.DEFAULT_GRANULARITY);
+    }
+    getJobStatus() {
+        return this.context.metadata.js;
+    }
+    resolveJobId(context) {
+        const jobId = this.config.stats?.id;
+        return jobId ? pipe_1.Pipe.resolve(jobId, context) : (0, utils_1.guid)();
+    }
+    resolveJobKey(context) {
+        const jobKey = this.config.stats?.key;
+        return jobKey ? pipe_1.Pipe.resolve(jobKey, context) : '';
+    }
+    async setStateNX(status) {
+        const jobId = this.context.metadata.jid;
+        if (!await this.store.setStateNX(jobId, this.engine.appId, status)) {
+            throw new errors_1.DuplicateJobError(jobId);
+        }
+    }
+    async setStats(transaction) {
+        const md = this.context.metadata;
+        if (md.key && this.config.stats?.measures) {
+            const config = await this.engine.getVID();
+            const reporter = new reporter_1.ReporterService(config, this.store, this.logger);
+            await this.store.setStats(md.key, md.jid, md.ts, reporter.resolveTriggerStatistics(this.config, this.context), config, transaction);
+        }
+    }
+}
+exports.Trigger = Trigger;

package/build/services/activities/worker.d.ts

@@ -0,0 +1,12 @@
+import { EngineService } from '../engine';
+import { ActivityData, ActivityMetadata, ActivityType, WorkerActivity } from '../../types/activity';
+import { JobState } from '../../types/job';
+import { ProviderTransaction } from '../../types/provider';
+import { Activity } from './activity';
+declare class Worker extends Activity {
+    config: WorkerActivity;
+    constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);
+    process(): Promise<string>;
+    execActivity(transaction: ProviderTransaction): Promise<string>;
+}
+export { Worker };

package/build/services/activities/worker.js

@@ -0,0 +1,106 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Worker = void 0;
+const errors_1 = require("../../modules/errors");
+const utils_1 = require("../../modules/utils");
+const collator_1 = require("../collator");
+const pipe_1 = require("../pipe");
+const telemetry_1 = require("../telemetry");
+const activity_1 = require("./activity");
+class Worker extends activity_1.Activity {
+    constructor(config, data, metadata, hook, engine, context) {
+        super(config, data, metadata, hook, engine, context);
+    }
+    //********  INITIAL ENTRY POINT (A)  ********//
+    async process() {
+        this.logger.debug('worker-process', {
+            jid: this.context.metadata.jid,
+            gid: this.context.metadata.gid,
+            aid: this.metadata.aid,
+        });
+        let telemetry;
+        try {
+            await this.verifyEntry();
+            telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
+            telemetry.startActivitySpan(this.leg);
+            this.mapInputData();
+            //save state and authorize reentry
+            const transaction = this.store.transact();
+            //todo: await this.registerTimeout();
+            const messageId = await this.execActivity(transaction);
+            await collator_1.CollatorService.authorizeReentry(this, transaction);
+            await this.setState(transaction);
+            await this.setStatus(0, transaction);
+            const txResponse = (await transaction.exec());
+            //telemetry
+            telemetry.mapActivityAttributes();
+            const jobStatus = this.resolveStatus(txResponse);
+            telemetry.setActivityAttributes({
+                'app.activity.mid': messageId,
+                'app.job.jss': jobStatus,
+            });
+            return this.context.metadata.aid;
+        }
+        catch (error) {
+            if (error instanceof errors_1.InactiveJobError) {
+                this.logger.error('await-inactive-job-error', { error });
+                return;
+            }
+            else if (error instanceof errors_1.GenerationalError) {
+                this.logger.info('process-event-generational-job-error', { error });
+                return;
+            }
+            else if (error instanceof errors_1.GetStateError) {
+                this.logger.error('worker-get-state-error', { error });
+                return;
+            }
+            else if (error instanceof errors_1.CollationError) {
+                if (error.fault === 'duplicate') {
+                    this.logger.info('worker-collation-overage', {
+                        job_id: this.context.metadata.jid,
+                        guid: this.context.metadata.guid,
+                    });
+                    return;
+                }
+                //unknown collation error
+                this.logger.error('worker-collation-error', { error });
+            }
+            else {
+                this.logger.error('worker-process-error', { error });
+            }
+            telemetry?.setActivityError(error.message);
+            throw error;
+        }
+        finally {
+            telemetry?.endActivitySpan();
+            this.logger.debug('worker-process-end', {
+                jid: this.context.metadata.jid,
+                gid: this.context.metadata.gid,
+                aid: this.metadata.aid,
+            });
+        }
+    }
+    async execActivity(transaction) {
+        const topic = pipe_1.Pipe.resolve(this.config.subtype, this.context);
+        const streamData = {
+            metadata: {
+                guid: (0, utils_1.guid)(),
+                jid: this.context.metadata.jid,
+                gid: this.context.metadata.gid,
+                dad: this.metadata.dad,
+                aid: this.metadata.aid,
+                topic,
+                spn: this.context['$self'].output.metadata.l1s,
+                trc: this.context.metadata.trc,
+            },
+            data: this.context.data,
+        };
+        if (this.config.retry) {
+            streamData.policies = {
+                retry: this.config.retry,
+            };
+        }
+        return (await this.engine.router?.publishMessage(topic, streamData, transaction));
+    }
+}
+exports.Worker = Worker;

package/build/services/collator/index.d.ts

@@ -0,0 +1,111 @@
+import { CollationStage } from '../../types/collator';
+import { ActivityDuplex } from '../../types/activity';
+import { HotMeshGraph } from '../../types/hotmesh';
+import { ProviderTransaction } from '../../types/provider';
+import { Activity } from '../activities/activity';
+import { Cycle } from '../activities/cycle';
+declare class CollatorService {
+    static targetLength: number;
+    /**
+     * Upon re/entry, verify that the job status is active
+     */
+    static assertJobActive(status: number, jobId: string, activityId: string, threshold?: number): void;
+    /**
+     * returns the dimensional address (dad) for the target; due
+     * to the nature of the notary system, the dad for leg 2 entry
+     * must target the `0` index while leg 2 exit must target the
+     * current index (0)
+     */
+    static getDimensionalAddress(activity: Activity, isEntry?: boolean): Record<string, string>;
+    /**
+     * resolves the dimensional address for the
+     * ancestor in the graph to go back to. this address
+     * is determined by trimming the last digits from
+     * the `dad` (including the target).
+     * the target activity index is then set to `0`, so that
+     * the origin node can be queried for approval/entry.
+     */
+    static resolveReentryDimension(activity: Cycle): string;
+    static notarizeEntry(activity: Activity, transaction?: ProviderTransaction): Promise<number>;
+    static authorizeReentry(activity: Activity, transaction?: ProviderTransaction): Promise<number>;
+    static notarizeEarlyExit(activity: Activity, transaction?: ProviderTransaction): Promise<number>;
+    static notarizeEarlyCompletion(activity: Activity, transaction?: ProviderTransaction): Promise<number>;
+    /**
+     * sets the synthetic inception key (in case of an overage occurs).
+     */
+    static notarizeInception(activity: Activity, guid: string, transaction: ProviderTransaction): Promise<void>;
+    /**
+     * ignore those ID collisions that are due to re-entry overages
+     */
+    static isInceptionOverage(activity: Activity, guid: string): Promise<boolean>;
+    /**
+     * verifies both the concrete and synthetic keys for the activity; concrete keys
+     * exist in the original model and are effectively the 'real' keys. In reality,
+     * hook activities are atomized during compilation to create a synthetic DAG that
+     * is used to track the status of the graph in a distributed environment. The
+     * synthetic key represents different dimensional realities and is used to
+     * track re-entry overages (it distinguishes between the original and re-entry).
+     * The essential challenge is: is this a re-entry that is purposeful in
+     * order to induce cycles, or is the re-entry due to a failure in the system?
+     */
+    static notarizeReentry(activity: Activity, guid: string, transaction?: ProviderTransaction): Promise<number>;
+    static notarizeContinuation(activity: Activity, transaction?: ProviderTransaction): Promise<number>;
+    static notarizeCompletion(activity: Activity, transaction?: ProviderTransaction): Promise<number>;
+    static getDigitAtIndex(num: number, targetDigitIndex: number): number | null;
+    static getDimensionalIndex(num: number): number | null;
+    static isDuplicate(num: number, targetDigitIndex: number): boolean;
+    static isInactive(num: number): boolean;
+    static isPrimed(amount: number, leg: ActivityDuplex): boolean;
+    /**
+     * During compilation, the graphs are compiled into structures necessary
+     * for distributed processing; these are referred to as 'synthetic DAGs',
+     * because they are not part of the original graph, but are used to track
+     * the status of the graph in a distributed environment. This check ensures
+     * that the 'synthetic key' is not a duplicate. (which is different than
+     * saying the 'key' is not a duplicate)
+     */
+    static verifySyntheticInteger(amount: number): void;
+    static verifyInteger(amount: number, leg: ActivityDuplex, stage: CollationStage): void;
+    static getDimensionsById(ancestors: string[], dad: string): Record<string, string>;
+    /**
+     * All non-trigger activities are assigned a status seed by their parent
+     */
+    static getSeed(): string;
+    /**
+     * All trigger activities are assigned a status seed in a completed state
+     */
+    static getTriggerSeed(): string;
+    /**
+     * entry point for compiler-type activities. This is called by the compiler
+     * to bind the sorted activity IDs to the trigger activity. These are then used
+     * at runtime by the activities to track job/activity status.
+     * @param graphs
+     */
+    static compile(graphs: HotMeshGraph[]): void;
+    /**
+     * binds the ancestor array to each activity.
+     * Used in conjunction with the dimensional
+     *  address (dad). If dad is `,0,1,0,0` and the
+     * ancestor array is `['t1', 'a1', 'a2']` for
+     * activity 'a3', then the SAVED DAD
+     * will always have the trailing
+     * 0's removed. This ensures that the addressing
+     * remains consistent even if the graph changes.
+     *   id    DAD           SAVED DAD
+     * * t1 => ,0        =>  [empty]
+     * * a1 => ,0,1      =>  ,0,1
+     * * a2 => ,0,1,0    =>  ,0,1
+     * * a3 => ,0,1,0,0  =>  ,0,1
+     *
+     */
+    static bindAncestorArray(graphs: HotMeshGraph[]): void;
+    /**
+     * All activities exist on a dimensional plane. Zero
+     * is the default. A value of
+     * `AxY,0,0,0,0,1,0,0` would reflect that
+     * an ancestor activity was dimensionalized beyond
+     * the default.
+     */
+    static getDimensionalSeed(index?: number): string;
+}
+export { CollatorService };