@hotmeshio/hotmesh 0.0.8 → 0.0.10

package/build/esm/services/engine/index.js

@@ -1,522 +0,0 @@
-import { KeyType } from '../../modules/key';
-import { formatISODate, getSubscriptionTopic, identifyRedisType, restoreHierarchy } from '../../modules/utils';
-import Activities from '../activities';
-import { CompilerService } from '../compiler';
-import { ReporterService } from '../reporter';
-import { SerializerService } from '../serializer';
-import { StoreSignaler } from '../signaler/store';
-import { StreamSignaler } from '../signaler/stream';
-import { RedisStoreService as RedisStore } from '../store/clients/redis';
-import { IORedisStoreService as IORedisStore } from '../store/clients/ioredis';
-import { RedisStreamService as RedisStream } from '../stream/clients/redis';
-import { IORedisStreamService as IORedisStream } from '../stream/clients/ioredis';
-import { IORedisSubService as IORedisSub } from '../sub/clients/ioredis';
-import { RedisSubService as RedisSub } from '../sub/clients/redis';
-import { TaskService } from '../task';
-import { StreamDataType, StreamRole, StreamStatus } from '../../types/stream';
-//wait time to see if a job is complete
-const OTT_WAIT_TIME = 1000;
-const STATUS_CODE_SUCCESS = 200;
-const STATUS_CODE_TIMEOUT = 504;
-class EngineService {
-    constructor() {
-        this.cacheMode = 'cache';
-        this.untilVersion = null;
-        this.jobCallbacks = {};
-        this.reporting = false;
-        this.jobId = 1;
-    }
-    static async init(namespace, appId, guid, config, logger) {
-        if (config.engine) {
-            const instance = new EngineService();
-            instance.verifyEngineFields(config);
-            instance.namespace = namespace;
-            instance.appId = appId;
-            instance.guid = guid;
-            instance.logger = logger;
-            await instance.initStoreChannel(config.engine.store);
-            await instance.initSubChannel(config.engine.sub);
-            await instance.initStreamChannel(config.engine.stream);
-            instance.streamSignaler = instance.initStreamSignaler(config);
-            instance.streamSignaler.consumeMessages(instance.stream.mintKey(KeyType.STREAMS, { appId: instance.appId }), 'ENGINE', instance.guid, instance.processStreamMessage.bind(instance));
-            //the storeSignaler service is used by the engine to create `webhooks`
-            //todo: unify/move to the task service (it manages all `signal` types)
-            instance.storeSignaler = new StoreSignaler(instance.store, logger);
-            //the task service is used by the engine to process `webhooks` and `timehooks`
-            instance.task = new TaskService(instance.store, logger);
-            return instance;
-        }
-    }
-    verifyEngineFields(config) {
-        if (!identifyRedisType(config.engine.store) ||
-            !identifyRedisType(config.engine.stream) ||
-            !identifyRedisType(config.engine.sub)) {
-            throw new Error('engine config must reference 3 redis client instances');
-        }
-    }
-    async initStoreChannel(store) {
-        if (identifyRedisType(store) === 'redis') {
-            this.store = new RedisStore(store);
-        }
-        else {
-            this.store = new IORedisStore(store);
-        }
-        await this.store.init(this.namespace, this.appId, this.logger);
-    }
-    async initSubChannel(sub) {
-        if (identifyRedisType(sub) === 'redis') {
-            this.subscribe = new RedisSub(sub);
-        }
-        else {
-            this.subscribe = new IORedisSub(sub);
-        }
-        await this.subscribe.init(this.namespace, this.appId, this.guid, this.logger);
-    }
-    async initStreamChannel(stream) {
-        if (identifyRedisType(stream) === 'redis') {
-            this.stream = new RedisStream(stream);
-        }
-        else {
-            this.stream = new IORedisStream(stream);
-        }
-        await this.stream.init(this.namespace, this.appId, this.logger);
-    }
-    initStreamSignaler(config) {
-        return new StreamSignaler({
-            namespace: this.namespace,
-            appId: this.appId,
-            guid: this.guid,
-            role: StreamRole.ENGINE,
-            reclaimDelay: config.engine.reclaimDelay,
-            reclaimCount: config.engine.reclaimCount,
-        }, this.stream, this.store, this.logger);
-    }
-    async getVID(vid) {
-        if (this.cacheMode === 'nocache') {
-            const app = await this.store.getApp(this.appId, true);
-            if (app.version.toString() === this.untilVersion.toString()) {
-                //new version is deployed; OK to cache again
-                if (!this.apps)
-                    this.apps = {};
-                this.apps[this.appId] = app;
-                this.setCacheMode('cache', app.version.toString());
-            }
-            return { id: this.appId, version: app.version };
-        }
-        else if (!this.apps && vid) {
-            this.apps = {};
-            this.apps[this.appId] = vid;
-            return vid;
-        }
-        else {
-            return { id: this.appId, version: this.apps?.[this.appId].version };
-        }
-    }
-    setCacheMode(cacheMode, untilVersion) {
-        this.logger.info(`engine-rule-cache-updated`, { mode: cacheMode, until: untilVersion });
-        this.cacheMode = cacheMode;
-        this.untilVersion = untilVersion;
-    }
-    async routeToSubscribers(topic, message) {
-        const jobCallback = this.jobCallbacks[message.metadata.jid];
-        if (jobCallback) {
-            this.delistJobCallback(message.metadata.jid);
-            jobCallback(topic, message);
-        }
-    }
-    async processWebHooks() {
-        this.task.processWebHooks((this.hook).bind(this));
-    }
-    async processTimeHooks() {
-        this.task.processTimeHooks((this.hookTime).bind(this));
-    }
-    async throttle(delayInMillis) {
-        this.streamSignaler.setThrottle(delayInMillis);
-    }
-    // ************* METADATA/MODEL METHODS *************
-    async initActivity(topic, data = {}, context) {
-        const [activityId, schema] = await this.getSchema(topic);
-        const ActivityHandler = Activities[schema.type];
-        if (ActivityHandler) {
-            const utc = formatISODate(new Date());
-            const metadata = {
-                aid: activityId,
-                atp: schema.type,
-                stp: schema.subtype,
-                ac: utc,
-                au: utc
-            };
-            const hook = null;
-            return new ActivityHandler(schema, data, metadata, hook, this, context);
-        }
-        else {
-            throw new Error(`activity type ${schema.type} not found`);
-        }
-    }
-    async getSchema(topic) {
-        const app = await this.store.getApp(this.appId);
-        if (!app) {
-            throw new Error(`no app found for id ${this.appId}`);
-        }
-        if (this.isPrivate(topic)) {
-            //private subscriptions use the schema id (.activityId)
-            const activityId = topic.substring(1);
-            const schema = await this.store.getSchema(activityId, await this.getVID(app));
-            return [activityId, schema];
-        }
-        else {
-            //public subscriptions use a topic (a.b.c) that is associated with a schema id
-            const activityId = await this.store.getSubscription(topic, await this.getVID(app));
-            if (activityId) {
-                const schema = await this.store.getSchema(activityId, await this.getVID(app));
-                return [activityId, schema];
-            }
-        }
-        throw new Error(`no subscription found for topic ${topic} in app ${this.appId} for app version ${app.version}`);
-    }
-    async getSettings() {
-        return await this.store.getSettings();
-    }
-    isPrivate(topic) {
-        return topic.startsWith('.');
-    }
-    // ************* COMPILER METHODS *************
-    async plan(pathOrYAML) {
-        const compiler = new CompilerService(this.store, this.logger);
-        return await compiler.plan(pathOrYAML);
-    }
-    async deploy(pathOrYAML) {
-        const compiler = new CompilerService(this.store, this.logger);
-        return await compiler.deploy(pathOrYAML);
-    }
-    // ************* REPORTER METHODS *************
-    async getStats(topic, query) {
-        const { id, version } = await this.getVID();
-        const reporter = new ReporterService({ id, version }, this.store, this.logger);
-        const resolvedQuery = await this.resolveQuery(topic, query);
-        return await reporter.getStats(resolvedQuery);
-    }
-    async getIds(topic, query, queryFacets = []) {
-        const { id, version } = await this.getVID();
-        const reporter = new ReporterService({ id, version }, this.store, this.logger);
-        const resolvedQuery = await this.resolveQuery(topic, query);
-        return await reporter.getIds(resolvedQuery, queryFacets);
-    }
-    async resolveQuery(topic, query) {
-        const trigger = await this.initActivity(topic, query.data);
-        await trigger.getState();
-        return {
-            end: query.end,
-            start: query.start,
-            range: query.range,
-            granularity: trigger.resolveGranularity(),
-            key: trigger.resolveJobKey(trigger.createInputContext()),
-            sparse: query.sparse,
-        };
-    }
-    // ****************** STREAM RE-ENTRY POINT *****************
-    async processStreamMessage(streamData) {
-        this.logger.debug('engine-process-stream-message', {
-            jid: streamData.metadata.jid,
-            dad: streamData.metadata.dad,
-            aid: streamData.metadata.aid,
-            status: streamData.status || StreamStatus.SUCCESS,
-            code: streamData.code || 200,
-        });
-        const context = {
-            metadata: {
-                jid: streamData.metadata.jid,
-                dad: streamData.metadata.dad,
-                aid: streamData.metadata.aid,
-            },
-            data: streamData.data,
-        };
-        if (streamData.type === StreamDataType.TIMEHOOK || streamData.type === StreamDataType.WEBHOOK || streamData.type === StreamDataType.TRANSITION) {
-            const activityHandler = await this.initActivity(`.${streamData.metadata.aid}`, context.data, context);
-            if (streamData.type === StreamDataType.TIMEHOOK) {
-                await activityHandler.processTimeHookEvent(streamData.metadata.jid);
-            }
-            else if (streamData.type === StreamDataType.TRANSITION) {
-                await activityHandler.process();
-            }
-            else {
-                await activityHandler.processWebHookEvent();
-            }
-        }
-        else if (streamData.type === StreamDataType.AWAIT) {
-            context.metadata = {
-                ...context.metadata,
-                pj: streamData.metadata.jid,
-                pd: streamData.metadata.dad,
-                pa: streamData.metadata.aid,
-                trc: streamData.metadata.trc,
-                spn: streamData.metadata.spn,
-            };
-            const activityHandler = await this.initActivity(streamData.metadata.topic, streamData.data, context);
-            await activityHandler.process();
-        }
-        else if (streamData.type === StreamDataType.RESULT) {
-            const activityHandler = await this.initActivity(`.${context.metadata.aid}`, streamData.data, context);
-            await activityHandler.processEvent(streamData.status, streamData.code);
-        }
-        else {
-            const activityHandler = await this.initActivity(`.${streamData.metadata.aid}`, streamData.data, context);
-            await activityHandler.processEvent(streamData.status, streamData.code);
-        }
-        this.logger.debug('engine-process-stream-message-end', {
-            jid: streamData.metadata.jid,
-            aid: streamData.metadata.aid
-        });
-    }
-    // ***************** `AWAIT` ACTIVITY RETURN RESPONSE ****************
-    async execAdjacentParent(context, jobOutput) {
-        if (this.hasParentJob(context)) {
-            //errors are stringified `StreamError` objects
-            const error = this.resolveError(jobOutput.metadata);
-            const spn = context['$self']?.output?.metadata?.l2s || context['$self']?.output?.metadata?.l1s;
-            const streamData = {
-                metadata: {
-                    jid: context.metadata.pj,
-                    dad: context.metadata.pd,
-                    aid: context.metadata.pa,
-                    trc: context.metadata.trc,
-                    spn,
-                },
-                type: StreamDataType.RESULT,
-                data: jobOutput.data,
-            };
-            if (error && error.code) {
-                streamData.status = StreamStatus.ERROR;
-                streamData.data = error;
-                streamData.code = error.code;
-            }
-            else {
-                streamData.status = StreamStatus.SUCCESS;
-                streamData.code = STATUS_CODE_SUCCESS;
-            }
-            return (await this.streamSignaler?.publishMessage(null, streamData));
-        }
-    }
-    hasParentJob(context) {
-        //todo: include the dimensional address (pd)
-        return Boolean(context.metadata.pj && context.metadata.pa);
-    }
-    resolveError(metadata) {
-        if (metadata && metadata.err) {
-            return JSON.parse(metadata.err);
-        }
-    }
-    // ****************** `SCRUB` CLEAN COMPLETED JOBS *****************
-    async scrub(jobId) {
-        await this.store.scrub(jobId);
-    }
-    // ****************** `HOOK` ACTIVITY RE-ENTRY POINT *****************
-    async hook(topic, data, dad) {
-        const hookRule = await this.storeSignaler.getHookRule(topic);
-        const [aid, schema] = await this.getSchema(`.${hookRule.to}`);
-        if (!dad) {
-            //assume dimensional address is singular (0)
-            // for ancestors and self if not provided
-            // todo: register
-            dad = ',0'.repeat(schema.ancestors.length + 1);
-        }
-        const streamData = {
-            type: StreamDataType.WEBHOOK,
-            metadata: {
-                //jid is unknown at this point; will be resolved using the data
-                aid,
-                dad,
-                topic
-            },
-            data,
-        };
-        await this.streamSignaler.publishMessage(null, streamData);
-    }
-    async hookTime(jobId, activityId) {
-        //the activityid is concatenated with its dimensional address (dad); split to resolve
-        const [aid, ...dimensions] = activityId.split(',');
-        const dad = `,${dimensions.join(',')}`;
-        const streamData = {
-            type: StreamDataType.TIMEHOOK,
-            metadata: {
-                jid: jobId,
-                aid,
-                dad,
-            },
-            data: { timestamp: Date.now() },
-        };
-        await this.streamSignaler.publishMessage(null, streamData);
-    }
-    async hookAll(hookTopic, data, query, queryFacets = []) {
-        const config = await this.getVID();
-        const hookRule = await this.storeSignaler.getHookRule(hookTopic);
-        if (hookRule) {
-            const subscriptionTopic = await getSubscriptionTopic(hookRule.to, this.store, config);
-            const resolvedQuery = await this.resolveQuery(subscriptionTopic, query);
-            const reporter = new ReporterService(config, this.store, this.logger);
-            const workItems = await reporter.getWorkItems(resolvedQuery, queryFacets);
-            const taskService = new TaskService(this.store, this.logger);
-            await taskService.enqueueWorkItems(workItems.map(workItem => `${hookTopic}::${workItem}::${JSON.stringify(data)}`));
-            this.store.publish(KeyType.QUORUM, { type: 'work', originator: this.guid }, this.appId);
-            return workItems;
-        }
-        else {
-            throw new Error(`unable to find hook rule for topic ${hookTopic}`);
-        }
-    }
-    // ********************** PUB/SUB ENTRY POINT **********************
-    //publish (returns just the job id)
-    async pub(topic, data, context) {
-        const activityHandler = await this.initActivity(topic, data, context);
-        if (activityHandler) {
-            return await activityHandler.process();
-        }
-        else {
-            throw new Error(`unable to process activity for topic ${topic}`);
-        }
-    }
-    //subscribe to all jobs for a topic
-    async sub(topic, callback) {
-        const subscriptionCallback = async (topic, message) => {
-            callback(message.topic, message.job);
-        };
-        return await this.subscribe.subscribe(KeyType.QUORUM, subscriptionCallback, this.appId, topic);
-    }
-    //unsubscribe to all jobs for a topic
-    async unsub(topic) {
-        return await this.subscribe.unsubscribe(KeyType.QUORUM, this.appId, topic);
-    }
-    //subscribe to all jobs for a wildcard topic
-    async psub(wild, callback) {
-        const subscriptionCallback = async (topic, message) => {
-            callback(message.topic, message.job);
-        };
-        return await this.subscribe.psubscribe(KeyType.QUORUM, subscriptionCallback, this.appId, wild);
-    }
-    //unsubscribe to all jobs for a wildcard topic
-    async punsub(wild) {
-        return await this.subscribe.punsubscribe(KeyType.QUORUM, this.appId, wild);
-    }
-    //publish and await (returns the job and data (if ready)); throws error with jobid if not
-    async pubsub(topic, data, context, timeout = OTT_WAIT_TIME) {
-        context = {
-            metadata: {
-                ngn: this.guid,
-                trc: context?.metadata?.trc,
-                spn: context?.metadata?.spn
-            }
-        };
-        const jobId = await this.pub(topic, data, context);
-        return new Promise((resolve, reject) => {
-            this.registerJobCallback(jobId, (topic, output) => {
-                if (output.metadata.err) {
-                    const error = JSON.parse(output.metadata.err);
-                    reject({
-                        ...error,
-                        job_id: output.metadata.jid,
-                    });
-                }
-                else {
-                    resolve(output);
-                }
-            });
-            setTimeout(() => {
-                this.delistJobCallback(jobId);
-                reject({
-                    code: STATUS_CODE_TIMEOUT,
-                    message: 'timeout',
-                    job_id: jobId
-                });
-            }, timeout);
-        });
-    }
-    async resolveOneTimeSubscription(context, jobOutput) {
-        //todo: subscriber should query for the job...only publish minimum context needed
-        if (this.hasOneTimeSubscription(context)) {
-            const message = {
-                type: 'job',
-                topic: context.metadata.jid,
-                job: restoreHierarchy(jobOutput),
-            };
-            this.store.publish(KeyType.QUORUM, message, this.appId, context.metadata.ngn);
-        }
-    }
-    async getPublishesTopic(context) {
-        const config = await this.getVID();
-        const activityId = context.metadata.aid || context['$self']?.output?.metadata?.aid;
-        const schema = await this.store.getSchema(activityId, config);
-        return schema.publishes;
-    }
-    async resolvePersistentSubscriptions(context, jobOutput) {
-        const topic = await this.getPublishesTopic(context);
-        if (topic) {
-            const message = {
-                type: 'job',
-                topic,
-                job: restoreHierarchy(jobOutput),
-            };
-            this.store.publish(KeyType.QUORUM, message, this.appId, `${topic}.${context.metadata.jid}`);
-        }
-    }
-    async add(streamData) {
-        return await this.streamSignaler.publishMessage(null, streamData);
-    }
-    registerJobCallback(jobId, jobCallback) {
-        this.jobCallbacks[jobId] = jobCallback;
-    }
-    delistJobCallback(jobId) {
-        delete this.jobCallbacks[jobId];
-    }
-    hasOneTimeSubscription(context) {
-        return Boolean(context.metadata.ngn);
-    }
-    // ***************** JOB COMPLETION/CLEANUP *****************
-    async runJobCompletionTasks(context) {
-        const isAwait = this.hasParentJob(context);
-        const isOneTimeSubscription = this.hasOneTimeSubscription(context);
-        const topic = await this.getPublishesTopic(context);
-        if (isAwait || isOneTimeSubscription || topic) {
-            const jobOutput = await this.getState(context.metadata.tpc, context.metadata.jid);
-            //always wait for stream pub/sub
-            await this.execAdjacentParent(context, jobOutput);
-            //no need to wait for standard pub/sub
-            this.resolveOneTimeSubscription(context, jobOutput);
-            this.resolvePersistentSubscriptions(context, jobOutput);
-        }
-        this.task.registerJobForCleanup(context.metadata.jid, context.metadata.expire);
-    }
-    // ****** GET JOB STATE/COLLATION STATUS BY ID *********
-    async getStatus(jobId) {
-        const { id: appId } = await this.getVID();
-        return this.store.getStatus(jobId, appId);
-    }
-    //todo: add 'options' parameter;
-    //      (e.g, if {dimensions:true}, use hscan to deliver
-    //      the full set of dimensional job data)
-    async getState(topic, jobId) {
-        const jobSymbols = await this.store.getSymbols(`$${topic}`);
-        const consumes = {
-            [`$${topic}`]: Object.keys(jobSymbols)
-        };
-        //job data exists at the 'zero' dimension; pass an empty object
-        const dIds = {};
-        const output = await this.store.getState(jobId, consumes, dIds);
-        if (!output) {
-            throw new Error(`not found ${jobId}`);
-        }
-        const [state, status] = output;
-        const stateTree = restoreHierarchy(state);
-        if (status && stateTree.metadata) {
-            stateTree.metadata.js = status;
-        }
-        return stateTree;
-    }
-    async compress(terms) {
-        const existingSymbols = await this.store.getSymbolValues();
-        const startIndex = Object.keys(existingSymbols).length;
-        const maxIndex = Math.pow(52, 2) - 1;
-        const newSymbols = SerializerService.filterSymVals(startIndex, maxIndex, existingSymbols, new Set(terms));
-        return await this.store.addSymbolValues(newSymbols);
-    }
-}
-export { EngineService };

package/build/esm/services/hotmesh/index.d.ts

@@ -1,45 +0,0 @@
-import { EngineService } from '../engine';
-import { ILogger } from '../logger';
-import { QuorumService } from '../quorum';
-import { WorkerService } from '../worker';
-import { JobState, JobData, JobOutput, JobStatus } from '../../types/job';
-import { HotMeshConfig, HotMeshManifest } from '../../types/hotmesh';
-import { JobMessageCallback } from '../../types/quorum';
-import { JobStatsInput, GetStatsOptions, IdsResponse, StatsResponse } from '../../types/stats';
-import { StreamData, StreamDataResponse } from '../../types/stream';
-declare class HotMeshService {
-    namespace: string;
-    appId: string;
-    guid: string;
-    engine: EngineService | null;
-    quorum: QuorumService | null;
-    workers: WorkerService[];
-    logger: ILogger;
-    verifyAndSetNamespace(namespace?: string): void;
-    verifyAndSetAppId(appId: string): void;
-    static init(config: HotMeshConfig): Promise<HotMeshService>;
-    initEngine(config: HotMeshConfig, logger: ILogger): Promise<void>;
-    initQuorum(config: HotMeshConfig, engine: EngineService, logger: ILogger): Promise<void>;
-    initWorkers(config: HotMeshConfig, logger: ILogger): Promise<void>;
-    pub(topic: string, data?: JobData, context?: JobState): Promise<string>;
-    sub(topic: string, callback: JobMessageCallback): Promise<void>;
-    unsub(topic: string): Promise<void>;
-    psub(wild: string, callback: JobMessageCallback): Promise<void>;
-    punsub(wild: string): Promise<void>;
-    pubsub(topic: string, data?: JobData, context?: JobState | null, timeout?: number): Promise<JobOutput>;
-    add(streamData: StreamData | StreamDataResponse): Promise<string>;
-    plan(path: string): Promise<HotMeshManifest>;
-    deploy(pathOrYAML: string): Promise<HotMeshManifest>;
-    activate(version: string, delay?: number): Promise<boolean>;
-    getStats(topic: string, query: JobStatsInput): Promise<StatsResponse>;
-    getStatus(jobId: string): Promise<JobStatus>;
-    getState(topic: string, jobId: string): Promise<JobOutput>;
-    getIds(topic: string, query: JobStatsInput, queryFacets?: any[]): Promise<IdsResponse>;
-    resolveQuery(topic: string, query: JobStatsInput): Promise<GetStatsOptions>;
-    scrub(jobId: string): Promise<void>;
-    hook(topic: string, data: JobData, dad?: string): Promise<JobStatus | void>;
-    hookAll(hookTopic: string, data: JobData, query: JobStatsInput, queryFacets?: string[]): Promise<string[]>;
-    stop(): Promise<void>;
-    compress(terms: string[]): Promise<boolean>;
-}
-export { HotMeshService };