@hotmeshio/hotmesh 0.4.0 → 0.4.1

package/build/services/quorum/index.d.ts

@@ -0,0 +1,90 @@
+/// <reference types="node" />
+import { EngineService } from '../engine';
+import { ILogger } from '../logger';
+import { StoreService } from '../store';
+import { SubService } from '../sub';
+import { CacheMode } from '../../types/cache';
+import { HotMeshConfig } from '../../types/hotmesh';
+import { ProviderClient, ProviderTransaction } from '../../types/provider';
+import { QuorumMessage, QuorumMessageCallback, QuorumProfile, RollCallMessage, SubscriptionCallback } from '../../types/quorum';
+declare class QuorumService {
+    namespace: string;
+    appId: string;
+    guid: string;
+    engine: EngineService;
+    profiles: QuorumProfile[];
+    store: StoreService<ProviderClient, ProviderTransaction> | null;
+    subscribe: SubService<ProviderClient> | null;
+    logger: ILogger;
+    cacheMode: CacheMode;
+    untilVersion: string | null;
+    quorum: number | null;
+    callbacks: QuorumMessageCallback[];
+    rollCallInterval: NodeJS.Timeout;
+    /**
+     * @private
+     */
+    constructor();
+    /**
+     * @private
+     */
+    static init(namespace: string, appId: string, guid: string, config: HotMeshConfig, engine: EngineService, logger: ILogger): Promise<QuorumService>;
+    /**
+     * @private
+     */
+    verifyQuorumFields(config: HotMeshConfig): void;
+    /**
+     * @private
+     */
+    initStoreChannel(store: ProviderClient): Promise<void>;
+    /**
+     * @private
+     */
+    initSubChannel(sub: ProviderClient, store: ProviderClient): Promise<void>;
+    /**
+     * @private
+     */
+    subscriptionHandler(): SubscriptionCallback;
+    /**
+     * @private
+     */
+    sayPong(appId: string, guid: string, originator: string, details?: boolean): Promise<void>;
+    /**
+     * A quorum-wide command to request a quorum count.
+     * @private
+     */
+    requestQuorum(delay?: number, details?: boolean): Promise<number>;
+    /**
+     * @private
+     */
+    doRollCall(message: RollCallMessage): Promise<void>;
+    /**
+     * @private
+     */
+    cancelRollCall(): void;
+    /**
+     * @private
+     */
+    stop(): void;
+    /**
+     * @private
+     */
+    pub(quorumMessage: QuorumMessage): Promise<boolean>;
+    /**
+     * @private
+     */
+    sub(callback: QuorumMessageCallback): Promise<void>;
+    /**
+     * @private
+     */
+    unsub(callback: QuorumMessageCallback): Promise<void>;
+    /**
+     * @private
+     */
+    rollCall(delay?: number): Promise<QuorumProfile[]>;
+    /**
+     * @private
+     */
+    activate(version: string, delay?: number, count?: number): Promise<boolean>;
+}
+export { QuorumService };

package/build/services/quorum/index.js

@@ -0,0 +1,263 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.QuorumService = void 0;
+const enums_1 = require("../../modules/enums");
+const utils_1 = require("../../modules/utils");
+const compiler_1 = require("../compiler");
+const hotmesh_1 = require("../../types/hotmesh");
+const factory_1 = require("../sub/factory");
+const factory_2 = require("../store/factory");
+class QuorumService {
+    /**
+     * @private
+     */
+    constructor() {
+        this.profiles = [];
+        this.cacheMode = 'cache';
+        this.untilVersion = null;
+        this.quorum = null;
+        this.callbacks = [];
+    }
+    /**
+     * @private
+     */
+    static async init(namespace, appId, guid, config, engine, logger) {
+        if (config.engine) {
+            const instance = new QuorumService();
+            instance.verifyQuorumFields(config);
+            instance.namespace = namespace;
+            instance.appId = appId;
+            instance.guid = guid;
+            instance.logger = logger;
+            instance.engine = engine;
+            await instance.initStoreChannel(config.engine.store);
+            await instance.initSubChannel(config.engine.sub, config.engine.pub ?? config.engine.store);
+            //general quorum subscription
+            await instance.subscribe.subscribe(hotmesh_1.KeyType.QUORUM, instance.subscriptionHandler(), appId);
+            //app-specific quorum subscription (used for pubsub one-time request/response)
+            await instance.subscribe.subscribe(hotmesh_1.KeyType.QUORUM, instance.subscriptionHandler(), appId, instance.guid);
+            instance.engine.processWebHooks();
+            instance.engine.processTimeHooks();
+            return instance;
+        }
+    }
+    /**
+     * @private
+     */
+    verifyQuorumFields(config) {
+        if (!(0, utils_1.identifyProvider)(config.engine.store) ||
+            !(0, utils_1.identifyProvider)(config.engine.sub)) {
+            throw new Error('quorum config must include `store` and `sub` fields.');
+        }
+    }
+    /**
+     * @private
+     */
+    async initStoreChannel(store) {
+        this.store = await factory_2.StoreServiceFactory.init(store, this.namespace, this.appId, this.logger);
+    }
+    /**
+     * @private
+     */
+    async initSubChannel(sub, store) {
+        this.subscribe = await factory_1.SubServiceFactory.init(sub, store, this.namespace, this.appId, this.guid, this.logger);
+    }
+    /**
+     * @private
+     */
+    subscriptionHandler() {
+        const self = this;
+        return async (topic, message) => {
+            self.logger.debug('quorum-event-received', { topic, type: message.type });
+            if (message.type === 'activate') {
+                self.engine.setCacheMode(message.cache_mode, message.until_version);
+            }
+            else if (message.type === 'ping') {
+                self.sayPong(self.appId, self.guid, message.originator, message.details);
+            }
+            else if (message.type === 'pong' && self.guid === message.originator) {
+                self.quorum = self.quorum + 1;
+                if (message.profile) {
+                    self.profiles.push(message.profile);
+                }
+            }
+            else if (message.type === 'throttle') {
+                self.engine.throttle(message.throttle);
+            }
+            else if (message.type === 'work') {
+                self.engine.processWebHooks();
+            }
+            else if (message.type === 'job') {
+                self.engine.routeToSubscribers(message.topic, message.job);
+            }
+            else if (message.type === 'cron') {
+                self.engine.processTimeHooks();
+            }
+            else if (message.type === 'rollcall') {
+                self.doRollCall(message);
+            }
+            //if there are any callbacks, call them
+            if (self.callbacks.length > 0) {
+                self.callbacks.forEach((cb) => cb(topic, message));
+            }
+        };
+    }
+    /**
+     * @private
+     */
+    async sayPong(appId, guid, originator, details = false) {
+        let profile;
+        if (details) {
+            const stream = this.engine.store.mintKey(hotmesh_1.KeyType.STREAMS, {
+                appId: this.appId,
+            });
+            profile = {
+                engine_id: this.guid,
+                namespace: this.namespace,
+                app_id: this.appId,
+                stream,
+                counts: this.engine.router.counts,
+                timestamp: (0, utils_1.formatISODate)(new Date()),
+                inited: this.engine.inited,
+                throttle: this.engine.router.throttle,
+                reclaimDelay: this.engine.router.reclaimDelay,
+                reclaimCount: this.engine.router.reclaimCount,
+                system: await (0, utils_1.getSystemHealth)(),
+            };
+        }
+        this.subscribe.publish(hotmesh_1.KeyType.QUORUM, {
+            type: 'pong',
+            guid,
+            originator,
+            profile,
+        }, appId);
+    }
+    /**
+     * A quorum-wide command to request a quorum count.
+     * @private
+     */
+    async requestQuorum(delay = enums_1.HMSH_QUORUM_DELAY_MS, details = false) {
+        const quorum = this.quorum;
+        this.quorum = 0;
+        this.profiles.length = 0;
+        await this.subscribe.publish(hotmesh_1.KeyType.QUORUM, {
+            type: 'ping',
+            originator: this.guid,
+            details,
+        }, this.appId);
+        await (0, utils_1.sleepFor)(delay);
+        return quorum;
+    }
+    /**
+     * @private
+     */
+    async doRollCall(message) {
+        let iteration = 0;
+        const max = !isNaN(message.max) ? message.max : enums_1.HMSH_QUORUM_ROLLCALL_CYCLES;
+        if (this.rollCallInterval)
+            clearTimeout(this.rollCallInterval);
+        const base = message.interval / 2;
+        const amount = base + Math.ceil(Math.random() * base);
+        do {
+            await (0, utils_1.sleepFor)(Math.ceil(Math.random() * 1000));
+            await this.sayPong(this.appId, this.guid, null, true);
+            if (!message.interval)
+                return;
+            const { promise, timerId } = (0, utils_1.XSleepFor)(amount * 1000);
+            this.rollCallInterval = timerId;
+            await promise;
+        } while (this.rollCallInterval && iteration++ < max - 1);
+    }
+    /**
+     * @private
+     */
+    cancelRollCall() {
+        if (this.rollCallInterval) {
+            clearTimeout(this.rollCallInterval);
+            delete this.rollCallInterval;
+        }
+    }
+    /**
+     * @private
+     */
+    stop() {
+        this.cancelRollCall();
+    }
+    /**
+     * @private
+     */
+    async pub(quorumMessage) {
+        return await this.subscribe.publish(hotmesh_1.KeyType.QUORUM, quorumMessage, this.appId, quorumMessage.topic || quorumMessage.guid);
+    }
+    /**
+     * @private
+     */
+    async sub(callback) {
+        this.callbacks.push(callback);
+    }
+    /**
+     * @private
+     */
+    async unsub(callback) {
+        this.callbacks = this.callbacks.filter((cb) => cb !== callback);
+    }
+    /**
+     * @private
+     */
+    async rollCall(delay = enums_1.HMSH_QUORUM_DELAY_MS) {
+        await this.requestQuorum(delay, true);
+        const stream_depths = await this.engine.stream.getStreamDepths(this.profiles);
+        this.profiles.forEach(async (profile, index) => {
+            //if nothing in the table, the depth will be 0
+            //todo: separate table for every worker stream?
+            profile.stream_depth = stream_depths?.[index]?.depth ?? 0;
+        });
+        return this.profiles;
+    }
+    /**
+     * @private
+     */
+    async activate(version, delay = enums_1.HMSH_QUORUM_DELAY_MS, count = 0) {
+        version = version.toString();
+        const canActivate = await this.store.reserveScoutRole('activate', Math.ceil(delay * 6 / 1000) + 1);
+        if (!canActivate) {
+            //another engine is already activating the app version
+            this.logger.debug('quorum-activation-awaiting', { version });
+            await (0, utils_1.sleepFor)(delay * 6);
+            const app = await this.store.getApp(this.appId, true);
+            return app?.active == true && app?.version === version;
+        }
+        const config = await this.engine.getVID();
+        await this.requestQuorum(delay);
+        const q1 = await this.requestQuorum(delay);
+        const q2 = await this.requestQuorum(delay);
+        const q3 = await this.requestQuorum(delay);
+        if (q1 && q1 === q2 && q2 === q3) {
+            this.logger.info('quorum-rollcall-succeeded', { q1, q2, q3 });
+            this.subscribe.publish(hotmesh_1.KeyType.QUORUM, { type: 'activate', cache_mode: 'nocache', until_version: version }, this.appId);
+            await new Promise((resolve) => setTimeout(resolve, delay));
+            await this.store.releaseScoutRole('activate');
+            //confirm we received the activation message
+            if (this.engine.untilVersion === version) {
+                this.logger.info('quorum-activation-succeeded', { version });
+                const { id } = config;
+                const compiler = new compiler_1.CompilerService(this.store, this.engine.stream, this.logger);
+                return await compiler.activate(id, version);
+            }
+            else {
+                this.logger.error('quorum-activation-error', { version });
+                throw new Error(`UntilVersion Not Received. Version ${version} not activated`);
+            }
+        }
+        else {
+            this.logger.warn('quorum-rollcall-error', { q1, q2, q3, count });
+            this.store.releaseScoutRole('activate');
+            if (count < enums_1.HMSH_ACTIVATION_MAX_RETRY) {
+                //increase the delay (give the quorum time to respond) and try again
+                return await this.activate(version, delay * 2, count + 1);
+            }
+            throw new Error(`Quorum not reached. Version ${version} not activated.`);
+        }
+    }
+}
+exports.QuorumService = QuorumService;

package/build/services/reporter/index.d.ts

@@ -0,0 +1,50 @@
+import { ILogger } from '../logger';
+import { StoreService } from '../store';
+import { TriggerActivity } from '../../types/activity';
+import { AppVID } from '../../types/app';
+import { ProviderClient, ProviderTransaction } from '../../types/provider';
+import { JobState } from '../../types/job';
+import { GetStatsOptions, StatsResponse, IdsResponse, StatsType, StatType } from '../../types/stats';
+declare class ReporterService {
+    private appVersion;
+    private logger;
+    private store;
+    static DEFAULT_GRANULARITY: string;
+    constructor(appVersion: AppVID, store: StoreService<ProviderClient, ProviderTransaction>, logger: ILogger);
+    getStats(options: GetStatsOptions): Promise<StatsResponse>;
+    private validateOptions;
+    private generateDateTimeSets;
+    private convertRangeToMinutes;
+    private buildRedisKey;
+    private aggregateData;
+    private buildStatsResponse;
+    private handleSegments;
+    private isoTimestampFromKeyTimestamp;
+    getIds(options: GetStatsOptions, facets: string[], idRange?: [number, number]): Promise<IdsResponse>;
+    private buildIdsResponse;
+    private buildTimeSegments;
+    getUniqueFacets(data: StatsResponse): string[];
+    getTargetForKey(key: string): string;
+    getTargetForTime(key: string): string;
+    getWorkItems(options: GetStatsOptions, facets: string[]): Promise<string[]>;
+    private buildWorkerLists;
+    /**
+     * called by `trigger` activity to generate the stats that should
+     * be saved to the database. doesn't actually save the stats, but
+     * just generates the info that should be saved
+     */
+    resolveTriggerStatistics({ stats: statsConfig }: TriggerActivity, context: JobState): StatsType;
+    isGeneralMetric(metric: string): boolean;
+    isMedianMetric(metric: string): boolean;
+    isIndexMetric(metric: string): boolean;
+    static isGeneralMetric(metric: string): boolean;
+    static isMedianMetric(metric: string): boolean;
+    static isIndexMetric(metric: string): boolean;
+    resolveMetric({ metric, target }: {
+        metric: any;
+        target: any;
+    }, context: JobState): StatType;
+    isCardinalMetric(metric: string): boolean;
+    resolveTarget(metric: string, target: string, resolvedValue: string): string;
+}
+export { ReporterService };