@hotmeshio/hotmesh 0.4.0 → 0.4.1

package/build/services/reporter/index.js

@@ -0,0 +1,348 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ReporterService = void 0;
+const pipe_1 = require("../pipe");
+class ReporterService {
+    constructor(appVersion, store, logger) {
+        this.appVersion = appVersion;
+        this.logger = logger;
+        this.store = store;
+    }
+    async getStats(options) {
+        this.logger.debug('reporter-getstats-started', options);
+        const { key, granularity, range, end, start } = options;
+        this.validateOptions(options);
+        const dateTimeSets = this.generateDateTimeSets(granularity, range, end, start);
+        const redisKeys = dateTimeSets.map((dateTime) => this.buildRedisKey(key, dateTime));
+        const rawData = await this.store.getJobStats(redisKeys);
+        const [count, aggregatedData] = this.aggregateData(rawData);
+        const statsResponse = this.buildStatsResponse(rawData, redisKeys, aggregatedData, count, options);
+        return statsResponse;
+    }
+    validateOptions(options) {
+        const { start, end, range, granularity } = options;
+        if (granularity !== 'infinity' &&
+            (start && end && range || !start && !end && !range)) {
+            throw new Error('Invalid combination of start, end, and range values. Provide either start+end, end+range, or start+range.');
+        }
+    }
+    generateDateTimeSets(granularity, range, end, start) {
+        if (granularity === 'infinity') {
+            //if granularity is infinity, it means a date/time sequence/slice is not used to further segment the statistics
+            return ['0'];
+        }
+        if (!range) {
+            range = '0m';
+        }
+        const granularitiesInMinutes = {
+            '5m': 5,
+            '10m': 10,
+            '15m': 15,
+            '30m': 30,
+            '1h': 60,
+        };
+        const granularityMinutes = granularitiesInMinutes[granularity];
+        if (!granularityMinutes) {
+            throw new Error('Invalid granularity value.');
+        }
+        const rangeMinutes = this.convertRangeToMinutes(range);
+        if (rangeMinutes === null) {
+            throw new Error('Invalid range value.');
+        }
+        // If start is provided, use it. Otherwise, calculate it from the end time and range.
+        let startTime;
+        let endTime;
+        if (start) {
+            startTime = new Date(start);
+            endTime = new Date(startTime.getTime() + rangeMinutes * 60 * 1000);
+        }
+        else {
+            endTime = end === 'NOW' ? new Date() : new Date(end);
+            startTime = new Date(endTime.getTime() - rangeMinutes * 60 * 1000);
+        }
+        // Round the start time to the nearest granularity unit
+        startTime.setUTCMinutes(Math.floor(startTime.getUTCMinutes() / granularityMinutes) *
+            granularityMinutes);
+        const dateTimeSets = [];
+        for (let time = startTime; time <= endTime; time.setUTCMinutes(time.getUTCMinutes() + granularityMinutes)) {
+            const formattedTime = [
+                time.getUTCFullYear(),
+                String(time.getUTCMonth() + 1).padStart(2, '0'),
+                String(time.getUTCDate()).padStart(2, '0'),
+                String(time.getUTCHours()).padStart(2, '0'),
+                String(time.getUTCMinutes()).padStart(2, '0'),
+            ].join('');
+            dateTimeSets.push(formattedTime);
+        }
+        return dateTimeSets;
+    }
+    convertRangeToMinutes(range) {
+        const timeUnit = range.slice(-1);
+        const value = parseInt(range.slice(0, -1), 10);
+        if (isNaN(value)) {
+            return null;
+        }
+        switch (timeUnit) {
+            case 'm':
+                return value;
+            case 'h':
+                return value * 60;
+            case 'd':
+                return value * 60 * 24;
+            default:
+                return null;
+        }
+    }
+    buildRedisKey(key, dateTime, subTarget = '') {
+        return `hmsh:${this.appVersion.id}:s:${key}:${dateTime}${subTarget ? ':' + subTarget : ''}`;
+    }
+    aggregateData(rawData) {
+        const aggregatedData = {};
+        let count = 0;
+        Object.entries(rawData).forEach(([_, data]) => {
+            for (const key in data) {
+                if (key.startsWith('count:')) {
+                    const target = key.slice('count:'.length);
+                    if (!aggregatedData[target]) {
+                        aggregatedData[target] = 0;
+                    }
+                    aggregatedData[target] += data[key];
+                }
+                else if (key === 'count') {
+                    count += data[key];
+                }
+            }
+        });
+        return [count, aggregatedData];
+    }
+    buildStatsResponse(rawData, redisKeys, aggregatedData, count, options) {
+        const measures = [];
+        const measureKeys = Object.keys(aggregatedData).filter((key) => key !== 'count');
+        let segments = undefined;
+        if (options.sparse !== true) {
+            segments = this.handleSegments(rawData, redisKeys);
+        }
+        measureKeys.forEach((key) => {
+            const measure = {
+                target: key,
+                type: 'count',
+                value: aggregatedData[key],
+            };
+            measures.push(measure);
+        });
+        const response = {
+            key: options.key,
+            granularity: options.granularity,
+            range: options.range,
+            end: options.end,
+            count,
+            measures: measures,
+        };
+        if (segments) {
+            response.segments = segments;
+        }
+        return response;
+    }
+    handleSegments(data, hashKeys) {
+        const segments = [];
+        hashKeys.forEach((hashKey, index) => {
+            const segmentData = [];
+            data[hashKey] &&
+                Object.entries(data[hashKey]).forEach(([key, value]) => {
+                    if (key.startsWith('count:')) {
+                        const target = key.slice('count:'.length);
+                        segmentData.push({ target, type: 'count', value });
+                    }
+                });
+            const isoTimestamp = this.isoTimestampFromKeyTimestamp(hashKey);
+            const count = data[hashKey] ? data[hashKey].count : 0;
+            segments.push({ count, time: isoTimestamp, measures: segmentData });
+        });
+        return segments;
+    }
+    isoTimestampFromKeyTimestamp(hashKey) {
+        if (hashKey.endsWith(':')) {
+            return '0';
+        }
+        const keyTimestamp = hashKey.slice(-12);
+        const year = keyTimestamp.slice(0, 4);
+        const month = keyTimestamp.slice(4, 6);
+        const day = keyTimestamp.slice(6, 8);
+        const hour = keyTimestamp.slice(8, 10);
+        const minute = keyTimestamp.slice(10, 12);
+        return `${year}-${month}-${day}T${hour}:${minute}Z`;
+    }
+    async getIds(options, facets, idRange = [0, -1]) {
+        if (!facets.length) {
+            const stats = await this.getStats(options);
+            facets = this.getUniqueFacets(stats);
+        }
+        const { key, granularity, range, end, start } = options;
+        this.validateOptions(options);
+        let redisKeys = [];
+        facets.forEach((facet) => {
+            const dateTimeSets = this.generateDateTimeSets(granularity, range, end, start);
+            redisKeys = redisKeys.concat(dateTimeSets.map((dateTime) => this.buildRedisKey(key, dateTime, `index:${facet}`)));
+        });
+        const idsData = await this.store.getJobIds(redisKeys, idRange);
+        const idsResponse = this.buildIdsResponse(idsData, options, facets);
+        return idsResponse;
+    }
+    buildIdsResponse(idsData, options, facets) {
+        const countsByFacet = {};
+        const measureKeys = Object.keys(idsData);
+        measureKeys.forEach((key) => {
+            const target = this.getTargetForKey(key);
+            const count = idsData[key].length;
+            if (countsByFacet[target]) {
+                countsByFacet[target] += count;
+            }
+            else {
+                countsByFacet[target] = count;
+            }
+        });
+        const counts = Object.entries(countsByFacet).map(([facet, count]) => ({ facet, count }));
+        const response = {
+            key: options.key,
+            facets,
+            granularity: options.granularity,
+            range: options.range,
+            start: options.start,
+            counts,
+            segments: this.buildTimeSegments(idsData),
+        };
+        return response;
+    }
+    buildTimeSegments(idsData) {
+        const measureKeys = Object.keys(idsData);
+        const timeSegments = {};
+        measureKeys.forEach((key) => {
+            const measure = {
+                type: 'ids',
+                target: this.getTargetForKey(key),
+                time: this.isoTimestampFromKeyTimestamp(this.getTargetForTime(key)),
+                count: idsData[key].length,
+                ids: idsData[key],
+            };
+            if (timeSegments[measure.time]) {
+                timeSegments[measure.time].push(measure);
+            }
+            else {
+                timeSegments[measure.time] = [measure];
+            }
+        });
+        const segments = Object.entries(timeSegments).map(([time, measures]) => ({
+            time,
+            measures,
+        }));
+        return segments;
+    }
+    getUniqueFacets(data) {
+        const targets = data.measures.map((measure) => measure.target);
+        return Array.from(new Set(targets));
+    }
+    getTargetForKey(key) {
+        return key.split(':index:')[1];
+    }
+    getTargetForTime(key) {
+        return key.split(':index:')[0];
+    }
+    async getWorkItems(options, facets) {
+        if (!facets.length) {
+            const stats = await this.getStats(options);
+            facets = this.getUniqueFacets(stats);
+        }
+        const { key, granularity, range, end, start } = options;
+        this.validateOptions(options);
+        let redisKeys = [];
+        facets.forEach((facet) => {
+            const dateTimeSets = this.generateDateTimeSets(granularity, range, end, start);
+            redisKeys = redisKeys.concat(dateTimeSets.map((dateTime) => this.buildRedisKey(key, dateTime, `index:${facet}`)));
+        });
+        const idsData = await this.store.getJobIds(redisKeys, [0, 1]);
+        const workerLists = this.buildWorkerLists(idsData);
+        return workerLists;
+    }
+    buildWorkerLists(idsData) {
+        const workerLists = [];
+        for (const key in idsData) {
+            if (idsData[key].length) {
+                workerLists.push(key);
+            }
+        }
+        return workerLists;
+    }
+    /**
+     * called by `trigger` activity to generate the stats that should
+     * be saved to the database. doesn't actually save the stats, but
+     * just generates the info that should be saved
+     */
+    resolveTriggerStatistics({ stats: statsConfig }, context) {
+        const stats = {
+            general: [],
+            index: [],
+            median: [],
+        };
+        stats.general.push({ metric: 'count', target: 'count', value: 1 });
+        for (const measure of statsConfig.measures) {
+            const metric = this.resolveMetric({ metric: measure.measure, target: measure.target }, context);
+            if (this.isGeneralMetric(measure.measure)) {
+                stats.general.push(metric);
+            }
+            else if (this.isMedianMetric(measure.measure)) {
+                stats.median.push(metric);
+            }
+            else if (this.isIndexMetric(measure.measure)) {
+                stats.index.push(metric);
+            }
+        }
+        return stats;
+    }
+    isGeneralMetric(metric) {
+        return ReporterService.isGeneralMetric(metric);
+    }
+    isMedianMetric(metric) {
+        return ReporterService.isMedianMetric(metric);
+    }
+    isIndexMetric(metric) {
+        return ReporterService.isIndexMetric(metric);
+    }
+    static isGeneralMetric(metric) {
+        return metric === 'sum' || metric === 'avg' || metric === 'count';
+    }
+    static isMedianMetric(metric) {
+        return metric === 'mdn';
+    }
+    static isIndexMetric(metric) {
+        return metric === 'index';
+    }
+    resolveMetric({ metric, target }, context) {
+        const pipe = new pipe_1.Pipe([[target]], context);
+        const resolvedValue = pipe.process().toString();
+        const resolvedTarget = this.resolveTarget(metric, target, resolvedValue);
+        if (metric === 'index') {
+            return { metric, target: resolvedTarget, value: context.metadata.jid };
+        }
+        else if (metric === 'count') {
+            return { metric, target: resolvedTarget, value: 1 };
+        }
+        return { metric, target: resolvedTarget, value: resolvedValue };
+    }
+    isCardinalMetric(metric) {
+        return metric === 'index' || metric === 'count';
+    }
+    resolveTarget(metric, target, resolvedValue) {
+        const trimmed = target.substring(1, target.length - 1);
+        const trimmedTarget = trimmed.split('.').slice(3).join('/');
+        let resolvedTarget;
+        if (this.isCardinalMetric(metric)) {
+            resolvedTarget = `${metric}:${trimmedTarget}:${resolvedValue}`;
+        }
+        else {
+            resolvedTarget = `${metric}:${trimmedTarget}`;
+        }
+        return resolvedTarget;
+    }
+}
+exports.ReporterService = ReporterService;
+ReporterService.DEFAULT_GRANULARITY = '5m';

package/build/services/router/config/index.d.ts

@@ -0,0 +1,11 @@
+import { HMSH_BLOCK_TIME_MS, HMSH_MAX_RETRIES, HMSH_MAX_TIMEOUT_MS, HMSH_GRADUATED_INTERVAL_MS, HMSH_CODE_UNACKED, HMSH_CODE_UNKNOWN, HMSH_STATUS_UNKNOWN, HMSH_XCLAIM_COUNT, HMSH_XCLAIM_DELAY_MS, HMSH_XPENDING_COUNT, MAX_DELAY, MAX_STREAM_BACKOFF, INITIAL_STREAM_BACKOFF, MAX_STREAM_RETRIES } from '../../../modules/enums';
+import { RouterConfig } from '../../../types/stream';
+export declare class RouterConfigManager {
+    static validateThrottle(delayInMillis: number): void;
+    static setDefaults(config: RouterConfig): RouterConfig & {
+        reclaimDelay: number;
+        reclaimCount: number;
+        readonly: boolean;
+    };
+}
+export { HMSH_BLOCK_TIME_MS, HMSH_MAX_RETRIES, HMSH_MAX_TIMEOUT_MS, HMSH_GRADUATED_INTERVAL_MS, HMSH_CODE_UNACKED, HMSH_CODE_UNKNOWN, HMSH_STATUS_UNKNOWN, HMSH_XCLAIM_COUNT, HMSH_XCLAIM_DELAY_MS, HMSH_XPENDING_COUNT, MAX_DELAY, MAX_STREAM_BACKOFF, INITIAL_STREAM_BACKOFF, MAX_STREAM_RETRIES, };

package/build/services/router/config/index.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MAX_STREAM_RETRIES = exports.INITIAL_STREAM_BACKOFF = exports.MAX_STREAM_BACKOFF = exports.MAX_DELAY = exports.HMSH_XPENDING_COUNT = exports.HMSH_XCLAIM_DELAY_MS = exports.HMSH_XCLAIM_COUNT = exports.HMSH_STATUS_UNKNOWN = exports.HMSH_CODE_UNKNOWN = exports.HMSH_CODE_UNACKED = exports.HMSH_GRADUATED_INTERVAL_MS = exports.HMSH_MAX_TIMEOUT_MS = exports.HMSH_MAX_RETRIES = exports.HMSH_BLOCK_TIME_MS = exports.RouterConfigManager = void 0;
+const enums_1 = require("../../../modules/enums");
+Object.defineProperty(exports, "HMSH_BLOCK_TIME_MS", { enumerable: true, get: function () { return enums_1.HMSH_BLOCK_TIME_MS; } });
+Object.defineProperty(exports, "HMSH_MAX_RETRIES", { enumerable: true, get: function () { return enums_1.HMSH_MAX_RETRIES; } });
+Object.defineProperty(exports, "HMSH_MAX_TIMEOUT_MS", { enumerable: true, get: function () { return enums_1.HMSH_MAX_TIMEOUT_MS; } });
+Object.defineProperty(exports, "HMSH_GRADUATED_INTERVAL_MS", { enumerable: true, get: function () { return enums_1.HMSH_GRADUATED_INTERVAL_MS; } });
+Object.defineProperty(exports, "HMSH_CODE_UNACKED", { enumerable: true, get: function () { return enums_1.HMSH_CODE_UNACKED; } });
+Object.defineProperty(exports, "HMSH_CODE_UNKNOWN", { enumerable: true, get: function () { return enums_1.HMSH_CODE_UNKNOWN; } });
+Object.defineProperty(exports, "HMSH_STATUS_UNKNOWN", { enumerable: true, get: function () { return enums_1.HMSH_STATUS_UNKNOWN; } });
+Object.defineProperty(exports, "HMSH_XCLAIM_COUNT", { enumerable: true, get: function () { return enums_1.HMSH_XCLAIM_COUNT; } });
+Object.defineProperty(exports, "HMSH_XCLAIM_DELAY_MS", { enumerable: true, get: function () { return enums_1.HMSH_XCLAIM_DELAY_MS; } });
+Object.defineProperty(exports, "HMSH_XPENDING_COUNT", { enumerable: true, get: function () { return enums_1.HMSH_XPENDING_COUNT; } });
+Object.defineProperty(exports, "MAX_DELAY", { enumerable: true, get: function () { return enums_1.MAX_DELAY; } });
+Object.defineProperty(exports, "MAX_STREAM_BACKOFF", { enumerable: true, get: function () { return enums_1.MAX_STREAM_BACKOFF; } });
+Object.defineProperty(exports, "INITIAL_STREAM_BACKOFF", { enumerable: true, get: function () { return enums_1.INITIAL_STREAM_BACKOFF; } });
+Object.defineProperty(exports, "MAX_STREAM_RETRIES", { enumerable: true, get: function () { return enums_1.MAX_STREAM_RETRIES; } });
+class RouterConfigManager {
+    static validateThrottle(delayInMillis) {
+        if (!Number.isInteger(delayInMillis) ||
+            delayInMillis < 0 ||
+            delayInMillis > enums_1.MAX_DELAY) {
+            throw new Error(`Throttle must be a non-negative integer and not exceed ${enums_1.MAX_DELAY} ms; send -1 to throttle indefinitely`);
+        }
+    }
+    static setDefaults(config) {
+        return {
+            ...config,
+            reclaimDelay: config.reclaimDelay || enums_1.HMSH_XCLAIM_DELAY_MS,
+            reclaimCount: config.reclaimCount || enums_1.HMSH_XCLAIM_COUNT,
+            readonly: config.readonly || false,
+        };
+    }
+}
+exports.RouterConfigManager = RouterConfigManager;

package/build/services/router/consumption/index.d.ts

@@ -0,0 +1,34 @@
+import { ILogger } from '../../logger';
+import { StreamService } from '../../stream';
+import { ThrottleManager } from '../throttling';
+import { ErrorHandler } from '../error-handling';
+import { LifecycleManager } from '../lifecycle';
+import { StreamData, StreamDataResponse } from '../../../types/stream';
+import { ProviderClient, ProviderTransaction } from '../../../types/provider';
+export declare class ConsumptionManager<S extends StreamService<ProviderClient, ProviderTransaction>> {
+    private stream;
+    private logger;
+    private throttleManager;
+    private errorHandler;
+    private lifecycleManager;
+    private reclaimDelay;
+    private reclaimCount;
+    private appId;
+    private role;
+    private errorCount;
+    private counts;
+    private hasReachedMaxBackoff;
+    private router;
+    constructor(stream: S, logger: ILogger, throttleManager: ThrottleManager, errorHandler: ErrorHandler, lifecycleManager: LifecycleManager<S>, reclaimDelay: number, reclaimCount: number, appId: string, role: any, router: any);
+    createGroup(stream: string, group: string): Promise<void>;
+    publishMessage(topic: string, streamData: StreamData | StreamDataResponse, transaction?: ProviderTransaction): Promise<string | ProviderTransaction>;
+    consumeMessages(stream: string, group: string, consumer: string, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<void>;
+    private consumeWithNotifications;
+    private consumeWithPolling;
+    consumeOne(stream: string, group: string, id: string, input: StreamData, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<void>;
+    execStreamLeg(input: StreamData, stream: string, id: string, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<StreamDataResponse>;
+    ackAndDelete(stream: string, group: string, id: string): Promise<void>;
+    ackAndDeleteBatch(stream: string, group: string, ids: string[]): Promise<void>;
+    publishResponse(input: StreamData, output: StreamDataResponse | void): Promise<string>;
+    isStreamMessage(result: any): boolean;
+}