@hotmeshio/hotmesh 0.0.8 → 0.0.10

package/build/esm/services/reporter/index.js

@@ -1,327 +0,0 @@
-import { Pipe } from '../pipe';
-class ReporterService {
-    constructor(appVersion, store, logger) {
-        this.appVersion = appVersion;
-        this.logger = logger;
-        this.store = store;
-    }
-    async getStats(options) {
-        this.logger.debug('reporter-getstats-started', options);
-        const { key, granularity, range, end, start } = options;
-        this.validateOptions(options);
-        const dateTimeSets = this.generateDateTimeSets(granularity, range, end, start);
-        const redisKeys = dateTimeSets.map((dateTime) => this.buildRedisKey(key, dateTime));
-        const rawData = await this.store.getJobStats(redisKeys);
-        const [count, aggregatedData] = this.aggregateData(rawData);
-        const statsResponse = this.buildStatsResponse(rawData, redisKeys, aggregatedData, count, options);
-        return statsResponse;
-    }
-    validateOptions(options) {
-        const { start, end, range } = options;
-        if (start && end && range || !start && !end && !range) {
-            throw new Error('Invalid combination of start, end, and range values. Provide either start+end, end+range, or start+range.');
-        }
-    }
-    generateDateTimeSets(granularity, range, end, start) {
-        if (!range) {
-            //pluck just a single value when no range provided
-            range = '0m';
-        }
-        const granularitiesInMinutes = {
-            '5m': 5,
-            '10m': 10,
-            '15m': 15,
-            '30m': 30,
-            '1h': 60,
-        };
-        const granularityMinutes = granularitiesInMinutes[granularity];
-        if (!granularityMinutes) {
-            throw new Error('Invalid granularity value.');
-        }
-        const rangeMinutes = this.convertRangeToMinutes(range);
-        if (rangeMinutes === null) {
-            throw new Error('Invalid range value.');
-        }
-        // If start is provided, use it. Otherwise, calculate it from the end time and range.
-        let startTime;
-        let endTime;
-        if (start) {
-            startTime = new Date(start);
-            endTime = new Date(startTime.getTime() + rangeMinutes * 60 * 1000);
-        }
-        else {
-            endTime = end === 'NOW' ? new Date() : new Date(end);
-            startTime = new Date(endTime.getTime() - rangeMinutes * 60 * 1000);
-        }
-        // Round the start time to the nearest granularity unit
-        startTime.setUTCMinutes(Math.floor(startTime.getUTCMinutes() / granularityMinutes) * granularityMinutes);
-        const dateTimeSets = [];
-        for (let time = startTime; time <= endTime; time.setUTCMinutes(time.getUTCMinutes() + granularityMinutes)) {
-            const formattedTime = [
-                time.getUTCFullYear(),
-                String(time.getUTCMonth() + 1).padStart(2, '0'),
-                String(time.getUTCDate()).padStart(2, '0'),
-                String(time.getUTCHours()).padStart(2, '0'),
-                String(time.getUTCMinutes()).padStart(2, '0'),
-            ].join('');
-            dateTimeSets.push(formattedTime);
-        }
-        return dateTimeSets;
-    }
-    convertRangeToMinutes(range) {
-        const timeUnit = range.slice(-1);
-        const value = parseInt(range.slice(0, -1), 10);
-        if (isNaN(value)) {
-            return null;
-        }
-        switch (timeUnit) {
-            case 'm':
-                return value;
-            case 'h':
-                return value * 60;
-            case 'd':
-                return value * 60 * 24;
-            default:
-                return null;
-        }
-    }
-    buildRedisKey(key, dateTime, subTarget = '') {
-        return `hmsh:${this.appVersion.id}:s:${key}:${dateTime}${subTarget ? ':' + subTarget : ''}`;
-    }
-    aggregateData(rawData) {
-        const aggregatedData = {};
-        let count = 0;
-        Object.entries(rawData).forEach(([_, data]) => {
-            for (const key in data) {
-                if (key.startsWith('count:')) {
-                    const target = key.slice('count:'.length);
-                    if (!aggregatedData[target]) {
-                        aggregatedData[target] = 0;
-                    }
-                    aggregatedData[target] += data[key];
-                }
-                else if (key === 'count') {
-                    count += data[key];
-                }
-            }
-        });
-        return [count, aggregatedData];
-    }
-    buildStatsResponse(rawData, redisKeys, aggregatedData, count, options) {
-        const measures = [];
-        const measureKeys = Object.keys(aggregatedData).filter((key) => key !== "count");
-        let segments = undefined;
-        if (options.sparse !== true) {
-            segments = this.handleSegments(rawData, redisKeys);
-        }
-        measureKeys.forEach((key) => {
-            const measure = {
-                target: key,
-                type: "count",
-                value: aggregatedData[key],
-            };
-            measures.push(measure);
-        });
-        const response = {
-            key: options.key,
-            granularity: options.granularity,
-            range: options.range,
-            end: options.end,
-            count,
-            measures: measures,
-        };
-        if (segments) {
-            response.segments = segments;
-        }
-        return response;
-    }
-    handleSegments(data, hashKeys) {
-        const segments = [];
-        hashKeys.forEach((hashKey, index) => {
-            const segmentData = [];
-            data[hashKey] && Object.entries(data[hashKey]).forEach(([key, value]) => {
-                if (key.startsWith('count:')) {
-                    const target = key.slice('count:'.length);
-                    segmentData.push({ target, type: 'count', value });
-                }
-            });
-            const isoTimestamp = this.isoTimestampFromKeyTimestamp(hashKey);
-            const count = data[hashKey] ? data[hashKey].count : 0;
-            segments.push({ count, time: isoTimestamp, measures: segmentData });
-        });
-        return segments;
-    }
-    isoTimestampFromKeyTimestamp(hashKey) {
-        const keyTimestamp = hashKey.slice(-12);
-        const year = keyTimestamp.slice(0, 4);
-        const month = keyTimestamp.slice(4, 6);
-        const day = keyTimestamp.slice(6, 8);
-        const hour = keyTimestamp.slice(8, 10);
-        const minute = keyTimestamp.slice(10, 12);
-        return `${year}-${month}-${day}T${hour}:${minute}Z`;
-    }
-    async getIds(options, facets, idRange = [0, -1]) {
-        if (!facets.length) {
-            const stats = await this.getStats(options);
-            facets = this.getUniqueFacets(stats);
-        }
-        const { key, granularity, range, end, start } = options;
-        this.validateOptions(options);
-        let redisKeys = [];
-        facets.forEach((facet) => {
-            const dateTimeSets = this.generateDateTimeSets(granularity, range, end, start);
-            redisKeys = redisKeys.concat(dateTimeSets.map((dateTime) => this.buildRedisKey(key, dateTime, `index:${facet}`)));
-        });
-        const idsData = await this.store.getJobIds(redisKeys, idRange);
-        const idsResponse = this.buildIdsResponse(idsData, options, facets);
-        return idsResponse;
-    }
-    buildIdsResponse(idsData, options, facets) {
-        const countsByFacet = {};
-        const measureKeys = Object.keys(idsData);
-        measureKeys.forEach((key) => {
-            const target = this.getTargetForKey(key);
-            const count = idsData[key].length;
-            if (countsByFacet[target]) {
-                countsByFacet[target] += count;
-            }
-            else {
-                countsByFacet[target] = count;
-            }
-        });
-        const counts = Object.entries(countsByFacet).map(([facet, count]) => ({ facet, count }));
-        const response = {
-            key: options.key,
-            facets,
-            granularity: options.granularity,
-            range: options.range,
-            start: options.start,
-            counts,
-            segments: this.buildTimeSegments(idsData),
-        };
-        return response;
-    }
-    buildTimeSegments(idsData) {
-        const measureKeys = Object.keys(idsData);
-        const timeSegments = {};
-        measureKeys.forEach((key) => {
-            const measure = {
-                type: 'ids',
-                target: this.getTargetForKey(key),
-                time: this.isoTimestampFromKeyTimestamp(this.getTargetForTime(key)),
-                count: idsData[key].length,
-                ids: idsData[key],
-            };
-            if (timeSegments[measure.time]) {
-                timeSegments[measure.time].push(measure);
-            }
-            else {
-                timeSegments[measure.time] = [measure];
-            }
-        });
-        const segments = Object.entries(timeSegments).map(([time, measures]) => ({
-            time,
-            measures,
-        }));
-        return segments;
-    }
-    getUniqueFacets(data) {
-        const targets = data.measures.map(measure => measure.target);
-        return Array.from(new Set(targets));
-    }
-    getTargetForKey(key) {
-        return key.split(':index:')[1];
-    }
-    getTargetForTime(key) {
-        return key.split(':index:')[0];
-    }
-    async getWorkItems(options, facets) {
-        if (!facets.length) {
-            const stats = await this.getStats(options);
-            facets = this.getUniqueFacets(stats);
-        }
-        const { key, granularity, range, end, start } = options;
-        this.validateOptions(options);
-        let redisKeys = [];
-        facets.forEach((facet) => {
-            const dateTimeSets = this.generateDateTimeSets(granularity, range, end, start);
-            redisKeys = redisKeys.concat(dateTimeSets.map((dateTime) => this.buildRedisKey(key, dateTime, `index:${facet}`)));
-        });
-        const idsData = await this.store.getJobIds(redisKeys, [0, 1]);
-        const workerLists = this.buildWorkerLists(idsData);
-        return workerLists;
-    }
-    buildWorkerLists(idsData) {
-        const workerLists = [];
-        for (const key in idsData) {
-            if (idsData[key].length) {
-                workerLists.push(key);
-            }
-        }
-        return workerLists;
-    }
-    /**
-     * called by `trigger` activity to generate the stats that should
-     * be saved to the database. doesn't actually save the stats, but
-     * just generates the info that should be saved
-     */
-    resolveTriggerStatistics({ stats: statsConfig }, context) {
-        const stats = {
-            general: [],
-            index: [],
-            median: []
-        };
-        stats.general.push({ metric: 'count', target: 'count', value: 1 });
-        for (const measure of statsConfig.measures) {
-            const metric = this.resolveMetric({ metric: measure.measure, target: measure.target }, context);
-            if (this.isGeneralMetric(measure.measure)) {
-                stats.general.push(metric);
-            }
-            else if (this.isMedianMetric(measure.measure)) {
-                stats.median.push(metric);
-            }
-            else if (this.isIndexMetric(measure.measure)) {
-                stats.index.push(metric);
-            }
-        }
-        return stats;
-    }
-    isGeneralMetric(metric) {
-        return metric === 'sum' || metric === 'avg' || metric === 'count';
-    }
-    isMedianMetric(metric) {
-        return metric === 'mdn';
-    }
-    isIndexMetric(metric) {
-        return metric === 'index';
-    }
-    resolveMetric({ metric, target }, context) {
-        const pipe = new Pipe([[target]], context);
-        const resolvedValue = pipe.process().toString();
-        const resolvedTarget = this.resolveTarget(metric, target, resolvedValue);
-        if (metric === 'index') {
-            return { metric, target: resolvedTarget, value: context.metadata.jid };
-        }
-        else if (metric === 'count') {
-            return { metric, target: resolvedTarget, value: 1 };
-        }
-        return { metric, target: resolvedTarget, value: resolvedValue };
-    }
-    isCardinalMetric(metric) {
-        return metric === 'index' || metric === 'count';
-    }
-    resolveTarget(metric, target, resolvedValue) {
-        const trimmed = target.substring(1, target.length - 1);
-        const trimmedTarget = trimmed.split('.').slice(3).join('/');
-        let resolvedTarget;
-        if (this.isCardinalMetric(metric)) {
-            resolvedTarget = `${metric}:${trimmedTarget}:${resolvedValue}`;
-        }
-        else {
-            resolvedTarget = `${metric}:${trimmedTarget}`;
-        }
-        return resolvedTarget;
-    }
-}
-ReporterService.DEFAULT_GRANULARITY = '5m';
-export { ReporterService };

package/build/esm/services/serializer/index.d.ts

@@ -1,41 +0,0 @@
-import { Consumes } from '../../types/activity';
-import { StringStringType, StringAnyType, SymbolMap, SymbolMaps, SymbolSets, Symbols } from '../../types/serializer';
-export declare const MDATA_SYMBOLS: {
-    SLOTS: number;
-    ACTIVITY: {
-        KEYS: string[];
-    };
-    ACTIVITY_UPDATE: {
-        KEYS: string[];
-    };
-    JOB: {
-        KEYS: string[];
-    };
-    JOB_UPDATE: {
-        KEYS: string[];
-    };
-};
-export declare class SerializerService {
-    dIds: StringStringType;
-    symKeys: SymbolMaps;
-    symReverseKeys: SymbolMaps;
-    symValMaps: SymbolMap;
-    symValReverseMaps: SymbolMap;
-    constructor();
-    abbreviate(consumes: Consumes, symbolNames: string[], fields?: string[]): string[];
-    resolveDimensionalIndex(path: string): string;
-    isJobPath(path: string): boolean;
-    resetSymbols(symKeys: SymbolSets, symVals: Symbols, dIds: StringStringType): void;
-    getReverseKeyMap(keyMap: SymbolMap, id?: string): SymbolMap;
-    getReverseValueMap(valueMap: SymbolMap): SymbolMap;
-    static filterSymVals(startIndex: number, maxIndex: number, existingSymbolValues: Symbols, proposedValues: Set<string>): Symbols;
-    compress(document: StringStringType, ids: string[]): StringStringType;
-    decompress(document: StringStringType, ids: string[]): StringStringType;
-    stringify(document: Record<string, any>): StringStringType;
-    parse(document: StringStringType): any;
-    toString(value: any): string | undefined;
-    fromString(value: string | undefined): any;
-    package(multiDimensionalDocument: StringAnyType, ids: string[]): StringStringType;
-    unpackage(document: StringStringType, ids: string[]): StringAnyType;
-    export(): SymbolSets;
-}

package/build/esm/services/serializer/index.js

@@ -1,251 +0,0 @@
-import { getSymVal } from '../../modules/utils';
-const dateReg = /^"\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z)?"$/;
-export const MDATA_SYMBOLS = {
-    SLOTS: 26,
-    ACTIVITY: {
-        KEYS: ['aid', 'dad', 'as', 'atp', 'stp', 'ac', 'au', 'err', 'l1s', 'l2s']
-    },
-    ACTIVITY_UPDATE: {
-        KEYS: ['au', 'err', 'l2s']
-    },
-    JOB: {
-        KEYS: ['ngn', 'tpc', 'pj', 'pd', 'pa', 'key', 'app', 'vrs', 'jid', 'aid', 'ts', 'jc', 'ju', 'js', 'err', 'trc']
-    },
-    JOB_UPDATE: {
-        KEYS: ['ju', 'err']
-    }
-};
-export class SerializerService {
-    constructor() {
-        this.resetSymbols({}, {}, {});
-    }
-    abbreviate(consumes, symbolNames, fields = []) {
-        for (const symbolName of symbolNames) {
-            const symbolSet = this.symKeys.get(symbolName);
-            const symbolPaths = consumes[symbolName];
-            for (const symbolPath of symbolPaths) {
-                const abbreviation = symbolSet.get(symbolPath);
-                if (abbreviation) {
-                    const dimensionalIndex = this.resolveDimensionalIndex(symbolPath);
-                    fields.push(`${abbreviation}${dimensionalIndex}`);
-                }
-                else {
-                    fields.push(symbolPath);
-                }
-            }
-        }
-        return fields;
-    }
-    resolveDimensionalIndex(path) {
-        if (this.isJobPath(path)) {
-            return '';
-        }
-        else {
-            const [activityId] = path.split('/');
-            if (activityId in this.dIds) {
-                return this.dIds[activityId];
-            }
-            else if ('$ADJACENT' in this.dIds) {
-                //else=> pre-authorizing adjacent activity entry
-                return this.dIds['$ADJACENT'];
-            }
-            return ',0';
-        }
-    }
-    isJobPath(path) {
-        return path.startsWith('data/') || path.startsWith('metadata/');
-    }
-    resetSymbols(symKeys, symVals, dIds) {
-        this.symKeys = new Map();
-        this.symReverseKeys = new Map();
-        for (const id in symKeys) {
-            this.symKeys.set(id, new Map(Object.entries(symKeys[id])));
-        }
-        this.symValMaps = new Map(Object.entries(symVals));
-        this.symValReverseMaps = this.getReverseValueMap(this.symValMaps);
-        this.dIds = dIds;
-    }
-    getReverseKeyMap(keyMap, id) {
-        let map = this.symReverseKeys.get(id);
-        if (!map) {
-            map = new Map();
-            for (let [key, val] of keyMap.entries()) {
-                map.set(val, key);
-            }
-            this.symReverseKeys.set(id, map);
-        }
-        return map;
-    }
-    getReverseValueMap(valueMap) {
-        const map = new Map();
-        for (let [key, val] of valueMap.entries()) {
-            map.set(val, key);
-        }
-        return map;
-    }
-    static filterSymVals(startIndex, maxIndex, existingSymbolValues, proposedValues) {
-        let newSymbolValues = {};
-        let currentSymbolValues = { ...existingSymbolValues };
-        let currentValuesSet = new Set(Object.values(currentSymbolValues));
-        for (let value of proposedValues) {
-            if (!currentValuesSet.has(value)) {
-                if (startIndex > maxIndex) {
-                    return newSymbolValues;
-                }
-                const symbol = getSymVal(startIndex);
-                startIndex++;
-                newSymbolValues[symbol] = value;
-                currentValuesSet.add(value);
-            }
-        }
-        return newSymbolValues;
-    }
-    compress(document, ids) {
-        if (this.symKeys.size === 0) {
-            return document;
-        }
-        let source = { ...document };
-        let result = {};
-        const compressWithMap = (abbreviationMap, id) => {
-            for (let key in source) {
-                if (key.startsWith(`${id}/`) || (id.startsWith('$') && ['data', 'metadata'].includes(key.split('/')[0]))) {
-                    const dimensionalIndex = this.resolveDimensionalIndex(key);
-                    let shortKey = abbreviationMap.get(key) || key;
-                    const shortDimensionalKey = `${shortKey}${dimensionalIndex}`;
-                    result[shortDimensionalKey] = source[key];
-                }
-            }
-        };
-        for (let id of ids) {
-            const abbreviationMap = this.symKeys.get(id);
-            if (abbreviationMap) {
-                compressWithMap(abbreviationMap, id);
-            }
-        }
-        return result;
-    }
-    decompress(document, ids) {
-        if (this.symKeys.size === 0) {
-            return document;
-        }
-        let result = { ...document };
-        const inflateWithMap = (abbreviationMap, id) => {
-            const reversedAbbreviationMap = this.getReverseKeyMap(abbreviationMap, id);
-            for (let key in result) {
-                //strip dimensional index from key
-                const shortKey = key.split(',')[0];
-                let longKey = reversedAbbreviationMap.get(shortKey);
-                if (longKey) {
-                    result[longKey] = result[key];
-                    delete result[key];
-                }
-            }
-        };
-        for (let id of ids) {
-            const abbreviationMap = this.symKeys.get(id);
-            if (abbreviationMap) {
-                inflateWithMap(abbreviationMap, id);
-            }
-        }
-        return result;
-    }
-    //stringify: convert a multi-dimensional document to a 2-d hash
-    stringify(document) {
-        let result = {};
-        for (let key in document) {
-            let value = this.toString(document[key]);
-            if (value) {
-                if (/^:*[a-zA-Z]{2}$/.test(value)) {
-                    value = ':' + value;
-                }
-                else if (this.symValReverseMaps.has(value)) {
-                    value = this.symValReverseMaps.get(value);
-                }
-                result[key] = value;
-            }
-        }
-        return result;
-    }
-    //parse: convert a 2-d hash to a multi-dimensional document
-    parse(document) {
-        let result = {};
-        for (let [key, value] of Object.entries(document)) {
-            if (value === undefined || value === null)
-                continue;
-            if (/^:+[a-zA-Z]{2}$/.test(value)) {
-                result[key] = value.slice(1);
-            }
-            else {
-                if (value?.length === 2 && this.symValMaps.has(value)) {
-                    value = this.symValMaps.get(value);
-                }
-                result[key] = this.fromString(value);
-            }
-        }
-        return result;
-    }
-    toString(value) {
-        switch (typeof value) {
-            case 'string':
-                break;
-            case 'boolean':
-                value = value ? '/t' : '/f';
-                break;
-            case 'number':
-                value = '/d' + value.toString();
-                break;
-            case 'undefined':
-                return undefined;
-            case 'object':
-                if (value === null) {
-                    value = '/n';
-                }
-                else {
-                    value = '/s' + JSON.stringify(value);
-                }
-                break;
-        }
-        return value;
-    }
-    fromString(value) {
-        if (typeof value !== 'string')
-            return undefined;
-        const prefix = value.slice(0, 2);
-        const rest = value.slice(2);
-        switch (prefix) {
-            case '/t': // boolean true
-                return true;
-            case '/f': // boolean false
-                return false;
-            case '/d': // number
-                return Number(rest);
-            case '/n': // null
-                return null;
-            case '/s': // object (JSON string)
-                if (dateReg.exec(rest)) {
-                    return new Date(JSON.parse(rest));
-                }
-                return JSON.parse(rest);
-            default: // string
-                return value;
-        }
-    }
-    package(multiDimensionalDocument, ids) {
-        const flatDocument = this.stringify(multiDimensionalDocument);
-        return this.compress(flatDocument, ids);
-    }
-    unpackage(document, ids) {
-        const multiDimensionalDocument = this.decompress(document, ids);
-        return this.parse(multiDimensionalDocument);
-    }
-    export() {
-        const obj = {};
-        for (const [id, map] of this.symKeys.entries()) {
-            obj[id] = {};
-            for (const [key, value] of map.entries()) {
-                obj[id][key] = value;
-            }
-        }
-        return obj;
-    }
-}

package/build/esm/services/signaler/store.d.ts

@@ -1,15 +0,0 @@
-import { ILogger } from '../logger';
-import { StoreService } from '../store';
-import { HookRule } from '../../types/hook';
-import { JobState } from '../../types/job';
-import { RedisClient, RedisMulti } from '../../types/redis';
-declare class StoreSignaler {
-    store: StoreService<RedisClient, RedisMulti>;
-    logger: ILogger;
-    constructor(store: StoreService<RedisClient, RedisMulti>, logger: ILogger);
-    getHookRule(topic: string): Promise<HookRule | undefined>;
-    registerWebHook(topic: string, context: JobState, multi?: RedisMulti): Promise<string>;
-    processWebHookSignal(topic: string, data: Record<string, unknown>): Promise<string>;
-    deleteWebHookSignal(topic: string, data: Record<string, unknown>): Promise<number>;
-}
-export { StoreSignaler };

package/build/esm/services/signaler/store.js

@@ -1,50 +0,0 @@
-class StoreSignaler {
-    constructor(store, logger) {
-        this.store = store;
-        this.logger = logger;
-    }
-    async getHookRule(topic) {
-        const rules = await this.store.getHookRules();
-        return rules?.[topic]?.[0];
-    }
-    async registerWebHook(topic, context, multi) {
-        const hookRule = await this.getHookRule(topic);
-        if (hookRule) {
-            const jobId = context.metadata.jid;
-            const hook = {
-                topic,
-                resolved: jobId,
-                jobId,
-            };
-            await this.store.setHookSignal(hook, multi);
-            return jobId;
-        }
-        else {
-            throw new Error('signaler.registerWebHook:error: hook rule not found');
-        }
-    }
-    async processWebHookSignal(topic, data) {
-        const hookRule = await this.getHookRule(topic);
-        if (hookRule) {
-            //todo: use the rule to generate `resolved`
-            const resolved = data.id;
-            const jobId = await this.store.getHookSignal(topic, resolved);
-            return jobId;
-        }
-        else {
-            throw new Error('signaler.process:error: hook rule not found');
-        }
-    }
-    async deleteWebHookSignal(topic, data) {
-        const hookRule = await this.getHookRule(topic);
-        if (hookRule) {
-            //todo: use the rule to generate `resolved`
-            const resolved = data.id;
-            return await this.store.deleteHookSignal(topic, resolved);
-        }
-        else {
-            throw new Error('signaler.process:error: hook rule not found');
-        }
-    }
-}
-export { StoreSignaler };