@hotmeshio/hotmesh 0.0.4 → 0.0.6

package/build/esm/services/reporter/index.js

@@ -0,0 +1,327 @@
+import { Pipe } from '../pipe';
+class ReporterService {
+    constructor(appVersion, store, logger) {
+        this.appVersion = appVersion;
+        this.logger = logger;
+        this.store = store;
+    }
+    async getStats(options) {
+        this.logger.debug('reporter-getstats-started', options);
+        const { key, granularity, range, end, start } = options;
+        this.validateOptions(options);
+        const dateTimeSets = this.generateDateTimeSets(granularity, range, end, start);
+        const redisKeys = dateTimeSets.map((dateTime) => this.buildRedisKey(key, dateTime));
+        const rawData = await this.store.getJobStats(redisKeys);
+        const [count, aggregatedData] = this.aggregateData(rawData);
+        const statsResponse = this.buildStatsResponse(rawData, redisKeys, aggregatedData, count, options);
+        return statsResponse;
+    }
+    validateOptions(options) {
+        const { start, end, range } = options;
+        if (start && end && range || !start && !end && !range) {
+            throw new Error('Invalid combination of start, end, and range values. Provide either start+end, end+range, or start+range.');
+        }
+    }
+    generateDateTimeSets(granularity, range, end, start) {
+        if (!range) {
+            //pluck just a single value when no range provided
+            range = '0m';
+        }
+        const granularitiesInMinutes = {
+            '5m': 5,
+            '10m': 10,
+            '15m': 15,
+            '30m': 30,
+            '1h': 60,
+        };
+        const granularityMinutes = granularitiesInMinutes[granularity];
+        if (!granularityMinutes) {
+            throw new Error('Invalid granularity value.');
+        }
+        const rangeMinutes = this.convertRangeToMinutes(range);
+        if (rangeMinutes === null) {
+            throw new Error('Invalid range value.');
+        }
+        // If start is provided, use it. Otherwise, calculate it from the end time and range.
+        let startTime;
+        let endTime;
+        if (start) {
+            startTime = new Date(start);
+            endTime = new Date(startTime.getTime() + rangeMinutes * 60 * 1000);
+        }
+        else {
+            endTime = end === 'NOW' ? new Date() : new Date(end);
+            startTime = new Date(endTime.getTime() - rangeMinutes * 60 * 1000);
+        }
+        // Round the start time to the nearest granularity unit
+        startTime.setUTCMinutes(Math.floor(startTime.getUTCMinutes() / granularityMinutes) * granularityMinutes);
+        const dateTimeSets = [];
+        for (let time = startTime; time <= endTime; time.setUTCMinutes(time.getUTCMinutes() + granularityMinutes)) {
+            const formattedTime = [
+                time.getUTCFullYear(),
+                String(time.getUTCMonth() + 1).padStart(2, '0'),
+                String(time.getUTCDate()).padStart(2, '0'),
+                String(time.getUTCHours()).padStart(2, '0'),
+                String(time.getUTCMinutes()).padStart(2, '0'),
+            ].join('');
+            dateTimeSets.push(formattedTime);
+        }
+        return dateTimeSets;
+    }
+    convertRangeToMinutes(range) {
+        const timeUnit = range.slice(-1);
+        const value = parseInt(range.slice(0, -1), 10);
+        if (isNaN(value)) {
+            return null;
+        }
+        switch (timeUnit) {
+            case 'm':
+                return value;
+            case 'h':
+                return value * 60;
+            case 'd':
+                return value * 60 * 24;
+            default:
+                return null;
+        }
+    }
+    buildRedisKey(key, dateTime, subTarget = '') {
+        return `hmsh:${this.appVersion.id}:s:${key}:${dateTime}${subTarget ? ':' + subTarget : ''}`;
+    }
+    aggregateData(rawData) {
+        const aggregatedData = {};
+        let count = 0;
+        Object.entries(rawData).forEach(([_, data]) => {
+            for (const key in data) {
+                if (key.startsWith('count:')) {
+                    const target = key.slice('count:'.length);
+                    if (!aggregatedData[target]) {
+                        aggregatedData[target] = 0;
+                    }
+                    aggregatedData[target] += data[key];
+                }
+                else if (key === 'count') {
+                    count += data[key];
+                }
+            }
+        });
+        return [count, aggregatedData];
+    }
+    buildStatsResponse(rawData, redisKeys, aggregatedData, count, options) {
+        const measures = [];
+        const measureKeys = Object.keys(aggregatedData).filter((key) => key !== "count");
+        let segments = undefined;
+        if (options.sparse !== true) {
+            segments = this.handleSegments(rawData, redisKeys);
+        }
+        measureKeys.forEach((key) => {
+            const measure = {
+                target: key,
+                type: "count",
+                value: aggregatedData[key],
+            };
+            measures.push(measure);
+        });
+        const response = {
+            key: options.key,
+            granularity: options.granularity,
+            range: options.range,
+            end: options.end,
+            count,
+            measures: measures,
+        };
+        if (segments) {
+            response.segments = segments;
+        }
+        return response;
+    }
+    handleSegments(data, hashKeys) {
+        const segments = [];
+        hashKeys.forEach((hashKey, index) => {
+            const segmentData = [];
+            data[hashKey] && Object.entries(data[hashKey]).forEach(([key, value]) => {
+                if (key.startsWith('count:')) {
+                    const target = key.slice('count:'.length);
+                    segmentData.push({ target, type: 'count', value });
+                }
+            });
+            const isoTimestamp = this.isoTimestampFromKeyTimestamp(hashKey);
+            const count = data[hashKey] ? data[hashKey].count : 0;
+            segments.push({ count, time: isoTimestamp, measures: segmentData });
+        });
+        return segments;
+    }
+    isoTimestampFromKeyTimestamp(hashKey) {
+        const keyTimestamp = hashKey.slice(-12);
+        const year = keyTimestamp.slice(0, 4);
+        const month = keyTimestamp.slice(4, 6);
+        const day = keyTimestamp.slice(6, 8);
+        const hour = keyTimestamp.slice(8, 10);
+        const minute = keyTimestamp.slice(10, 12);
+        return `${year}-${month}-${day}T${hour}:${minute}Z`;
+    }
+    async getIds(options, facets, idRange = [0, -1]) {
+        if (!facets.length) {
+            const stats = await this.getStats(options);
+            facets = this.getUniqueFacets(stats);
+        }
+        const { key, granularity, range, end, start } = options;
+        this.validateOptions(options);
+        let redisKeys = [];
+        facets.forEach((facet) => {
+            const dateTimeSets = this.generateDateTimeSets(granularity, range, end, start);
+            redisKeys = redisKeys.concat(dateTimeSets.map((dateTime) => this.buildRedisKey(key, dateTime, `index:${facet}`)));
+        });
+        const idsData = await this.store.getJobIds(redisKeys, idRange);
+        const idsResponse = this.buildIdsResponse(idsData, options, facets);
+        return idsResponse;
+    }
+    buildIdsResponse(idsData, options, facets) {
+        const countsByFacet = {};
+        const measureKeys = Object.keys(idsData);
+        measureKeys.forEach((key) => {
+            const target = this.getTargetForKey(key);
+            const count = idsData[key].length;
+            if (countsByFacet[target]) {
+                countsByFacet[target] += count;
+            }
+            else {
+                countsByFacet[target] = count;
+            }
+        });
+        const counts = Object.entries(countsByFacet).map(([facet, count]) => ({ facet, count }));
+        const response = {
+            key: options.key,
+            facets,
+            granularity: options.granularity,
+            range: options.range,
+            start: options.start,
+            counts,
+            segments: this.buildTimeSegments(idsData),
+        };
+        return response;
+    }
+    buildTimeSegments(idsData) {
+        const measureKeys = Object.keys(idsData);
+        const timeSegments = {};
+        measureKeys.forEach((key) => {
+            const measure = {
+                type: 'ids',
+                target: this.getTargetForKey(key),
+                time: this.isoTimestampFromKeyTimestamp(this.getTargetForTime(key)),
+                count: idsData[key].length,
+                ids: idsData[key],
+            };
+            if (timeSegments[measure.time]) {
+                timeSegments[measure.time].push(measure);
+            }
+            else {
+                timeSegments[measure.time] = [measure];
+            }
+        });
+        const segments = Object.entries(timeSegments).map(([time, measures]) => ({
+            time,
+            measures,
+        }));
+        return segments;
+    }
+    getUniqueFacets(data) {
+        const targets = data.measures.map(measure => measure.target);
+        return Array.from(new Set(targets));
+    }
+    getTargetForKey(key) {
+        return key.split(':index:')[1];
+    }
+    getTargetForTime(key) {
+        return key.split(':index:')[0];
+    }
+    async getWorkItems(options, facets) {
+        if (!facets.length) {
+            const stats = await this.getStats(options);
+            facets = this.getUniqueFacets(stats);
+        }
+        const { key, granularity, range, end, start } = options;
+        this.validateOptions(options);
+        let redisKeys = [];
+        facets.forEach((facet) => {
+            const dateTimeSets = this.generateDateTimeSets(granularity, range, end, start);
+            redisKeys = redisKeys.concat(dateTimeSets.map((dateTime) => this.buildRedisKey(key, dateTime, `index:${facet}`)));
+        });
+        const idsData = await this.store.getJobIds(redisKeys, [0, 1]);
+        const workerLists = this.buildWorkerLists(idsData);
+        return workerLists;
+    }
+    buildWorkerLists(idsData) {
+        const workerLists = [];
+        for (const key in idsData) {
+            if (idsData[key].length) {
+                workerLists.push(key);
+            }
+        }
+        return workerLists;
+    }
+    /**
+     * called by `trigger` activity to generate the stats that should
+     * be saved to the database. doesn't actually save the stats, but
+     * just generates the info that should be saved
+     */
+    resolveTriggerStatistics({ stats: statsConfig }, context) {
+        const stats = {
+            general: [],
+            index: [],
+            median: []
+        };
+        stats.general.push({ metric: 'count', target: 'count', value: 1 });
+        for (const measure of statsConfig.measures) {
+            const metric = this.resolveMetric({ metric: measure.measure, target: measure.target }, context);
+            if (this.isGeneralMetric(measure.measure)) {
+                stats.general.push(metric);
+            }
+            else if (this.isMedianMetric(measure.measure)) {
+                stats.median.push(metric);
+            }
+            else if (this.isIndexMetric(measure.measure)) {
+                stats.index.push(metric);
+            }
+        }
+        return stats;
+    }
+    isGeneralMetric(metric) {
+        return metric === 'sum' || metric === 'avg' || metric === 'count';
+    }
+    isMedianMetric(metric) {
+        return metric === 'mdn';
+    }
+    isIndexMetric(metric) {
+        return metric === 'index';
+    }
+    resolveMetric({ metric, target }, context) {
+        const pipe = new Pipe([[target]], context);
+        const resolvedValue = pipe.process().toString();
+        const resolvedTarget = this.resolveTarget(metric, target, resolvedValue);
+        if (metric === 'index') {
+            return { metric, target: resolvedTarget, value: context.metadata.jid };
+        }
+        else if (metric === 'count') {
+            return { metric, target: resolvedTarget, value: 1 };
+        }
+        return { metric, target: resolvedTarget, value: resolvedValue };
+    }
+    isCardinalMetric(metric) {
+        return metric === 'index' || metric === 'count';
+    }
+    resolveTarget(metric, target, resolvedValue) {
+        const trimmed = target.substring(1, target.length - 1);
+        const trimmedTarget = trimmed.split('.').slice(3).join('/');
+        let resolvedTarget;
+        if (this.isCardinalMetric(metric)) {
+            resolvedTarget = `${metric}:${trimmedTarget}:${resolvedValue}`;
+        }
+        else {
+            resolvedTarget = `${metric}:${trimmedTarget}`;
+        }
+        return resolvedTarget;
+    }
+}
+ReporterService.DEFAULT_GRANULARITY = '5m';
+export { ReporterService };

package/build/esm/services/serializer/index.d.ts

@@ -0,0 +1,41 @@
+import { Consumes } from '../../types/activity';
+import { StringStringType, StringAnyType, SymbolMap, SymbolMaps, SymbolSets, Symbols } from '../../types/serializer';
+export declare const MDATA_SYMBOLS: {
+    SLOTS: number;
+    ACTIVITY: {
+        KEYS: string[];
+    };
+    ACTIVITY_UPDATE: {
+        KEYS: string[];
+    };
+    JOB: {
+        KEYS: string[];
+    };
+    JOB_UPDATE: {
+        KEYS: string[];
+    };
+};
+export declare class SerializerService {
+    dIds: StringStringType;
+    symKeys: SymbolMaps;
+    symReverseKeys: SymbolMaps;
+    symValMaps: SymbolMap;
+    symValReverseMaps: SymbolMap;
+    constructor();
+    abbreviate(consumes: Consumes, symbolNames: string[], fields?: string[]): string[];
+    resolveDimensionalIndex(path: string): string;
+    isJobPath(path: string): boolean;
+    resetSymbols(symKeys: SymbolSets, symVals: Symbols, dIds: StringStringType): void;
+    getReverseKeyMap(keyMap: SymbolMap, id?: string): SymbolMap;
+    getReverseValueMap(valueMap: SymbolMap): SymbolMap;
+    static filterSymVals(startIndex: number, maxIndex: number, existingSymbolValues: Symbols, proposedValues: Set<string>): Symbols;
+    compress(document: StringStringType, ids: string[]): StringStringType;
+    decompress(document: StringStringType, ids: string[]): StringStringType;
+    stringify(document: Record<string, any>): StringStringType;
+    parse(document: StringStringType): any;
+    toString(value: any): string | undefined;
+    fromString(value: string | undefined): any;
+    package(multiDimensionalDocument: StringAnyType, ids: string[]): StringStringType;
+    unpackage(document: StringStringType, ids: string[]): StringAnyType;
+    export(): SymbolSets;
+}

package/build/esm/services/serializer/index.js

@@ -0,0 +1,251 @@
+import { getSymVal } from '../../modules/utils';
+const dateReg = /^"\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z)?"$/;
+export const MDATA_SYMBOLS = {
+    SLOTS: 26,
+    ACTIVITY: {
+        KEYS: ['aid', 'dad', 'as', 'atp', 'stp', 'ac', 'au', 'err', 'l1s', 'l2s']
+    },
+    ACTIVITY_UPDATE: {
+        KEYS: ['au', 'err', 'l2s']
+    },
+    JOB: {
+        KEYS: ['ngn', 'tpc', 'pj', 'pd', 'pa', 'key', 'app', 'vrs', 'jid', 'aid', 'ts', 'jc', 'ju', 'js', 'err', 'trc']
+    },
+    JOB_UPDATE: {
+        KEYS: ['ju', 'err']
+    }
+};
+export class SerializerService {
+    constructor() {
+        this.resetSymbols({}, {}, {});
+    }
+    abbreviate(consumes, symbolNames, fields = []) {
+        for (const symbolName of symbolNames) {
+            const symbolSet = this.symKeys.get(symbolName);
+            const symbolPaths = consumes[symbolName];
+            for (const symbolPath of symbolPaths) {
+                const abbreviation = symbolSet.get(symbolPath);
+                if (abbreviation) {
+                    const dimensionalIndex = this.resolveDimensionalIndex(symbolPath);
+                    fields.push(`${abbreviation}${dimensionalIndex}`);
+                }
+                else {
+                    fields.push(symbolPath);
+                }
+            }
+        }
+        return fields;
+    }
+    resolveDimensionalIndex(path) {
+        if (this.isJobPath(path)) {
+            return '';
+        }
+        else {
+            const [activityId] = path.split('/');
+            if (activityId in this.dIds) {
+                return this.dIds[activityId];
+            }
+            else if ('$ADJACENT' in this.dIds) {
+                //else=> pre-authorizing adjacent activity entry
+                return this.dIds['$ADJACENT'];
+            }
+            return ',0';
+        }
+    }
+    isJobPath(path) {
+        return path.startsWith('data/') || path.startsWith('metadata/');
+    }
+    resetSymbols(symKeys, symVals, dIds) {
+        this.symKeys = new Map();
+        this.symReverseKeys = new Map();
+        for (const id in symKeys) {
+            this.symKeys.set(id, new Map(Object.entries(symKeys[id])));
+        }
+        this.symValMaps = new Map(Object.entries(symVals));
+        this.symValReverseMaps = this.getReverseValueMap(this.symValMaps);
+        this.dIds = dIds;
+    }
+    getReverseKeyMap(keyMap, id) {
+        let map = this.symReverseKeys.get(id);
+        if (!map) {
+            map = new Map();
+            for (let [key, val] of keyMap.entries()) {
+                map.set(val, key);
+            }
+            this.symReverseKeys.set(id, map);
+        }
+        return map;
+    }
+    getReverseValueMap(valueMap) {
+        const map = new Map();
+        for (let [key, val] of valueMap.entries()) {
+            map.set(val, key);
+        }
+        return map;
+    }
+    static filterSymVals(startIndex, maxIndex, existingSymbolValues, proposedValues) {
+        let newSymbolValues = {};
+        let currentSymbolValues = { ...existingSymbolValues };
+        let currentValuesSet = new Set(Object.values(currentSymbolValues));
+        for (let value of proposedValues) {
+            if (!currentValuesSet.has(value)) {
+                if (startIndex > maxIndex) {
+                    return newSymbolValues;
+                }
+                const symbol = getSymVal(startIndex);
+                startIndex++;
+                newSymbolValues[symbol] = value;
+                currentValuesSet.add(value);
+            }
+        }
+        return newSymbolValues;
+    }
+    compress(document, ids) {
+        if (this.symKeys.size === 0) {
+            return document;
+        }
+        let source = { ...document };
+        let result = {};
+        const compressWithMap = (abbreviationMap, id) => {
+            for (let key in source) {
+                if (key.startsWith(`${id}/`) || (id.startsWith('$') && ['data', 'metadata'].includes(key.split('/')[0]))) {
+                    const dimensionalIndex = this.resolveDimensionalIndex(key);
+                    let shortKey = abbreviationMap.get(key) || key;
+                    const shortDimensionalKey = `${shortKey}${dimensionalIndex}`;
+                    result[shortDimensionalKey] = source[key];
+                }
+            }
+        };
+        for (let id of ids) {
+            const abbreviationMap = this.symKeys.get(id);
+            if (abbreviationMap) {
+                compressWithMap(abbreviationMap, id);
+            }
+        }
+        return result;
+    }
+    decompress(document, ids) {
+        if (this.symKeys.size === 0) {
+            return document;
+        }
+        let result = { ...document };
+        const inflateWithMap = (abbreviationMap, id) => {
+            const reversedAbbreviationMap = this.getReverseKeyMap(abbreviationMap, id);
+            for (let key in result) {
+                //strip dimensional index from key
+                const shortKey = key.split(',')[0];
+                let longKey = reversedAbbreviationMap.get(shortKey);
+                if (longKey) {
+                    result[longKey] = result[key];
+                    delete result[key];
+                }
+            }
+        };
+        for (let id of ids) {
+            const abbreviationMap = this.symKeys.get(id);
+            if (abbreviationMap) {
+                inflateWithMap(abbreviationMap, id);
+            }
+        }
+        return result;
+    }
+    //stringify: convert a multi-dimensional document to a 2-d hash
+    stringify(document) {
+        let result = {};
+        for (let key in document) {
+            let value = this.toString(document[key]);
+            if (value) {
+                if (/^:*[a-zA-Z]{2}$/.test(value)) {
+                    value = ':' + value;
+                }
+                else if (this.symValReverseMaps.has(value)) {
+                    value = this.symValReverseMaps.get(value);
+                }
+                result[key] = value;
+            }
+        }
+        return result;
+    }
+    //parse: convert a 2-d hash to a multi-dimensional document
+    parse(document) {
+        let result = {};
+        for (let [key, value] of Object.entries(document)) {
+            if (value === undefined || value === null)
+                continue;
+            if (/^:+[a-zA-Z]{2}$/.test(value)) {
+                result[key] = value.slice(1);
+            }
+            else {
+                if (value?.length === 2 && this.symValMaps.has(value)) {
+                    value = this.symValMaps.get(value);
+                }
+                result[key] = this.fromString(value);
+            }
+        }
+        return result;
+    }
+    toString(value) {
+        switch (typeof value) {
+            case 'string':
+                break;
+            case 'boolean':
+                value = value ? '/t' : '/f';
+                break;
+            case 'number':
+                value = '/d' + value.toString();
+                break;
+            case 'undefined':
+                return undefined;
+            case 'object':
+                if (value === null) {
+                    value = '/n';
+                }
+                else {
+                    value = '/s' + JSON.stringify(value);
+                }
+                break;
+        }
+        return value;
+    }
+    fromString(value) {
+        if (typeof value !== 'string')
+            return undefined;
+        const prefix = value.slice(0, 2);
+        const rest = value.slice(2);
+        switch (prefix) {
+            case '/t': // boolean true
+                return true;
+            case '/f': // boolean false
+                return false;
+            case '/d': // number
+                return Number(rest);
+            case '/n': // null
+                return null;
+            case '/s': // object (JSON string)
+                if (dateReg.exec(rest)) {
+                    return new Date(JSON.parse(rest));
+                }
+                return JSON.parse(rest);
+            default: // string
+                return value;
+        }
+    }
+    package(multiDimensionalDocument, ids) {
+        const flatDocument = this.stringify(multiDimensionalDocument);
+        return this.compress(flatDocument, ids);
+    }
+    unpackage(document, ids) {
+        const multiDimensionalDocument = this.decompress(document, ids);
+        return this.parse(multiDimensionalDocument);
+    }
+    export() {
+        const obj = {};
+        for (const [id, map] of this.symKeys.entries()) {
+            obj[id] = {};
+            for (const [key, value] of map.entries()) {
+                obj[id][key] = value;
+            }
+        }
+        return obj;
+    }
+}

package/build/esm/services/signaler/store.d.ts

@@ -0,0 +1,15 @@
+import { ILogger } from '../logger';
+import { StoreService } from '../store';
+import { HookRule } from '../../types/hook';
+import { JobState } from '../../types/job';
+import { RedisClient, RedisMulti } from '../../types/redis';
+declare class StoreSignaler {
+    store: StoreService<RedisClient, RedisMulti>;
+    logger: ILogger;
+    constructor(store: StoreService<RedisClient, RedisMulti>, logger: ILogger);
+    getHookRule(topic: string): Promise<HookRule | undefined>;
+    registerWebHook(topic: string, context: JobState, multi?: RedisMulti): Promise<string>;
+    processWebHookSignal(topic: string, data: Record<string, unknown>): Promise<string>;
+    deleteWebHookSignal(topic: string, data: Record<string, unknown>): Promise<number>;
+}
+export { StoreSignaler };

package/build/esm/services/signaler/store.js

@@ -0,0 +1,50 @@
+class StoreSignaler {
+    constructor(store, logger) {
+        this.store = store;
+        this.logger = logger;
+    }
+    async getHookRule(topic) {
+        const rules = await this.store.getHookRules();
+        return rules?.[topic]?.[0];
+    }
+    async registerWebHook(topic, context, multi) {
+        const hookRule = await this.getHookRule(topic);
+        if (hookRule) {
+            const jobId = context.metadata.jid;
+            const hook = {
+                topic,
+                resolved: jobId,
+                jobId,
+            };
+            await this.store.setHookSignal(hook, multi);
+            return jobId;
+        }
+        else {
+            throw new Error('signaler.registerWebHook:error: hook rule not found');
+        }
+    }
+    async processWebHookSignal(topic, data) {
+        const hookRule = await this.getHookRule(topic);
+        if (hookRule) {
+            //todo: use the rule to generate `resolved`
+            const resolved = data.id;
+            const jobId = await this.store.getHookSignal(topic, resolved);
+            return jobId;
+        }
+        else {
+            throw new Error('signaler.process:error: hook rule not found');
+        }
+    }
+    async deleteWebHookSignal(topic, data) {
+        const hookRule = await this.getHookRule(topic);
+        if (hookRule) {
+            //todo: use the rule to generate `resolved`
+            const resolved = data.id;
+            return await this.store.deleteHookSignal(topic, resolved);
+        }
+        else {
+            throw new Error('signaler.process:error: hook rule not found');
+        }
+    }
+}
+export { StoreSignaler };