@hotmeshio/hotmesh 0.4.0 → 0.4.1

package/build/services/search/providers/redis/redis.js

@@ -0,0 +1,134 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedisSearchService = void 0;
+const index_1 = require("../../index");
+class RedisSearchService extends index_1.SearchService {
+    constructor(searchClient, storeClient) {
+        super(searchClient, storeClient);
+    }
+    async init(namespace, appId, logger) {
+        this.namespace = namespace;
+        this.appId = appId;
+        this.logger = logger;
+    }
+    async createSearchIndex(indexName, prefixes, schema) {
+        try {
+            await this.searchClient.sendCommand([
+                'FT.CREATE',
+                indexName,
+                'ON',
+                'HASH',
+                'PREFIX',
+                prefixes.length.toString(),
+                ...prefixes,
+                'SCHEMA',
+                ...schema,
+            ]);
+        }
+        catch (error) {
+            this.logger.info('Error creating search index', { error });
+            throw error;
+        }
+    }
+    async listSearchIndexes() {
+        try {
+            const indexes = await this.searchClient.sendCommand(['FT._LIST']);
+            return indexes;
+        }
+        catch (error) {
+            this.logger.info('Error listing search indexes', { error });
+            throw error;
+        }
+    }
+    async updateContext(key, fields) {
+        //no-op;
+        throw new Error('Not implemented');
+    }
+    async setFields(key, fields) {
+        try {
+            const result = await this.searchClient.HSET(key, fields);
+            return Number(result);
+        }
+        catch (error) {
+            this.logger.error(`Error setting fields for key: ${key}`, { error });
+            throw error;
+        }
+    }
+    async getField(key, field) {
+        try {
+            return await this.searchClient.HGET(key, field);
+        }
+        catch (error) {
+            this.logger.error(`Error getting field ${field} for key: ${key}`, {
+                error,
+            });
+            throw error;
+        }
+    }
+    async getFields(key, fields) {
+        try {
+            return await this.searchClient.HMGET(key, [...fields]);
+        }
+        catch (error) {
+            this.logger.error(`Error getting fields for key: ${key}`, { error });
+            throw error;
+        }
+    }
+    async getAllFields(key) {
+        try {
+            return await this.searchClient.HGETALL(key);
+        }
+        catch (error) {
+            this.logger.error(`Error getting fields for key: ${key}`, { error });
+            throw error;
+        }
+    }
+    async deleteFields(key, fields) {
+        try {
+            const result = await this.searchClient.HDEL(key, fields);
+            return Number(result);
+        }
+        catch (error) {
+            this.logger.error(`Error deleting fields for key: ${key}`, { error });
+            throw error;
+        }
+    }
+    async incrementFieldByFloat(key, field, increment) {
+        try {
+            const result = await this.searchClient.HINCRBYFLOAT(key, field, increment);
+            return Number(result);
+        }
+        catch (error) {
+            this.logger.error(`Error incrementing field ${field} for key: ${key}`, {
+                error,
+            });
+            throw error;
+        }
+    }
+    async sendQuery(...query) {
+        try {
+            return await this.searchClient.sendCommand(query);
+        }
+        catch (error) {
+            this.logger.error('Error executing query', { error });
+            throw error;
+        }
+    }
+    async sendIndexedQuery(index, query) {
+        try {
+            if (query[0]?.startsWith('FT.')) {
+                return (await this.searchClient.sendCommand(query));
+            }
+            return (await this.searchClient.sendCommand([
+                'FT.SEARCH',
+                index,
+                ...query,
+            ]));
+        }
+        catch (error) {
+            this.logger.error('Error executing query', { error });
+            throw error;
+        }
+    }
+}
+exports.RedisSearchService = RedisSearchService;

package/build/services/serializer/index.d.ts

@@ -0,0 +1,42 @@
+import { Consumes } from '../../types/activity';
+import { StringStringType, StringAnyType, SymbolMap, SymbolMaps, SymbolSets, Symbols } from '../../types/serializer';
+export declare const MDATA_SYMBOLS: {
+    SLOTS: number;
+    ACTIVITY: {
+        KEYS: string[];
+    };
+    ACTIVITY_UPDATE: {
+        KEYS: string[];
+    };
+    JOB: {
+        KEYS: string[];
+    };
+    JOB_UPDATE: {
+        KEYS: string[];
+    };
+};
+export declare class SerializerService {
+    dIds: StringStringType;
+    symKeys: SymbolMaps;
+    symReverseKeys: SymbolMaps;
+    symValMaps: SymbolMap;
+    symValReverseMaps: SymbolMap;
+    constructor();
+    abbreviate(consumes: Consumes, symbolNames: string[], fields?: string[]): string[];
+    resolveDimensionalIndex(path: string): string;
+    isJobPath(path: string): boolean;
+    resetSymbols(symKeys: SymbolSets, symVals: Symbols, dIds: StringStringType): void;
+    getReverseKeyMap(keyMap: SymbolMap, id?: string): SymbolMap;
+    getReverseValueMap(valueMap: SymbolMap): SymbolMap;
+    static filterSymVals(startIndex: number, maxIndex: number, existingSymbolValues: Symbols, proposedValues: Set<string>): Symbols;
+    compress(document: StringStringType, ids: string[]): StringStringType;
+    isLiteralKeyType(key: string): boolean;
+    decompress(document: StringStringType, ids: string[]): StringStringType;
+    stringify(document: Record<string, any>): StringStringType;
+    parse(document: StringStringType): any;
+    static toString(value: any): string | undefined;
+    static fromString(value: string | undefined): any;
+    package(multiDimensionalDocument: StringAnyType, ids: string[]): StringStringType;
+    unpackage(document: StringStringType, ids: string[]): StringAnyType;
+    export(): SymbolSets;
+}

package/build/services/serializer/index.js

@@ -0,0 +1,282 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SerializerService = exports.MDATA_SYMBOLS = void 0;
+const utils_1 = require("../../modules/utils");
+const dateReg = /^"\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z)?"$/;
+exports.MDATA_SYMBOLS = {
+    SLOTS: 26,
+    ACTIVITY: {
+        KEYS: ['aid', 'dad', 'as', 'atp', 'stp', 'ac', 'au', 'err', 'l1s', 'l2s'],
+    },
+    ACTIVITY_UPDATE: {
+        KEYS: ['au', 'err', 'l2s'],
+    },
+    JOB: {
+        KEYS: [
+            'ngn',
+            'tpc',
+            'pj',
+            'pg',
+            'pd',
+            'px',
+            'pa',
+            'key',
+            'app',
+            'vrs',
+            'jid',
+            'gid',
+            'aid',
+            'ts',
+            'jc',
+            'ju',
+            'js',
+            'err',
+            'trc',
+        ],
+    },
+    JOB_UPDATE: {
+        KEYS: ['ju', 'err'],
+    },
+};
+class SerializerService {
+    constructor() {
+        this.resetSymbols({}, {}, {});
+    }
+    abbreviate(consumes, symbolNames, fields = []) {
+        for (const symbolName of symbolNames) {
+            const symbolSet = this.symKeys.get(symbolName);
+            const symbolPaths = consumes[symbolName];
+            for (const symbolPath of symbolPaths) {
+                const abbreviation = symbolSet.get(symbolPath);
+                if (abbreviation) {
+                    const dimensionalIndex = this.resolveDimensionalIndex(symbolPath);
+                    fields.push(`${abbreviation}${dimensionalIndex}`);
+                }
+                else {
+                    fields.push(symbolPath);
+                }
+            }
+        }
+        return fields;
+    }
+    resolveDimensionalIndex(path) {
+        if (this.isJobPath(path)) {
+            return '';
+        }
+        else {
+            const [activityId] = path.split('/');
+            if (activityId in this.dIds) {
+                return this.dIds[activityId];
+            }
+            else if ('$ADJACENT' in this.dIds) {
+                //else=> pre-authorizing adjacent activity entry
+                return this.dIds['$ADJACENT'];
+            }
+            return ',0';
+        }
+    }
+    isJobPath(path) {
+        return path.startsWith('data/') || path.startsWith('metadata/');
+    }
+    resetSymbols(symKeys, symVals, dIds) {
+        this.symKeys = new Map();
+        this.symReverseKeys = new Map();
+        for (const id in symKeys) {
+            this.symKeys.set(id, new Map(Object.entries(symKeys[id])));
+        }
+        this.symValMaps = new Map(Object.entries(symVals));
+        this.symValReverseMaps = this.getReverseValueMap(this.symValMaps);
+        this.dIds = dIds;
+    }
+    getReverseKeyMap(keyMap, id) {
+        let map = this.symReverseKeys.get(id);
+        if (!map) {
+            map = new Map();
+            for (const [key, val] of keyMap.entries()) {
+                map.set(val, key);
+            }
+            this.symReverseKeys.set(id, map);
+        }
+        return map;
+    }
+    getReverseValueMap(valueMap) {
+        const map = new Map();
+        for (const [key, val] of valueMap.entries()) {
+            map.set(val, key);
+        }
+        return map;
+    }
+    static filterSymVals(startIndex, maxIndex, existingSymbolValues, proposedValues) {
+        const newSymbolValues = {};
+        const currentSymbolValues = { ...existingSymbolValues };
+        const currentValuesSet = new Set(Object.values(currentSymbolValues));
+        for (const value of proposedValues) {
+            if (!currentValuesSet.has(value)) {
+                if (startIndex > maxIndex) {
+                    return newSymbolValues;
+                }
+                const symbol = (0, utils_1.getSymVal)(startIndex);
+                startIndex++;
+                newSymbolValues[symbol] = value;
+                currentValuesSet.add(value);
+            }
+        }
+        return newSymbolValues;
+    }
+    compress(document, ids) {
+        if (this.symKeys.size === 0) {
+            return document;
+        }
+        const source = { ...document };
+        const result = {};
+        const compressWithMap = (abbreviationMap, id) => {
+            for (const key in source) {
+                if (key.startsWith(`${id}/`) ||
+                    id.startsWith('$') &&
+                        ['data', 'metadata'].includes(key.split('/')[0])) {
+                    const dimensionalIndex = this.resolveDimensionalIndex(key);
+                    const shortKey = abbreviationMap.get(key) || key;
+                    const shortDimensionalKey = `${shortKey}${dimensionalIndex}`;
+                    result[shortDimensionalKey] = source[key];
+                }
+                else if (!(key in result) && this.isLiteralKeyType(key)) {
+                    //mark (-) and search (_)
+                    result[key] = source[key];
+                }
+            }
+        };
+        for (const id of ids) {
+            const abbreviationMap = this.symKeys.get(id);
+            if (abbreviationMap) {
+                compressWithMap(abbreviationMap, id);
+            }
+        }
+        return result;
+    }
+    isLiteralKeyType(key) {
+        return key.startsWith('-') || key.startsWith('_');
+    }
+    decompress(document, ids) {
+        if (this.symKeys.size === 0) {
+            return document;
+        }
+        const result = { ...document };
+        const inflateWithMap = (abbreviationMap, id) => {
+            const reversedAbbreviationMap = this.getReverseKeyMap(abbreviationMap, id);
+            for (const key in result) {
+                //strip dimensional index from key
+                const shortKey = key.split(',')[0];
+                const longKey = reversedAbbreviationMap.get(shortKey);
+                if (longKey) {
+                    result[longKey] = result[key];
+                    delete result[key];
+                }
+            }
+        };
+        for (const id of ids) {
+            const abbreviationMap = this.symKeys.get(id);
+            if (abbreviationMap) {
+                inflateWithMap(abbreviationMap, id);
+            }
+        }
+        return result;
+    }
+    //stringify: convert a n-dimensional document to a 2-d hash
+    stringify(document) {
+        const result = {};
+        for (const key in document) {
+            const value = SerializerService.toString(document[key]);
+            if (value) {
+                // if (/^:*[a-zA-Z]{2}$/.test(value)) {
+                //   value = ':' + value;
+                // } else if (this.symValReverseMaps.has(value)) {
+                //   value = this.symValReverseMaps.get(value);
+                // }
+                result[key] = value;
+            }
+        }
+        return result;
+    }
+    //parse: convert a 2-d hash to a n-dimensional document
+    parse(document) {
+        const result = {};
+        for (const [key, value] of Object.entries(document)) {
+            if (value === undefined || value === null)
+                continue;
+            // if (/^:+[a-zA-Z]{2}$/.test(value)) {
+            //   result[key] = value.slice(1);
+            // } else {
+            //   if (value?.length === 2 && this.symValMaps.has(value)) {
+            //     value = this.symValMaps.get(value);
+            //   }
+            result[key] = SerializerService.fromString(value);
+            // }
+        }
+        return result;
+    }
+    static toString(value) {
+        switch (typeof value) {
+            case 'string':
+                break;
+            case 'boolean':
+                value = value ? '/t' : '/f';
+                break;
+            case 'number':
+                value = '/d' + value.toString();
+                break;
+            case 'undefined':
+                return undefined;
+            case 'object':
+                if (value === null) {
+                    value = '/n';
+                }
+                else {
+                    value = '/s' + JSON.stringify(value);
+                }
+                break;
+        }
+        return value;
+    }
+    static fromString(value) {
+        if (typeof value !== 'string')
+            return undefined;
+        const prefix = value.slice(0, 2);
+        const rest = value.slice(2);
+        switch (prefix) {
+            case '/t': // boolean true
+                return true;
+            case '/f': // boolean false
+                return false;
+            case '/d': // number
+                return Number(rest);
+            case '/n': // null
+                return null;
+            case '/s': // object (JSON string)
+                if (dateReg.exec(rest)) {
+                    return new Date(JSON.parse(rest));
+                }
+                return JSON.parse(rest);
+            default: // string
+                return value;
+        }
+    }
+    package(multiDimensionalDocument, ids) {
+        const flatDocument = this.stringify(multiDimensionalDocument);
+        return this.compress(flatDocument, ids);
+    }
+    unpackage(document, ids) {
+        const multiDimensionalDocument = this.decompress(document, ids);
+        return this.parse(multiDimensionalDocument);
+    }
+    export() {
+        const obj = {};
+        for (const [id, map] of this.symKeys.entries()) {
+            obj[id] = {};
+            for (const [key, value] of map.entries()) {
+                obj[id][key] = value;
+            }
+        }
+        return obj;
+    }
+}
+exports.SerializerService = SerializerService;

package/build/services/store/cache.d.ts

@@ -0,0 +1,67 @@
+/**
+ * The Cache is a key/value store and used to store commonly accessed Redis metadata
+ * (mainly the execution rules for the app) to save time accessing them as they
+ * are immutable per verison. Rules are only ejected when a new version
+ * (a new distributed executable) is deployed to the quorum
+ * and the cache is invalidated/cleared of the prior version.
+ */
+import { ActivityType } from '../../types/activity';
+import { HookRule } from '../../types/hook';
+import { HotMeshApp, HotMeshSettings } from '../../types/hotmesh';
+import { Symbols } from '../../types/serializer';
+import { Transitions } from '../../types/transition';
+declare class Cache {
+    settings: HotMeshSettings;
+    appId: string;
+    apps: Record<string, HotMeshApp>;
+    schemas: Record<string, ActivityType>;
+    subscriptions: Record<string, Record<string, string>>;
+    symbols: Record<string, Symbols>;
+    symvals: Record<string, Symbols>;
+    transitions: Record<string, Record<string, unknown>>;
+    hookRules: Record<string, Record<string, HookRule[]>>;
+    workItems: Record<string, string>;
+    /**
+     * The cache is ALWAYS initialized with HotMeshSettings. The other parameters are optional.
+     * @param settings
+     * @param apps
+     * @param schemas
+     * @param subscriptions
+     * @param transitions
+     * @param hookRules
+     */
+    constructor(appId: string, settings: HotMeshSettings, apps?: Record<string, HotMeshApp>, schemas?: Record<string, ActivityType>, subscriptions?: Record<string, Record<string, string>>, symbols?: Record<string, Symbols>, symvals?: Record<string, Symbols>, transitions?: Record<string, Record<string, unknown>>, hookRules?: Record<string, Record<string, HookRule[]>>, workItems?: Record<string, string>);
+    /**
+     * invalidate the cache; settings are not invalidated!
+     */
+    invalidate(): void;
+    getSettings(): HotMeshSettings;
+    setSettings(settings: HotMeshSettings): void;
+    getApps(): Record<string, HotMeshApp>;
+    getApp(appId: string): HotMeshApp;
+    setApps(apps: Record<string, HotMeshApp>): void;
+    setApp(appId: string, app: HotMeshApp): void;
+    getSchemas(appId: string, version: string): Record<string, ActivityType>;
+    getSchema(appId: string, version: string, activityId: string): ActivityType;
+    setSchemas(appId: string, version: string, schemas: Record<string, ActivityType>): void;
+    setSchema(appId: string, version: string, activityId: string, schema: ActivityType): void;
+    getSubscriptions(appId: string, version: string): Record<string, string>;
+    getSubscription(appId: string, version: string, topic: string): unknown;
+    setSubscriptions(appId: string, version: string, subscriptions: Record<string, string>): void;
+    getSymbols(appId: string, targetEntityId: string): Symbols;
+    setSymbols(appId: string, targetEntityId: string, symbols: Symbols): void;
+    deleteSymbols(appId: string, targetEntityId: string): void;
+    getSymbolValues(appId: string): Symbols;
+    setSymbolValues(appId: string, symvals: Symbols): void;
+    deleteSymbolValues(appId: string): void;
+    getTransitions(appId: string, version: string): Transitions;
+    setTransitions(appId: string, version: string, transitions: Transitions): void;
+    getHookRules(appId: string): Record<string, HookRule[]>;
+    setHookRules(appId: string, hookRules: Record<string, HookRule[]>): void;
+    getSignals(appId: string, version: string): Record<string, unknown>;
+    setSignals(appId: string, version: string): Record<string, unknown>;
+    getActiveTaskQueue(appId: string): string;
+    setWorkItem(appId: string, workItem: string): void;
+    removeWorkItem(appId: string): void;
+}
+export { Cache };

package/build/services/store/cache.js

@@ -0,0 +1,128 @@
+"use strict";
+/**
+ * The Cache is a key/value store and used to store commonly accessed Redis metadata
+ * (mainly the execution rules for the app) to save time accessing them as they
+ * are immutable per verison. Rules are only ejected when a new version
+ * (a new distributed executable) is deployed to the quorum
+ * and the cache is invalidated/cleared of the prior version.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Cache = void 0;
+class Cache {
+    /**
+     * The cache is ALWAYS initialized with HotMeshSettings. The other parameters are optional.
+     * @param settings
+     * @param apps
+     * @param schemas
+     * @param subscriptions
+     * @param transitions
+     * @param hookRules
+     */
+    constructor(appId, settings, apps = {}, schemas = {}, subscriptions = {}, symbols = {}, symvals = {}, transitions = {}, hookRules = {}, workItems = {}) {
+        this.appId = appId;
+        this.settings = settings;
+        this.apps = apps;
+        this.schemas = schemas;
+        this.subscriptions = subscriptions;
+        this.symbols = symbols;
+        this.symvals = symvals;
+        this.transitions = transitions;
+        this.hookRules = hookRules;
+        this.workItems = workItems;
+    }
+    /**
+     * invalidate the cache; settings are not invalidated!
+     */
+    invalidate() {
+        this.apps = {};
+        this.schemas = {};
+        this.subscriptions = {};
+        this.transitions = {};
+        this.hookRules = {};
+    }
+    getSettings() {
+        return this.settings;
+    }
+    setSettings(settings) {
+        this.settings = settings;
+    }
+    getApps() {
+        return this.apps;
+    }
+    getApp(appId) {
+        return this.apps[appId];
+    }
+    setApps(apps) {
+        this.apps = apps;
+    }
+    setApp(appId, app) {
+        this.apps[appId] = app;
+    }
+    getSchemas(appId, version) {
+        return this.schemas[`${appId}/${version}`];
+    }
+    getSchema(appId, version, activityId) {
+        return this.schemas?.[`${appId}/${version}`]?.[activityId];
+    }
+    setSchemas(appId, version, schemas) {
+        this.schemas[`${appId}/${version}`] = schemas;
+    }
+    setSchema(appId, version, activityId, schema) {
+        this.schemas[`${appId}/${version}`][activityId] = schema;
+    }
+    getSubscriptions(appId, version) {
+        return this.subscriptions[`${appId}/${version}`];
+    }
+    getSubscription(appId, version, topic) {
+        return this.subscriptions?.[`${appId}/${version}`]?.[topic];
+    }
+    setSubscriptions(appId, version, subscriptions) {
+        this.subscriptions[`${appId}/${version}`] = subscriptions;
+    }
+    getSymbols(appId, targetEntityId) {
+        return this.symbols[`${appId}/${targetEntityId}`];
+    }
+    setSymbols(appId, targetEntityId, symbols) {
+        this.symbols[`${appId}/${targetEntityId}`] = symbols;
+    }
+    deleteSymbols(appId, targetEntityId) {
+        delete this.symbols[`${appId}/${targetEntityId}`];
+    }
+    getSymbolValues(appId) {
+        return this.symvals[`${appId}`];
+    }
+    setSymbolValues(appId, symvals) {
+        this.symvals[`${appId}`] = symvals;
+    }
+    deleteSymbolValues(appId) {
+        delete this.symvals[`${appId}`];
+    }
+    getTransitions(appId, version) {
+        return this.transitions[`${appId}/${version}`];
+    }
+    setTransitions(appId, version, transitions) {
+        this.transitions[`${appId}/${version}`] = transitions;
+    }
+    getHookRules(appId) {
+        return this.hookRules[`${appId}`];
+    }
+    setHookRules(appId, hookRules) {
+        this.hookRules[`${appId}`] = hookRules;
+    }
+    getSignals(appId, version) {
+        throw new Error('SIGNAL (getHooks) is not supported');
+    }
+    setSignals(appId, version) {
+        throw new Error('SIGNAL (setHook) is not supported');
+    }
+    getActiveTaskQueue(appId) {
+        return this.workItems[appId];
+    }
+    setWorkItem(appId, workItem) {
+        this.workItems[appId] = workItem;
+    }
+    removeWorkItem(appId) {
+        delete this.workItems[appId];
+    }
+}
+exports.Cache = Cache;

package/build/services/store/factory.d.ts

@@ -0,0 +1,8 @@
+import { ILogger } from '../logger';
+import { ProviderClient, ProviderTransaction } from '../../types/provider';
+import { StoreInitializable } from './providers/store-initializable';
+import { StoreService } from './index';
+declare class StoreServiceFactory {
+    static init(providerClient: ProviderClient, namespace: string, appId: string, logger: ILogger): Promise<StoreService<ProviderClient, ProviderTransaction> & StoreInitializable>;
+}
+export { StoreServiceFactory };

package/build/services/store/factory.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StoreServiceFactory = void 0;
+const utils_1 = require("../../modules/utils");
+const ioredis_1 = require("./providers/redis/ioredis");
+const redis_1 = require("./providers/redis/redis");
+const postgres_1 = require("./providers/postgres/postgres");
+class StoreServiceFactory {
+    static async init(providerClient, namespace, appId, logger) {
+        let service;
+        if ((0, utils_1.identifyProvider)(providerClient) === 'redis') {
+            service = new redis_1.RedisStoreService(providerClient);
+        }
+        else if ((0, utils_1.identifyProvider)(providerClient) === 'ioredis') {
+            service = new ioredis_1.IORedisStoreService(providerClient);
+        }
+        else if ((0, utils_1.identifyProvider)(providerClient) === 'postgres') {
+            service = new postgres_1.PostgresStoreService(providerClient);
+        } //etc
+        await service.init(namespace, appId, logger);
+        return service;
+    }
+}
+exports.StoreServiceFactory = StoreServiceFactory;