@hotmeshio/hotmesh 0.4.0 → 0.4.1

package/build/services/router/lifecycle/index.d.ts

@@ -0,0 +1,27 @@
+import { ILogger } from '../../logger';
+import { StreamService } from '../../stream';
+import { ProviderClient, ProviderTransaction } from '../../../types/provider';
+export declare class InstanceRegistry {
+    private static instances;
+    static add(router: any): void;
+    static remove(router: any): void;
+    static stopAll(): Promise<void>;
+    static getInstances(): Set<any>;
+}
+export declare class LifecycleManager<S extends StreamService<ProviderClient, ProviderTransaction>> {
+    private shouldConsume;
+    private readonly;
+    private topic;
+    private logger;
+    private stream;
+    private isUsingNotifications;
+    constructor(readonly: boolean, topic: string | undefined, logger: ILogger, stream: S);
+    getShouldConsume(): boolean;
+    setShouldConsume(value: boolean): void;
+    getIsUsingNotifications(): boolean;
+    setIsUsingNotifications(value: boolean): void;
+    isReadonly(): boolean;
+    isStopped(group: string, consumer: string, stream: string): boolean;
+    startConsuming(router: any): Promise<void>;
+    stopConsuming(router: any): Promise<void>;
+}

package/build/services/router/lifecycle/index.js

@@ -0,0 +1,80 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LifecycleManager = exports.InstanceRegistry = void 0;
+const utils_1 = require("../../../modules/utils");
+const config_1 = require("../config");
+class InstanceRegistry {
+    static add(router) {
+        InstanceRegistry.instances.add(router);
+    }
+    static remove(router) {
+        InstanceRegistry.instances.delete(router);
+    }
+    static async stopAll() {
+        const stopPromises = [];
+        for (const instance of [...InstanceRegistry.instances]) {
+            stopPromises.push(instance.stopConsuming());
+        }
+        await Promise.all(stopPromises);
+        await (0, utils_1.sleepFor)(config_1.HMSH_BLOCK_TIME_MS * 2);
+    }
+    static getInstances() {
+        return new Set(InstanceRegistry.instances);
+    }
+}
+InstanceRegistry.instances = new Set();
+exports.InstanceRegistry = InstanceRegistry;
+class LifecycleManager {
+    constructor(readonly, topic, logger, stream) {
+        this.shouldConsume = false;
+        this.isUsingNotifications = false;
+        this.readonly = readonly;
+        this.topic = topic;
+        this.logger = logger;
+        this.stream = stream;
+    }
+    getShouldConsume() {
+        return this.shouldConsume;
+    }
+    setShouldConsume(value) {
+        this.shouldConsume = value;
+    }
+    getIsUsingNotifications() {
+        return this.isUsingNotifications;
+    }
+    setIsUsingNotifications(value) {
+        this.isUsingNotifications = value;
+    }
+    isReadonly() {
+        return this.readonly;
+    }
+    isStopped(group, consumer, stream) {
+        if (!this.shouldConsume) {
+            this.logger.info(`router-stream-stopped`, {
+                group,
+                consumer,
+                stream,
+            });
+        }
+        return !this.shouldConsume;
+    }
+    async startConsuming(router) {
+        this.shouldConsume = true;
+        InstanceRegistry.add(router);
+    }
+    async stopConsuming(router) {
+        this.shouldConsume = false;
+        this.logger.info(`router-stream-stopping`, this.topic ? { topic: this.topic } : undefined);
+        InstanceRegistry.remove(router);
+        // If using notifications, properly clean up
+        if (this.isUsingNotifications && this.stream.stopNotificationConsumer) {
+            try {
+                await this.stream.cleanup?.();
+            }
+            catch (error) {
+                this.logger.error('router-stream-cleanup-error', { error });
+            }
+        }
+    }
+}
+exports.LifecycleManager = LifecycleManager;

package/build/services/router/telemetry/index.d.ts

@@ -0,0 +1,11 @@
+import { StreamData, StreamRole } from '../../../types/stream';
+export declare class RouterTelemetry {
+    private telemetryService;
+    constructor(appId: string);
+    startStreamSpan(input: StreamData, role: StreamRole): void;
+    setStreamError(error: string): void;
+    setStreamErrorFromOutput(output: any): void;
+    setStreamErrorFromException(err: Error): void;
+    setStreamAttributes(attributes: Record<string, any>): void;
+    endStreamSpan(): void;
+}

package/build/services/router/telemetry/index.js

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RouterTelemetry = void 0;
+const telemetry_1 = require("../../telemetry");
+const stream_1 = require("../../../types/stream");
+const config_1 = require("../config");
+class RouterTelemetry {
+    constructor(appId) {
+        this.telemetryService = new telemetry_1.TelemetryService(appId);
+    }
+    startStreamSpan(input, role) {
+        this.telemetryService.startStreamSpan(input, role);
+    }
+    setStreamError(error) {
+        this.telemetryService.setStreamError(error);
+    }
+    setStreamErrorFromOutput(output) {
+        if (output?.status === stream_1.StreamStatus.ERROR) {
+            this.telemetryService.setStreamError(`Function Status Code ${output.code || config_1.HMSH_CODE_UNKNOWN}`);
+        }
+    }
+    setStreamErrorFromException(err) {
+        this.telemetryService.setStreamError(err.message);
+    }
+    setStreamAttributes(attributes) {
+        this.telemetryService.setStreamAttributes(attributes);
+    }
+    endStreamSpan() {
+        this.telemetryService.endStreamSpan();
+    }
+}
+exports.RouterTelemetry = RouterTelemetry;

package/build/services/router/throttling/index.d.ts

@@ -0,0 +1,23 @@
+export declare class ThrottleManager {
+    private throttle;
+    private isSleeping;
+    private sleepPromiseResolve;
+    private innerPromiseResolve;
+    private sleepTimeout;
+    constructor(initialThrottle?: number);
+    getThrottle(): number;
+    setThrottle(delayInMillis: number): void;
+    isPaused(): boolean;
+    /**
+     * An adjustable throttle that will interrupt a sleeping
+     * router if the throttle is reduced and the sleep time
+     * has elapsed. If the throttle is increased, or if
+     * the sleep time has not elapsed, the router will continue
+     * to sleep until the new termination point. This
+     * allows for dynamic, elastic throttling with smooth
+     * acceleration and deceleration.
+     */
+    customSleep(): Promise<void>;
+    cancelThrottle(): void;
+    private resetThrottleState;
+}

package/build/services/router/throttling/index.js

@@ -0,0 +1,76 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ThrottleManager = void 0;
+const config_1 = require("../config");
+class ThrottleManager {
+    constructor(initialThrottle = 0) {
+        this.throttle = 0;
+        this.isSleeping = false;
+        this.sleepPromiseResolve = null;
+        this.innerPromiseResolve = null;
+        this.sleepTimeout = null;
+        this.throttle = initialThrottle;
+    }
+    getThrottle() {
+        return this.throttle;
+    }
+    setThrottle(delayInMillis) {
+        const wasDecreased = delayInMillis < this.throttle;
+        this.throttle = delayInMillis;
+        // If the throttle was decreased, and we're in the middle of a sleep cycle, adjust immediately
+        if (wasDecreased) {
+            if (this.sleepTimeout) {
+                clearTimeout(this.sleepTimeout);
+            }
+            if (this.innerPromiseResolve) {
+                this.innerPromiseResolve();
+            }
+        }
+    }
+    isPaused() {
+        return this.throttle === config_1.MAX_DELAY;
+    }
+    /**
+     * An adjustable throttle that will interrupt a sleeping
+     * router if the throttle is reduced and the sleep time
+     * has elapsed. If the throttle is increased, or if
+     * the sleep time has not elapsed, the router will continue
+     * to sleep until the new termination point. This
+     * allows for dynamic, elastic throttling with smooth
+     * acceleration and deceleration.
+     */
+    async customSleep() {
+        if (this.throttle === 0)
+            return;
+        if (this.isSleeping)
+            return;
+        this.isSleeping = true;
+        const startTime = Date.now(); //anchor the origin
+        await new Promise(async (outerResolve) => {
+            this.sleepPromiseResolve = outerResolve;
+            let elapsedTime = Date.now() - startTime;
+            while (elapsedTime < this.throttle) {
+                await new Promise((innerResolve) => {
+                    this.innerPromiseResolve = innerResolve;
+                    this.sleepTimeout = setTimeout(innerResolve, this.throttle - elapsedTime);
+                });
+                elapsedTime = Date.now() - startTime;
+            }
+            this.resetThrottleState();
+            outerResolve();
+        });
+    }
+    cancelThrottle() {
+        if (this.sleepTimeout) {
+            clearTimeout(this.sleepTimeout);
+        }
+        this.resetThrottleState();
+    }
+    resetThrottleState() {
+        this.sleepPromiseResolve = null;
+        this.innerPromiseResolve = null;
+        this.isSleeping = false;
+        this.sleepTimeout = null;
+    }
+}
+exports.ThrottleManager = ThrottleManager;

package/build/services/search/factory.d.ts

@@ -0,0 +1,7 @@
+import { ILogger } from '../logger';
+import { ProviderClient } from '../../types/provider';
+import { SearchService } from './index';
+declare class SearchServiceFactory {
+    static init(providerClient: ProviderClient, storeProviderClient: ProviderClient | undefined, namespace: string, appId: string, logger: ILogger): Promise<SearchService<ProviderClient>>;
+}
+export { SearchServiceFactory };

package/build/services/search/factory.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SearchServiceFactory = void 0;
+const utils_1 = require("../../modules/utils");
+const postgres_1 = require("./providers/postgres/postgres");
+const ioredis_1 = require("./providers/redis/ioredis");
+const redis_1 = require("./providers/redis/redis");
+class SearchServiceFactory {
+    static async init(providerClient, storeProviderClient, namespace, appId, logger) {
+        let service;
+        if ((0, utils_1.identifyProvider)(providerClient) === 'postgres') {
+            service = new postgres_1.PostgresSearchService(providerClient, storeProviderClient);
+        }
+        else if ((0, utils_1.identifyProvider)(providerClient) === 'redis') {
+            service = new redis_1.RedisSearchService(providerClient, storeProviderClient);
+        }
+        else {
+            service = new ioredis_1.IORedisSearchService(providerClient, storeProviderClient);
+        }
+        await service.init(namespace, appId, logger);
+        return service;
+    }
+}
+exports.SearchServiceFactory = SearchServiceFactory;

package/build/services/search/index.d.ts

@@ -0,0 +1,23 @@
+import { ILogger } from '../logger';
+import { ProviderClient } from '../../types/provider';
+declare abstract class SearchService<ClientProvider extends ProviderClient> {
+    protected searchClient: ClientProvider;
+    protected storeClient: ClientProvider;
+    protected namespace: string;
+    protected logger: ILogger;
+    protected appId: string;
+    constructor(searchClient: ClientProvider, storeClient?: ClientProvider);
+    abstract init(namespace: string, appId: string, logger: ILogger): Promise<void>;
+    abstract createSearchIndex(indexName: string, prefixes: string[], schema: string[]): Promise<void>;
+    abstract listSearchIndexes(): Promise<string[]>;
+    abstract setFields(key: string, fields: Record<string, string>): Promise<any>;
+    abstract updateContext(key: string, fields: Record<string, string>): Promise<any>;
+    abstract getField(key: string, field: string): Promise<string>;
+    abstract getFields(key: string, fields: string[]): Promise<string[]>;
+    abstract getAllFields(key: string): Promise<Record<string, string>>;
+    abstract deleteFields(key: string, fields: string[]): Promise<number>;
+    abstract incrementFieldByFloat(key: string, field: string, increment: number): Promise<number>;
+    abstract sendQuery(query: any): Promise<any>;
+    abstract sendIndexedQuery(index: string, query: any[]): Promise<any>;
+}
+export { SearchService };

package/build/services/search/index.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SearchService = void 0;
+class SearchService {
+    constructor(searchClient, storeClient) {
+        this.searchClient = searchClient;
+        this.storeClient = storeClient;
+    }
+}
+exports.SearchService = SearchService;

package/build/services/search/providers/postgres/postgres.d.ts

@@ -0,0 +1,25 @@
+import { SearchService } from '../../index';
+import { ILogger } from '../../../logger';
+import { PostgresClientType } from '../../../../types/postgres';
+import { ProviderClient, ProviderTransaction } from '../../../../types/provider';
+declare class PostgresSearchService extends SearchService<PostgresClientType & ProviderClient> {
+    pgClient: PostgresClientType;
+    transact(): ProviderTransaction;
+    constructor(searchClient: PostgresClientType & ProviderClient, storeClient?: PostgresClientType & ProviderClient);
+    init(namespace: string, appId: string, logger: ILogger): Promise<void>;
+    createSearchIndex(indexName: string, prefixes: string[], schema: string[]): Promise<void>;
+    listSearchIndexes(): Promise<string[]>;
+    updateContext(key: string, fields: Record<string, string>): Promise<any>;
+    setFields(key: string, fields: Record<string, string>): Promise<any>;
+    getField(key: string, field: string): Promise<string>;
+    getFields(key: string, fields: string[]): Promise<string[]>;
+    getAllFields(key: string): Promise<Record<string, string>>;
+    deleteFields(key: string, fields: string[]): Promise<number>;
+    incrementFieldByFloat(key: string, field: string, increment: number): Promise<number>;
+    sendQuery(query: string): Promise<any>;
+    /**
+     * assume aggregation type query
+     */
+    sendIndexedQuery(type: string, queryParams?: any[]): Promise<any[]>;
+}
+export { PostgresSearchService };

package/build/services/search/providers/postgres/postgres.js

@@ -0,0 +1,149 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PostgresSearchService = void 0;
+const index_1 = require("../../index");
+const kvsql_1 = require("../../../store/providers/postgres/kvsql");
+class PostgresSearchService extends index_1.SearchService {
+    transact() {
+        return this.storeClient.transact();
+    }
+    constructor(searchClient, storeClient) {
+        super(searchClient, storeClient);
+        this.pgClient = searchClient; //raw pg client (to send raw sql)
+        this.searchClient = new kvsql_1.KVSQL(//wrapped pg client (to send as redis commands)
+        searchClient, this.namespace, this.appId);
+    }
+    async init(namespace, appId, logger) {
+        //bind appId and namespace to searchClient once initialized
+        // (it uses these values to construct keys for the store)
+        this.searchClient.namespace = this.namespace = namespace;
+        this.searchClient.appId = this.appId = appId;
+        this.namespace = namespace;
+        this.appId = appId;
+        this.logger = logger;
+    }
+    async createSearchIndex(indexName, prefixes, schema) {
+        //no-op
+    }
+    async listSearchIndexes() {
+        return [];
+    }
+    async updateContext(key, fields) {
+        try {
+            const result = await this.searchClient.hset(key, fields);
+            return isNaN(result) ? result : Number(result);
+        }
+        catch (error) {
+            this.logger.error(`postgres-search-set-fields-error`, { key, error });
+            throw error;
+        }
+    }
+    async setFields(key, fields) {
+        try {
+            const result = await this.searchClient.hset(key, fields);
+            const isGetOperation = '@context:get' in fields;
+            if (isGetOperation) {
+                return result;
+            }
+            return isNaN(result) ? result : Number(result);
+        }
+        catch (error) {
+            this.logger.error(`postgres-search-set-fields-error`, { key, error });
+            throw error;
+        }
+    }
+    async getField(key, field) {
+        try {
+            return await this.searchClient.hget(key, field);
+        }
+        catch (error) {
+            this.logger.error(`postgres-search-get-field-error`, {
+                key,
+                field,
+                error,
+            });
+            throw error;
+        }
+    }
+    async getFields(key, fields) {
+        try {
+            return await this.searchClient.hmget(key, [...fields]);
+        }
+        catch (error) {
+            this.logger.error(`postgres-search-get-fields-error`, {
+                key,
+                fields,
+                error,
+            });
+            throw error;
+        }
+    }
+    async getAllFields(key) {
+        try {
+            return await this.searchClient.hgetall(key);
+        }
+        catch (error) {
+            this.logger.error(`postgres-search-get-all-fields-error`, {
+                key,
+                error,
+            });
+            throw error;
+        }
+    }
+    async deleteFields(key, fields) {
+        try {
+            const result = await this.searchClient.hdel(key, fields);
+            return Number(result);
+        }
+        catch (error) {
+            this.logger.error(`postgres-search-delete-fields-error`, {
+                key,
+                fields,
+                error,
+            });
+            throw error;
+        }
+    }
+    async incrementFieldByFloat(key, field, increment) {
+        try {
+            const result = await this.searchClient.hincrbyfloat(key, field, increment);
+            return Number(result);
+        }
+        catch (error) {
+            this.logger.error(`postgres-increment-field-error`, {
+                key,
+                field,
+                error,
+            });
+            throw error;
+        }
+    }
+    async sendQuery(query) {
+        try {
+            //exec raw sql (local call, not meant for external use); return raw result
+            return await this.pgClient.query(query);
+        }
+        catch (error) {
+            this.logger.error(`postgres-send-query-error`, { query, error });
+            throw error;
+        }
+    }
+    /**
+     * assume aggregation type query
+     */
+    async sendIndexedQuery(type, queryParams = []) {
+        const [sql, ...params] = queryParams;
+        try {
+            const res = await this.pgClient.query(sql, params.length ? params : undefined);
+            return res.rows;
+        }
+        catch (error) {
+            this.logger.error(`postgres-send-indexed-query-error`, {
+                query: sql,
+                error,
+            });
+            throw error;
+        }
+    }
+}
+exports.PostgresSearchService = PostgresSearchService;

package/build/services/search/providers/redis/ioredis.d.ts

@@ -0,0 +1,19 @@
+import { SearchService } from '../../index';
+import { ILogger } from '../../../logger';
+import { IORedisClientType } from '../../../../types/redis';
+declare class IORedisSearchService extends SearchService<IORedisClientType> {
+    constructor(searchClient: IORedisClientType, storeClient?: IORedisClientType);
+    init(namespace: string, appId: string, logger: ILogger): Promise<void>;
+    createSearchIndex(indexName: string, prefixes: string[], schema: string[]): Promise<void>;
+    listSearchIndexes(): Promise<string[]>;
+    updateContext(key: string, fields: Record<string, string>): Promise<any>;
+    setFields(key: string, fields: Record<string, string>): Promise<number>;
+    getField(key: string, field: string): Promise<string>;
+    getFields(key: string, fields: string[]): Promise<string[]>;
+    getAllFields(key: string): Promise<Record<string, string>>;
+    deleteFields(key: string, fields: string[]): Promise<number>;
+    incrementFieldByFloat(key: string, field: string, increment: number): Promise<number>;
+    sendQuery(...query: [string, ...string[]]): Promise<any>;
+    sendIndexedQuery(index: string, query: string[]): Promise<string[]>;
+}
+export { IORedisSearchService };

package/build/services/search/providers/redis/ioredis.js

@@ -0,0 +1,121 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IORedisSearchService = void 0;
+const index_1 = require("../../index");
+class IORedisSearchService extends index_1.SearchService {
+    constructor(searchClient, storeClient) {
+        super(searchClient, storeClient);
+    }
+    async init(namespace, appId, logger) {
+        this.namespace = namespace;
+        this.appId = appId;
+        this.logger = logger;
+    }
+    async createSearchIndex(indexName, prefixes, schema) {
+        try {
+            await this.searchClient.call('FT.CREATE', indexName, 'ON', 'HASH', 'PREFIX', prefixes.length.toString(), ...prefixes, 'SCHEMA', ...schema);
+        }
+        catch (error) {
+            this.logger.info('Error creating search index', { error });
+            throw error;
+        }
+    }
+    async listSearchIndexes() {
+        try {
+            const indexes = await this.searchClient.call('FT._LIST');
+            return indexes;
+        }
+        catch (error) {
+            this.logger.info('Error listing search indexes', { error });
+            throw error;
+        }
+    }
+    async updateContext(key, fields) {
+        //no-op;
+        throw new Error('Not implemented');
+    }
+    async setFields(key, fields) {
+        try {
+            const result = await this.searchClient.hset(key, fields);
+            return Number(result);
+        }
+        catch (error) {
+            this.logger.error(`Error setting fields for key: ${key}`, { error });
+            throw error;
+        }
+    }
+    async getField(key, field) {
+        try {
+            return await this.searchClient.hget(key, field);
+        }
+        catch (error) {
+            this.logger.error(`Error getting field ${field} for key: ${key}`, {
+                error,
+            });
+            throw error;
+        }
+    }
+    async getFields(key, fields) {
+        try {
+            return await this.searchClient.hmget(key, [...fields]);
+        }
+        catch (error) {
+            this.logger.error(`Error getting fields for key: ${key}`, { error });
+            throw error;
+        }
+    }
+    async getAllFields(key) {
+        try {
+            return await this.searchClient.hgetall(key);
+        }
+        catch (error) {
+            this.logger.error(`Error getting fields for key: ${key}`, { error });
+            throw error;
+        }
+    }
+    async deleteFields(key, fields) {
+        try {
+            const result = await this.searchClient.hdel(key, ...fields);
+            return Number(result);
+        }
+        catch (error) {
+            this.logger.error(`Error deleting fields for key: ${key}`, { error });
+            throw error;
+        }
+    }
+    async incrementFieldByFloat(key, field, increment) {
+        try {
+            const result = await this.searchClient.hincrbyfloat(key, field, increment);
+            return Number(result);
+        }
+        catch (error) {
+            this.logger.error(`Error incrementing field ${field} for key: ${key}`, {
+                error,
+            });
+            throw error;
+        }
+    }
+    async sendQuery(...query) {
+        try {
+            return await this.searchClient.call(...query);
+        }
+        catch (error) {
+            this.logger.error('Error executing query', { error });
+            throw error;
+        }
+    }
+    async sendIndexedQuery(index, query) {
+        try {
+            if (query[0]?.startsWith('FT.')) {
+                const [cmd, ...rest] = query;
+                return (await this.searchClient.call(cmd, ...rest));
+            }
+            return (await this.searchClient.call('FT.SEARCH', index, ...query));
+        }
+        catch (error) {
+            this.logger.error('Error executing query', { error });
+            throw error;
+        }
+    }
+}
+exports.IORedisSearchService = IORedisSearchService;

package/build/services/search/providers/redis/redis.d.ts

@@ -0,0 +1,19 @@
+import { SearchService } from '../../index';
+import { ILogger } from '../../../logger';
+import { RedisRedisClientType } from '../../../../types/redis';
+declare class RedisSearchService extends SearchService<RedisRedisClientType> {
+    constructor(searchClient: RedisRedisClientType, storeClient?: RedisRedisClientType);
+    init(namespace: string, appId: string, logger: ILogger): Promise<void>;
+    createSearchIndex(indexName: string, prefixes: string[], schema: string[]): Promise<void>;
+    listSearchIndexes(): Promise<string[]>;
+    updateContext(key: string, fields: Record<string, string>): Promise<any>;
+    setFields(key: string, fields: Record<string, string>): Promise<number>;
+    getField(key: string, field: string): Promise<string>;
+    getFields(key: string, fields: string[]): Promise<string[]>;
+    getAllFields(key: string): Promise<Record<string, string>>;
+    deleteFields(key: string, fields: string[]): Promise<number>;
+    incrementFieldByFloat(key: string, field: string, increment: number): Promise<number>;
+    sendQuery(...query: any[]): Promise<any>;
+    sendIndexedQuery(index: string, query: string[]): Promise<string[]>;
+}
+export { RedisSearchService };