@hotmeshio/hotmesh 0.4.0 → 0.4.1

package/build/services/stream/providers/redis/ioredis.js

@@ -0,0 +1,272 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IORedisStreamService = void 0;
+const index_1 = require("../../index");
+const utils_1 = require("../../../../modules/utils");
+const key_1 = require("../../../../modules/key");
+const enums_1 = require("../../../../modules/enums");
+class IORedisStreamService extends index_1.StreamService {
+    constructor(streamClient, storeClient, config = {}) {
+        super(streamClient, storeClient, config);
+    }
+    async init(namespace, appId, logger) {
+        this.namespace = namespace;
+        this.logger = logger;
+        this.appId = appId;
+    }
+    mintKey(type, params) {
+        if (!this.namespace)
+            throw new Error('namespace not set');
+        return key_1.KeyService.mintKey(this.namespace, type, {
+            ...params,
+            appId: this.appId,
+        });
+    }
+    transact() {
+        return this.streamClient.multi();
+    }
+    // Core streaming operations
+    async createStream(streamName) {
+        try {
+            // streams are created when you add messages.
+            // To create an empty stream, we can add and delete a dummy message.
+            const dummyId = await this.streamClient.xadd(streamName, '*', 'field', 'value');
+            await this.streamClient.xdel(streamName, dummyId);
+            return true;
+        }
+        catch (error) {
+            this.logger.error(`Error creating stream ${streamName}`, { error });
+            throw error;
+        }
+    }
+    async deleteStream(streamName) {
+        try {
+            const result = await this.streamClient.del(streamName);
+            return result > 0;
+        }
+        catch (error) {
+            this.logger.error(`Error deleting stream ${streamName}`, { error });
+            throw error;
+        }
+    }
+    // Consumer group operations
+    async createConsumerGroup(key, groupName) {
+        try {
+            return ((await this.storeClient.xgroup('CREATE', key, groupName, '$', 'MKSTREAM')) === 'OK');
+        }
+        catch (err) {
+            this.logger.debug('stream-mkstream-caught', { key, group: groupName });
+            throw err;
+        }
+    }
+    async deleteConsumerGroup(streamName, groupName) {
+        try {
+            const result = await this.streamClient.xgroup('DESTROY', streamName, groupName);
+            return result === 1;
+        }
+        catch (error) {
+            this.logger.error(`Error deleting consumer group ${groupName} for stream ${streamName}`, { error });
+            throw error;
+        }
+    }
+    // Message operations
+    async publishMessages(streamName, messages, options) {
+        try {
+            const multi = options?.transaction ||
+                (messages.length > 1 && this.storeClient.multi());
+            let response;
+            for (const message of messages) {
+                response = await (multi || this.storeClient).xadd(streamName, '*', 'message', message);
+            }
+            if (multi && !options?.transaction) {
+                //only exec if we created the multi;
+                //otherwise caller is responsible
+                return (await multi.exec()).map((result) => result[1]);
+            }
+            else {
+                return [response];
+            }
+        }
+        catch (error) {
+            this.logger.error(`ioredis-xadd-error key: ${streamName}`, { error });
+            throw error;
+        }
+    }
+    async consumeMessages(streamName, groupName, consumerName, options) {
+        try {
+            const result = await this.streamClient.xreadgroup('GROUP', groupName, consumerName, 'BLOCK', options?.blockTimeout ?? enums_1.HMSH_BLOCK_TIME_MS, 'STREAMS', streamName, '>');
+            const response = [];
+            if ((0, utils_1.isStreamMessage)(result)) {
+                const [[, messages]] = result;
+                for (const [id, message] of messages) {
+                    response.push({
+                        id,
+                        data: (0, utils_1.parseStreamMessage)(message[1]),
+                    });
+                }
+            }
+            else {
+                return [];
+            }
+            return response;
+        }
+        catch (error) {
+            this.logger.error(`Error consuming messages from ${streamName}`, {
+                error,
+            });
+            throw error;
+        }
+    }
+    async ackAndDelete(stream, group, ids) {
+        const multi = this.storeClient.multi();
+        this.acknowledgeMessages(stream, group, ids, { multi });
+        this.deleteMessages(stream, group, ids, { multi });
+        await multi.exec();
+        return ids.length;
+    }
+    async acknowledgeMessages(stream, group, ids, options) {
+        try {
+            if (options?.multi) {
+                options.multi.xack(stream, group, ...ids);
+                return options.multi;
+            }
+            else {
+                return await this.streamClient.xack(stream, group, ...ids);
+            }
+        }
+        catch (error) {
+            this.logger.error(`Error in acknowledging messages: [${ids}] in group: ${group} for key: ${stream}`, { error });
+            throw error;
+        }
+    }
+    async deleteMessages(stream, group, ids, options) {
+        try {
+            if (options?.multi) {
+                options.multi.xdel(stream, ...ids);
+                return options.multi;
+            }
+            else {
+                return await this.streamClient.xdel(stream, ...ids);
+            }
+        }
+        catch (error) {
+            this.logger.error(`Error in deleting messages: ${ids} for key: ${stream}`, { error });
+            throw error;
+        }
+    }
+    async getPendingMessages(stream, group, count, consumer) {
+        const start = '-';
+        const end = '+';
+        try {
+            const args = [stream, group];
+            if (start)
+                args.push(start);
+            if (end)
+                args.push(end);
+            if (count !== undefined)
+                args.push(count.toString());
+            if (consumer)
+                args.push(consumer);
+            try {
+                return (await this.streamClient.call('XPENDING', ...args));
+            }
+            catch (error) {
+                this.logger.error('err, args', { error }, args);
+            }
+        }
+        catch (error) {
+            this.logger.error(`Error in retrieving pending messages for [stream ${stream}], [group ${group}]`, { error });
+            throw error;
+        }
+    }
+    // Retry Method
+    async retryMessages(streamName, groupName, options) {
+        let pendingMessages = [];
+        const pendingMessagesInfo = await this.getPendingMessages(streamName, groupName, options?.limit); //[[ '1688768134881-0', 'testConsumer1', 1017, 1 ]]
+        for (const pendingMessageInfo of pendingMessagesInfo) {
+            if (Array.isArray(pendingMessageInfo)) {
+                const [id, , elapsedTimeMs, deliveryCount] = pendingMessageInfo;
+                if (elapsedTimeMs > options?.minIdleTime) {
+                    const reclaimedMessage = await this.claimMessage(streamName, groupName, options?.consumerName, options?.minIdleTime, id);
+                    pendingMessages = pendingMessages.concat(reclaimedMessage);
+                }
+            }
+        }
+        return pendingMessages;
+    }
+    async claimMessage(streamName, groupName, consumerName, minIdleTime, messageId, ...args) {
+        try {
+            const message = (await this.streamClient.xclaim(streamName, groupName, consumerName, minIdleTime, messageId, ...args));
+            return {
+                id: message[0][0],
+                data: (0, utils_1.parseStreamMessage)(message[0][1][1]),
+            };
+        }
+        catch (error) {
+            this.logger.error(`Error in claiming message with id: ${messageId} in group: ${groupName} for key: ${streamName}`, { error });
+            throw error;
+        }
+    }
+    async getStreamStats(streamName) {
+        return {
+            messageCount: await this.getStreamDepth(streamName),
+        };
+    }
+    async getStreamDepth(streamName, options) {
+        try {
+            if (options?.multi) {
+                options.multi.xlen(streamName);
+                return 0;
+            }
+            const length = await this.streamClient.xlen(streamName);
+            return length;
+        }
+        catch (error) {
+            this.logger.error(`Error getting depth for ${streamName}`, { error });
+            throw error;
+        }
+    }
+    async getStreamDepths(streamNames) {
+        const multi = this.storeClient.multi();
+        const uniqueStreams = new Map(); // to store unique streams
+        // Add unique streams to the multi command
+        streamNames.forEach((profile) => {
+            if (!uniqueStreams.has(profile.stream)) {
+                uniqueStreams.set(profile.stream, -1); // initialize depth to -1 as a placeholder
+                this.getStreamDepth(profile.stream, { multi });
+            }
+        });
+        // Execute all commands
+        const streamDepthResults = (await multi.exec());
+        // Update the uniqueStreams map with the actual depths from multi.exec() results
+        Array.from(uniqueStreams.keys()).forEach((stream, idx) => {
+            uniqueStreams.set(stream, streamDepthResults[idx][1]);
+        });
+        // Map back to the original `streamNames` array with correct depths
+        const updatedNames = streamNames.map((profile) => {
+            return {
+                stream: profile.stream,
+                depth: uniqueStreams.get(profile.stream) || 0,
+            };
+        });
+        return updatedNames;
+    }
+    async trimStream(streamName, options) {
+        //no-op for now
+        return 0;
+    }
+    // Provider-specific helpers
+    getProviderSpecificFeatures() {
+        return {
+            supportsBatching: true,
+            supportsDeadLetterQueue: false,
+            supportsOrdering: true,
+            supportsTrimming: true,
+            supportsRetry: true,
+            supportsNotifications: false,
+            maxMessageSize: 512 * 1024 * 1024,
+            maxBatchSize: 1000,
+        };
+    }
+}
+exports.IORedisStreamService = IORedisStreamService;

package/build/services/stream/providers/redis/redis.d.ts

@@ -0,0 +1,61 @@
+import { ILogger } from '../../../logger';
+import { StreamService } from '../../index';
+import { RedisRedisClientType, RedisRedisMultiType } from '../../../../types/redis';
+import { PublishMessageConfig, StreamConfig, StreamMessage, StreamStats } from '../../../../types/stream';
+import { KeyStoreParams, StringAnyType } from '../../../../types';
+import { KeyType } from '../../../../modules/key';
+import { ProviderTransaction } from '../../../../types/provider';
+declare class RedisStreamService extends StreamService<RedisRedisClientType, RedisRedisMultiType> {
+    constructor(streamClient: RedisRedisClientType, storeClient: RedisRedisClientType, config?: StreamConfig);
+    init(namespace: string, appId: string, logger: ILogger): Promise<void>;
+    mintKey(type: KeyType, params: KeyStoreParams): string;
+    transact(): ProviderTransaction;
+    createStream(streamName: string): Promise<boolean>;
+    deleteStream(streamName: string): Promise<boolean>;
+    createConsumerGroup(streamName: string, groupName: string): Promise<boolean>;
+    deleteConsumerGroup(streamName: string, groupName: string): Promise<boolean>;
+    publishMessages(streamName: string, messages: string[], options?: PublishMessageConfig): Promise<string[]>;
+    consumeMessages(streamName: string, groupName: string, consumerName: string, options?: {
+        batchSize?: number;
+        blockTimeout?: number;
+    }): Promise<StreamMessage[]>;
+    ackAndDelete(stream: string, group: string, ids: string[]): Promise<number>;
+    acknowledgeMessages(stream: string, group: string, ids: string[], options?: StringAnyType): Promise<number | RedisRedisMultiType>;
+    deleteMessages(stream: string, group: string, ids: string[], options?: StringAnyType): Promise<number | RedisRedisMultiType>;
+    getPendingMessages(stream: string, group: string, count?: number, consumer?: string): Promise<[string, string, number, [string, number][]][] | [string, string, number, number]>;
+    retryMessages(streamName: string, groupName: string, options?: {
+        consumerName?: string;
+        minIdleTime?: number;
+        messageIds?: string[];
+        delay?: number;
+        maxRetries?: number;
+        limit?: number;
+    }): Promise<StreamMessage[]>;
+    claimMessage(stream: string, group: string, consumer: string, minIdleTime: number, id: string, ...args: string[]): Promise<StreamMessage>;
+    getStreamStats(streamName: string): Promise<StreamStats>;
+    getStreamDepth(streamName: string, options?: {
+        multi: RedisRedisMultiType;
+    }): Promise<number>;
+    getStreamDepths(streamNames: {
+        stream: string;
+    }[]): Promise<{
+        stream: string;
+        depth: number;
+    }[]>;
+    trimStream(streamName: string, options: {
+        maxLen?: number;
+        maxAge?: number;
+        exactLimit?: boolean;
+    }): Promise<number>;
+    getProviderSpecificFeatures(): {
+        supportsBatching: boolean;
+        supportsDeadLetterQueue: boolean;
+        supportsOrdering: boolean;
+        supportsTrimming: boolean;
+        supportsRetry: boolean;
+        supportsNotifications: boolean;
+        maxMessageSize: number;
+        maxBatchSize: number;
+    };
+}
+export { RedisStreamService };

package/build/services/stream/providers/redis/redis.js

@@ -0,0 +1,305 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedisStreamService = void 0;
+const index_1 = require("../../index");
+const utils_1 = require("../../../../modules/utils");
+const key_1 = require("../../../../modules/key");
+const enums_1 = require("../../../../modules/enums");
+class RedisStreamService extends index_1.StreamService {
+    constructor(streamClient, storeClient, config = {}) {
+        super(streamClient, storeClient, config);
+    }
+    async init(namespace, appId, logger) {
+        this.namespace = namespace;
+        this.logger = logger;
+        this.appId = appId;
+    }
+    mintKey(type, params) {
+        if (!this.namespace)
+            throw new Error('namespace not set');
+        return key_1.KeyService.mintKey(this.namespace, type, {
+            ...params,
+            appId: this.appId,
+        });
+    }
+    transact() {
+        return this.streamClient.multi();
+    }
+    // Core streaming operations
+    async createStream(streamName) {
+        try {
+            // streams are created when you add messages.
+            // To create an empty stream, we can add and delete a dummy message.
+            const dummyId = await this.streamClient.XADD(streamName, '*', {
+                field: 'value',
+            });
+            await this.streamClient.XDEL(streamName, dummyId);
+            return true;
+        }
+        catch (error) {
+            this.logger.error(`Error creating stream ${streamName}`, { error });
+            throw error;
+        }
+    }
+    async deleteStream(streamName) {
+        try {
+            const result = await this.streamClient.DEL(streamName);
+            return result > 0;
+        }
+        catch (error) {
+            this.logger.error(`Error deleting stream ${streamName}`, { error });
+            throw error;
+        }
+    }
+    // Consumer group operations
+    async createConsumerGroup(streamName, groupName) {
+        try {
+            const result = (await this.storeClient.sendCommand([
+                'XGROUP',
+                'CREATE',
+                streamName,
+                groupName,
+                '$',
+                'MKSTREAM',
+            ]));
+            return result === 'OK';
+        }
+        catch (error) {
+            const streamType = 'with MKSTREAM';
+            this.logger.debug(`x-group-error ${streamType} for key: ${streamName} and group: ${groupName}`, { error });
+            throw error;
+        }
+    }
+    async deleteConsumerGroup(streamName, groupName) {
+        try {
+            const result = await this.streamClient.xGroupDestroy(streamName, groupName);
+            return result;
+        }
+        catch (error) {
+            this.logger.error(`Error deleting consumer group ${groupName} for stream ${streamName}`, { error });
+            throw error;
+        }
+    }
+    async publishMessages(streamName, messages, options) {
+        try {
+            const multi = options?.transaction ||
+                (messages.length > 1 && this.storeClient.multi());
+            let response;
+            for (const message of messages) {
+                response = await (multi || this.storeClient).XADD(streamName, '*', {
+                    message: message,
+                });
+            }
+            if (multi && !options?.transaction) {
+                //only exec if we created the multi;
+                //otherwise caller is responsible
+                return (await multi.exec()).map((result) => result[1]);
+            }
+            else {
+                return [response];
+            }
+        }
+        catch (error) {
+            this.logger.error(`ioredis-xadd-error key: ${streamName}`, { error });
+            throw error;
+        }
+    }
+    async consumeMessages(streamName, groupName, consumerName, options) {
+        const command = 'GROUP';
+        const blockOption = 'BLOCK';
+        const streamsOption = 'STREAMS';
+        const id = '>';
+        try {
+            const result = await this.streamClient.sendCommand([
+                'XREADGROUP',
+                command,
+                groupName,
+                consumerName,
+                blockOption,
+                options?.blockTimeout?.toString() ?? enums_1.HMSH_BLOCK_TIME_MS.toString(),
+                streamsOption,
+                streamName,
+                id,
+            ]);
+            const response = [];
+            if ((0, utils_1.isStreamMessage)(result)) {
+                const [[, messages]] = result;
+                for (const [id, message] of messages) {
+                    response.push({
+                        id,
+                        data: (0, utils_1.parseStreamMessage)(message[1]),
+                    });
+                }
+            }
+            else {
+                return [];
+            }
+            return response;
+        }
+        catch (error) {
+            this.logger.error(`Error consuming messages from ${streamName}`, {
+                error,
+            });
+            throw error;
+        }
+    }
+    async ackAndDelete(stream, group, ids) {
+        const multi = this.storeClient.multi();
+        await this.acknowledgeMessages(stream, group, ids, { multi });
+        await this.deleteMessages(stream, group, ids, { multi });
+        await multi.exec();
+        return ids.length;
+    }
+    async acknowledgeMessages(stream, group, ids, options) {
+        try {
+            if (options?.multi) {
+                options.multi.XACK(stream, group, ids);
+                return options.multi;
+            }
+            else {
+                return await this.streamClient.XACK(stream, group, ...ids);
+            }
+        }
+        catch (error) {
+            this.logger.error(`Error in acknowledging messages in group: ${group} for key: ${stream}`, { error });
+            throw error;
+        }
+    }
+    async deleteMessages(stream, group, ids, options) {
+        try {
+            if (options?.multi) {
+                options.multi.xDel(stream, ids);
+                return options.multi;
+            }
+            else {
+                return await this.streamClient.XDEL(stream, ...ids);
+            }
+        }
+        catch (error) {
+            this.logger.error(`Error in deleting messages with ids: ${ids.join(',')} for key: ${stream}`, { error });
+            throw error;
+        }
+    }
+    async getPendingMessages(stream, group, count, consumer) {
+        try {
+            const start = '-';
+            const end = '+';
+            const args = [stream, group];
+            if (start)
+                args.push(start);
+            if (end)
+                args.push(end);
+            if (count !== undefined)
+                args.push(count.toString());
+            if (consumer)
+                args.push(consumer);
+            try {
+                return await this.streamClient.sendCommand(['XPENDING', ...args]);
+            }
+            catch (error) {
+                this.logger.error('error, args', { error }, args);
+            }
+        }
+        catch (error) {
+            this.logger.error(`Error retrieving pending messages for group: ${group} in key: ${stream}`, { error });
+            throw error;
+        }
+    }
+    // Retry Method
+    async retryMessages(streamName, groupName, options) {
+        let pendingMessages = [];
+        const pendingMessagesInfo = await this.getPendingMessages(streamName, groupName, options?.limit); //[[ '1688768134881-0', 'testConsumer1', 1017, 1 ]]
+        for (const pendingMessageInfo of pendingMessagesInfo) {
+            if (Array.isArray(pendingMessageInfo)) {
+                const [id, , elapsedTimeMs, deliveryCount] = pendingMessageInfo;
+                if (elapsedTimeMs > options?.minIdleTime) {
+                    const reclaimedMessage = await this.claimMessage(streamName, groupName, options?.consumerName, options?.minIdleTime, id);
+                    pendingMessages = pendingMessages.concat(reclaimedMessage);
+                }
+            }
+        }
+        return pendingMessages;
+    }
+    async claimMessage(stream, group, consumer, minIdleTime, id, ...args) {
+        try {
+            const message = (await this.streamClient.sendCommand([
+                'XCLAIM',
+                stream,
+                group,
+                consumer,
+                minIdleTime.toString(),
+                id,
+                ...args,
+            ]));
+            return {
+                id: message[0][0],
+                data: (0, utils_1.parseStreamMessage)(message[0][1][1]),
+            };
+        }
+        catch (error) {
+            this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${stream}`, { error });
+            throw error;
+        }
+    }
+    async getStreamStats(streamName) {
+        return {
+            messageCount: await this.getStreamDepth(streamName),
+        };
+    }
+    async getStreamDepth(streamName, options) {
+        try {
+            if (options?.multi) {
+                options.multi.XLEN(streamName);
+                return 0;
+            }
+            const length = await this.streamClient.XLEN(streamName);
+            return length;
+        }
+        catch (error) {
+            this.logger.error(`Error getting depth for ${streamName}`, { error });
+            throw error;
+        }
+    }
+    async getStreamDepths(streamNames) {
+        const multi = this.storeClient.multi();
+        const uniqueStreams = new Map(); // Use a Map to track unique streams
+        // Add unique streams to the multi command
+        streamNames.forEach((profile) => {
+            if (!uniqueStreams.has(profile.stream)) {
+                uniqueStreams.set(profile.stream, -1); // Initialize depth to -1 as a placeholder
+                this.getStreamDepth(profile.stream, { multi: multi });
+            }
+        });
+        // Execute all commands in multi (ioredis strips null fields)
+        const streamDepthResults = (await multi.exec());
+        // Update the uniqueStreams map with actual depths from multi.exec() results
+        Array.from(uniqueStreams.keys()).forEach((stream, idx) => {
+            uniqueStreams.set(stream, streamDepthResults[idx]);
+        });
+        // Map back to the original `streamNames` array with correct depths
+        return streamNames.map((profile) => {
+            return {
+                stream: profile.stream,
+                depth: uniqueStreams.get(profile.stream) || 0,
+            };
+        });
+    }
+    async trimStream(streamName, options) {
+        //no-op for now
+        return 0;
+    }
+    // Provider-specific helpers
+    getProviderSpecificFeatures() {
+        return {
+            supportsBatching: true,
+            supportsDeadLetterQueue: false,
+            supportsOrdering: true,
+            supportsTrimming: true,
+            supportsRetry: true,
+            supportsNotifications: false,
+            maxMessageSize: 512 * 1024 * 1024,
+            maxBatchSize: 1000,
+        };
+    }
+}
+exports.RedisStreamService = RedisStreamService;

package/build/services/stream/providers/stream-initializable.d.ts

@@ -0,0 +1,4 @@
+import { ILogger } from '../../logger';
+export interface StreamInitializable {
+    init(namespace: string, appId: string, logger: ILogger): Promise<void>;
+}

package/build/services/stream/providers/stream-initializable.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/build/services/sub/factory.d.ts

@@ -0,0 +1,7 @@
+import { ILogger } from '../logger';
+import { ProviderClient } from '../../types/provider';
+import { SubService } from './index';
+declare class SubServiceFactory {
+    static init(providerSubClient: ProviderClient, providerPubClient: ProviderClient, namespace: string, appId: string, engineId: string, logger: ILogger): Promise<SubService<ProviderClient>>;
+}
+export { SubServiceFactory };

package/build/services/sub/factory.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SubServiceFactory = void 0;
+const utils_1 = require("../../modules/utils");
+const redis_1 = require("./providers/redis/redis");
+const postgres_1 = require("./providers/postgres/postgres");
+const nats_1 = require("./providers/nats/nats");
+const ioredis_1 = require("./providers/redis/ioredis");
+class SubServiceFactory {
+    static async init(providerSubClient, providerPubClient, namespace, appId, engineId, logger) {
+        let service;
+        const providerType = (0, utils_1.identifyProvider)(providerSubClient);
+        if (providerType === 'nats') {
+            service = new nats_1.NatsSubService(providerSubClient, providerPubClient);
+        }
+        else if (providerType === 'redis') {
+            service = new redis_1.RedisSubService(providerSubClient, providerPubClient);
+        }
+        else if (providerType === 'postgres') {
+            service = new postgres_1.PostgresSubService(providerSubClient, providerPubClient);
+        }
+        else {
+            service = new ioredis_1.IORedisSubService(providerSubClient, providerPubClient);
+        }
+        await service.init(namespace, appId, engineId, logger);
+        return service;
+    }
+}
+exports.SubServiceFactory = SubServiceFactory;