@sprout-idws/sprout-redis 1.0.0

package/dist/stream/redis-stream.service.js

@@ -0,0 +1,200 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+var RedisStreamService_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedisStreamService = void 0;
+const common_1 = require("@nestjs/common");
+const sprout_context_1 = require("@sprout-idws/sprout-context");
+const ioredis_1 = __importDefault(require("ioredis"));
+const lua_scripts_loader_1 = require("./lua-scripts.loader");
+let RedisStreamService = RedisStreamService_1 = class RedisStreamService {
+    constructor(redis, contextService) {
+        this.redis = redis;
+        this.contextService = contextService;
+        this.logger = new common_1.Logger(RedisStreamService_1.name);
+        this.luaScriptsLoader = new lua_scripts_loader_1.LuaScriptsLoader();
+    }
+    async publishToStream(streamName, message) {
+        message.context = message.context ?? this.contextService.getCurrentEncodedContext();
+        const args = ['*', 'message', JSON.stringify(message)];
+        return this.redis.xadd(streamName, ...args);
+    }
+    async createConsumerGroup(streamName, groupName, startId = '0') {
+        try {
+            await this.redis.xgroup('CREATE', streamName, groupName, startId, 'MKSTREAM');
+            this.logger.debug(`Created consumer group ${groupName} for stream ${streamName}`);
+        }
+        catch (error) {
+            const err = error;
+            if (!err.message?.includes('BUSYGROUP')) {
+                throw error;
+            }
+        }
+    }
+    async readFromConsumerGroup({ streamName, groupName, consumerName, count = 10, blockTime = 1000, }) {
+        const result = await this.redis.xreadgroup('GROUP', groupName, consumerName, 'COUNT', count, 'BLOCK', blockTime, 'STREAMS', streamName, '>');
+        if (!result || result.length === 0) {
+            return [];
+        }
+        const entries = [];
+        for (const [_, messages] of result) {
+            for (const [id, fields] of messages) {
+                const messageIndex = fields.indexOf('message');
+                if (messageIndex !== -1) {
+                    const messageValue = fields[messageIndex + 1];
+                    const messageData = JSON.parse(messageValue);
+                    entries.push({
+                        id,
+                        data: messageData.data ?? {},
+                        context: messageData.context,
+                        metadata: messageData.metadata,
+                    });
+                }
+            }
+        }
+        return entries;
+    }
+    async sendToDlq(streamName, groupName, messageId) {
+        const dlqStream = `${streamName}:dlq`;
+        const moveToDlqScript = this.luaScriptsLoader.getScript('move-to-dlq.lua');
+        await this.redis.eval(moveToDlqScript, 1, streamName, dlqStream, groupName, messageId);
+    }
+    async ackDelete(streamName, groupName, messageId) {
+        const luaScript = this.luaScriptsLoader.getScript('ack-delete.lua');
+        await this.redis.eval(luaScript, 1, streamName, groupName, messageId);
+    }
+    async delayedRetry(streamName, groupName, messageId, delayedQueueKey, score, member) {
+        const luaScript = this.luaScriptsLoader.getScript('delayed-retry.lua');
+        await this.redis.eval(luaScript, 2, streamName, delayedQueueKey, groupName, messageId, `${score}`, member);
+    }
+    async getAllStreams(pattern = '*') {
+        try {
+            const streams = [];
+            let cursor = '0';
+            do {
+                const [nextCursor, keys] = await this.redis.scan(cursor, 'MATCH', pattern, 'COUNT', 100);
+                cursor = nextCursor;
+                for (const key of keys) {
+                    const type = await this.redis.type(key);
+                    if (type === 'stream') {
+                        streams.push(key);
+                    }
+                }
+            } while (cursor !== '0');
+            return streams;
+        }
+        catch (error) {
+            this.logger.warn(`Failed to get all streams:`, error);
+            return [];
+        }
+    }
+    async getConsumerGroups(streamName) {
+        const result = await this.redis.xinfo('GROUPS', streamName);
+        const groups = [];
+        if (!result) {
+            return [];
+        }
+        if (!Array.isArray(result)) {
+            this.logger.warn(`Unexpected result type from XINFO GROUPS for stream ${streamName}: ${typeof result}`);
+            return [];
+        }
+        for (const groupData of result) {
+            if (Array.isArray(groupData)) {
+                const group = { name: '' };
+                for (let j = 0; j < groupData.length; j += 2) {
+                    const key = groupData[j];
+                    const value = groupData[j + 1];
+                    if (key === 'name') {
+                        group.name = value;
+                        break;
+                    }
+                }
+                if (group.name) {
+                    groups.push(group);
+                }
+            }
+            else {
+                this.logger.debug(`Unexpected group data format in XINFO GROUPS for stream ${streamName}: ${typeof groupData}`);
+            }
+        }
+        return groups;
+    }
+    async deleteConsumer(streamName, groupName, consumerName) {
+        await this.redis.xgroup('DELCONSUMER', streamName, groupName, consumerName);
+    }
+    async republishDelayedMessages(queueKey, currentTime, limit = 1000) {
+        this.logger.debug(`Checking for delayed messages to republish from queue ${queueKey} at ${currentTime}`);
+        const luaScript = this.luaScriptsLoader.getScript('republish-delayed-messages.lua');
+        const republishedCount = (await this.redis.eval(luaScript, 1, queueKey, `${currentTime}`, `${limit}`));
+        if (republishedCount > 0) {
+            this.logger.debug(`Republished ${republishedCount} delayed messages from queue ${queueKey}`);
+        }
+        return republishedCount;
+    }
+    async getConsumers(streamName, groupName) {
+        const result = await this.redis.xinfo('CONSUMERS', streamName, groupName);
+        const consumers = [];
+        if (!result) {
+            return [];
+        }
+        if (!Array.isArray(result)) {
+            this.logger.warn(`Unexpected result type from XINFO CONSUMERS for stream ${streamName}, group ${groupName}: ${typeof result}`);
+            return [];
+        }
+        for (const consumerData of result) {
+            if (!Array.isArray(consumerData)) {
+                continue;
+            }
+            const consumer = {
+                name: '',
+                pending: 0,
+                idle: 0,
+            };
+            for (let j = 0; j < consumerData.length; j += 2) {
+                const key = consumerData[j];
+                const value = consumerData[j + 1];
+                if (key === 'name') {
+                    consumer.name = value;
+                }
+                else if (key === 'pending') {
+                    consumer.pending = Number(value);
+                }
+                else if (key === 'idle') {
+                    consumer.idle = Number(value);
+                }
+            }
+            if (consumer.name) {
+                consumers.push(consumer);
+            }
+        }
+        return consumers;
+    }
+    async republishInactiveConsumerMessages(streamName, groupName, inactiveConsumerName) {
+        const luaScript = this.luaScriptsLoader.getScript('republish-inactive-consumer-messages.lua');
+        const args = [groupName, inactiveConsumerName];
+        const result = await this.redis.eval(luaScript, 1, streamName, ...args);
+        return { republishedCount: result };
+    }
+};
+exports.RedisStreamService = RedisStreamService;
+exports.RedisStreamService = RedisStreamService = RedisStreamService_1 = __decorate([
+    (0, common_1.Injectable)(),
+    __param(0, (0, common_1.Inject)('REDIS_CLIENT')),
+    __metadata("design:paramtypes", [ioredis_1.default,
+        sprout_context_1.ContextService])
+], RedisStreamService);
+//# sourceMappingURL=redis-stream.service.js.map

package/dist/stream/redis-stream.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"redis-stream.service.js","sourceRoot":"","sources":["../../src/stream/redis-stream.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;AAAA,2CAA4D;AAC5D,gEAA6D;AAC7D,sDAA4B;AAC5B,6DAAwD;AAejD,IAAM,kBAAkB,0BAAxB,MAAM,kBAAkB;IAI7B,YAC0B,KAA6B,EACpC,cAA8B;QADN,UAAK,GAAL,KAAK,CAAO;QACpC,mBAAc,GAAd,cAAc,CAAgB;QALhC,WAAM,GAAG,IAAI,eAAM,CAAC,oBAAkB,CAAC,IAAI,CAAC,CAAC;QAO5D,IAAI,CAAC,gBAAgB,GAAG,IAAI,qCAAgB,EAAE,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,UAAkB,EAAE,OAAsB;QAC9D,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc,CAAC,wBAAwB,EAAE,CAAC;QACpF,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;QACvD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,mBAAmB,CACvB,UAAkB,EAClB,SAAiB,EACjB,UAAkB,GAAG;QAErB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;YAC9E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,SAAS,eAAe,UAAU,EAAE,CAAC,CAAC;QACpF,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,MAAM,GAAG,GAAG,KAA6B,CAAC;YAC1C,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBACxC,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAC,EAC1B,UAAU,EACV,SAAS,EACT,YAAY,EACZ,KAAK,GAAG,EAAE,EACV,SAAS,GAAG,IAAI,GAOjB;QACC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CACxC,OAAO,EACP,SAAS,EACT,YAAY,EACZ,OAAO,EACP,KAAK,EACL,OAAO,EACP,SAAS,EACT,SAAS,EACT,UAAU,EACV,GAAG,CACJ,CAAC;QAEF,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,OAAO,GAAoB,EAAE,CAAC;QACpC,KAAK,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,MAAoD,EAAE,CAAC;YACjF,KAAK,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACpC,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;gBAE/C,IAAI,YAAY,KAAK,CAAC,CAAC,EAAE,CAAC;oBACxB,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC;oBAC9C,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAI1C,CAAC;oBAEF,OAAO,CAAC,IAAI,CAAC;wBACX,EAAE;wBACF,IAAI,EAAE,WAAW,CAAC,IAAI,IAAI,EAAE;wBAC5B,OAAO,EAAE,WAAW,CAAC,OAAO;wBAC5B,QAAQ,EAAE,WAAW,CAAC,QAAQ;qBAC/B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,UAAkB,EAAE,SAAiB,EAAE,SAAiB;QACtE,MAAM,SAAS,GAAG,GAAG,UAAU,MAAM,CAAC;QACtC,MAAM,eAAe,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;QAE3E,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACzF,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,UAAkB,EAAE,SAAiB,EAAE,SAAiB;QACtE,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;QAEpE,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACxE,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,UAAkB,EAClB,SAAiB,EACjB,SAAiB,EACjB,eAAuB,EACvB,KAAa,EACb,MAAc;QAEd,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;QAEvE,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CACnB,SAAS,EACT,CAAC,EACD,UAAU,EACV,eAAe,EACf,SAAS,EACT,SAAS,EACT,GAAG,KAAK,EAAE,EACV,MAAM,CACP,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,UAAkB,GAAG;QACvC,IAAI,CAAC;YACH,MAAM,OAAO,GAAa,EAAE,CAAC;YAC7B,IAAI,MAAM,GAAG,GAAG,CAAC;YAEjB,GAAG,CAAC;gBACF,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;gBACzF,MAAM,GAAG,UAAU,CAAC;gBAEpB,KAAK,MAAM,GAAG,IAAI,IAAgB,EAAE,CAAC;oBACnC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACxC,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;wBACtB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACpB,CAAC;gBACH,CAAC;YACH,CAAC,QAAQ,MAAM,KAAK,GAAG,EAAE;YAEzB,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE,KAAK,CAAC,CAAC;YACtD,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,UAAkB;QACxC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAC5D,MAAM,MAAM,GAA4B,EAAE,CAAC;QAE3C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,uDAAuD,UAAU,KAAK,OAAO,MAAM,EAAE,CACtF,CAAC;YACF,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;YAC/B,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC7B,MAAM,KAAK,GAAqB,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;gBAE7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC7C,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;oBACzB,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;oBAE/B,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;wBACnB,KAAK,CAAC,IAAI,GAAG,KAAe,CAAC;wBAC7B,MAAM;oBACR,CAAC;gBACH,CAAC;gBAED,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;oBACf,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACrB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,2DAA2D,UAAU,KAAK,OAAO,SAAS,EAAE,CAC7F,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,UAAkB,EAAE,SAAiB,EAAE,YAAoB;QAC9E,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,wBAAwB,CAC5B,QAAgB,EAChB,WAAmB,EACnB,QAAgB,IAAI;QAEpB,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,yDAAyD,QAAQ,OAAO,WAAW,EAAE,CACtF,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,gCAAgC,CAAC,CAAC;QACpF,MAAM,gBAAgB,GAAG,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAC7C,SAAS,EACT,CAAC,EACD,QAAQ,EACR,GAAG,WAAW,EAAE,EAChB,GAAG,KAAK,EAAE,CACX,CAAW,CAAC;QAEb,IAAI,gBAAgB,GAAG,CAAC,EAAE,CAAC;YACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,gBAAgB,gCAAgC,QAAQ,EAAE,CAAC,CAAC;QAC/F,CAAC;QAED,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,UAAkB,EAClB,SAAiB;QAEjB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;QAC1E,MAAM,SAAS,GAA2D,EAAE,CAAC;QAE7E,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,0DAA0D,UAAU,WAAW,SAAS,KAAK,OAAO,MAAM,EAAE,CAC7G,CAAC;YACF,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,KAAK,MAAM,YAAY,IAAI,MAAM,EAAE,CAAC;YAClC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,CAAC;gBACjC,SAAS;YACX,CAAC;YAED,MAAM,QAAQ,GAAoD;gBAChE,IAAI,EAAE,EAAE;gBACR,OAAO,EAAE,CAAC;gBACV,IAAI,EAAE,CAAC;aACR,CAAC;YAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;gBAChD,MAAM,GAAG,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;gBAC5B,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAElC,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;oBACnB,QAAQ,CAAC,IAAI,GAAG,KAAe,CAAC;gBAClC,CAAC;qBAAM,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;oBAC7B,QAAQ,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBACnC,CAAC;qBAAM,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;oBAC1B,QAAQ,CAAC,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBAChC,CAAC;YACH,CAAC;YAED,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAClB,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,iCAAiC,CACrC,UAAkB,EAClB,SAAiB,EACjB,oBAA4B;QAE5B,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,0CAA0C,CAAC,CAAC;QAE9F,MAAM,IAAI,GAAG,CAAC,SAAS,EAAE,oBAAoB,CAAC,CAAC;QAE/C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,CAAC;QACxE,OAAO,EAAE,gBAAgB,EAAE,MAAgB,EAAE,CAAC;IAChD,CAAC;CACF,CAAA;AAzRY,gDAAkB;6BAAlB,kBAAkB;IAD9B,IAAA,mBAAU,GAAE;IAMR,WAAA,IAAA,eAAM,EAAC,cAAc,CAAC,CAAA;qCAAyB,iBAAK;QACpB,+BAAc;GANtC,kBAAkB,CAyR9B"}

package/dist/stream/stream-consumer-monitor.service.d.ts

@@ -0,0 +1,19 @@
+import { OnModuleInit } from '@nestjs/common';
+import { PeriodicExecutorService } from './periodic-executor.service';
+import { RedisStreamService } from './redis-stream.service';
+export declare class StreamConsumerMonitorService implements OnModuleInit {
+    private readonly redisStreamService;
+    private readonly periodicExecutorService;
+    private readonly logger;
+    private executorInstance;
+    private readonly CONSUMER_MONITOR_INTERNAL_MS;
+    private readonly CONSUMER_IDLE_TIME_THRESHOLD_MULTIPLIER;
+    private get minIdleTimeMs();
+    constructor(redisStreamService: RedisStreamService, periodicExecutorService: PeriodicExecutorService);
+    onModuleInit(): Promise<void>;
+    start(): Promise<void>;
+    private monitorInactiveConsumers;
+    private checkAndClaimInactiveMessages;
+    private republishInactiveConsumerMessages;
+}
+//# sourceMappingURL=stream-consumer-monitor.service.d.ts.map

package/dist/stream/stream-consumer-monitor.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stream-consumer-monitor.service.d.ts","sourceRoot":"","sources":["../../src/stream/stream-consumer-monitor.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAClE,OAAO,EAA4B,uBAAuB,EAAE,MAAM,6BAA6B,CAAC;AAChG,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAG5D,qBACa,4BAA6B,YAAW,YAAY;IAc7D,OAAO,CAAC,QAAQ,CAAC,kBAAkB;IACnC,OAAO,CAAC,QAAQ,CAAC,uBAAuB;IAd1C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiD;IAExE,OAAO,CAAC,gBAAgB,CAAyC;IAEjE,OAAO,CAAC,QAAQ,CAAC,4BAA4B,CAAa;IAE1D,OAAO,CAAC,QAAQ,CAAC,uCAAuC,CAAM;IAE9D,OAAO,KAAK,aAAa,GAExB;gBAGkB,kBAAkB,EAAE,kBAAkB,EACtC,uBAAuB,EAAE,uBAAuB;IAG7D,YAAY;IAKZ,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAed,wBAAwB;YAgBxB,6BAA6B;YAuB7B,iCAAiC;CAuBhD"}

package/dist/stream/stream-consumer-monitor.service.js

@@ -0,0 +1,85 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var StreamConsumerMonitorService_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StreamConsumerMonitorService = void 0;
+const common_1 = require("@nestjs/common");
+const periodic_executor_service_1 = require("./periodic-executor.service");
+const redis_stream_service_1 = require("./redis-stream.service");
+const stream_consumer_service_1 = require("./stream-consumer.service");
+let StreamConsumerMonitorService = StreamConsumerMonitorService_1 = class StreamConsumerMonitorService {
+    get minIdleTimeMs() {
+        return stream_consumer_service_1.CONSUMER_INTERVAL_TIME * this.CONSUMER_IDLE_TIME_THRESHOLD_MULTIPLIER;
+    }
+    constructor(redisStreamService, periodicExecutorService) {
+        this.redisStreamService = redisStreamService;
+        this.periodicExecutorService = periodicExecutorService;
+        this.logger = new common_1.Logger(StreamConsumerMonitorService_1.name);
+        this.executorInstance = null;
+        this.CONSUMER_MONITOR_INTERNAL_MS = 60 * 1000;
+        this.CONSUMER_IDLE_TIME_THRESHOLD_MULTIPLIER = 10;
+    }
+    async onModuleInit() {
+        await this.start();
+        this.logger.log('StreamConsumerMonitorService started');
+    }
+    async start() {
+        if (this.executorInstance?.isActive) {
+            this.logger.warn('Consumer monitor is already running');
+            return;
+        }
+        this.executorInstance = await this.periodicExecutorService.startExecutor({
+            intervalMs: this.CONSUMER_MONITOR_INTERNAL_MS,
+            name: 'stream-consumer-monitor',
+        }, () => this.monitorInactiveConsumers());
+    }
+    async monitorInactiveConsumers() {
+        if (!this.executorInstance?.isActive) {
+            return;
+        }
+        const streams = await this.redisStreamService.getAllStreams();
+        for (const streamName of streams) {
+            const groups = await this.redisStreamService.getConsumerGroups(streamName);
+            for (const group of groups) {
+                await this.checkAndClaimInactiveMessages(streamName, group.name);
+            }
+        }
+    }
+    async checkAndClaimInactiveMessages(streamName, groupName) {
+        const consumers = await this.redisStreamService.getConsumers(streamName, groupName);
+        for (const consumer of consumers) {
+            if (consumer.idle < this.minIdleTimeMs) {
+                continue;
+            }
+            if (consumer.pending > 0) {
+                await this.republishInactiveConsumerMessages(streamName, groupName, consumer);
+            }
+            await this.redisStreamService.deleteConsumer(streamName, groupName, consumer.name);
+            this.logger.warn(`Deleted inactive consumer ${consumer.name} from group ${groupName} in stream ${streamName}`);
+        }
+    }
+    async republishInactiveConsumerMessages(streamName, groupName, consumer) {
+        let totalRepublishedCount = 0;
+        do {
+            const { republishedCount: currentRepublishedCount } = await this.redisStreamService.republishInactiveConsumerMessages(streamName, groupName, consumer.name);
+            totalRepublishedCount += currentRepublishedCount;
+        } while (totalRepublishedCount < consumer.pending);
+        this.logger.warn(`Republished ${totalRepublishedCount} messages from inactive consumer ${consumer.name} in group ${groupName} for stream ${streamName}`);
+        return totalRepublishedCount;
+    }
+};
+exports.StreamConsumerMonitorService = StreamConsumerMonitorService;
+exports.StreamConsumerMonitorService = StreamConsumerMonitorService = StreamConsumerMonitorService_1 = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [redis_stream_service_1.RedisStreamService,
+        periodic_executor_service_1.PeriodicExecutorService])
+], StreamConsumerMonitorService);
+//# sourceMappingURL=stream-consumer-monitor.service.js.map

package/dist/stream/stream-consumer-monitor.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stream-consumer-monitor.service.js","sourceRoot":"","sources":["../../src/stream/stream-consumer-monitor.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAAkE;AAClE,2EAAgG;AAChG,iEAA4D;AAC5D,uEAA2F;AAGpF,IAAM,4BAA4B,oCAAlC,MAAM,4BAA4B;IASvC,IAAY,aAAa;QACvB,OAAO,gDAAoB,GAAG,IAAI,CAAC,uCAAuC,CAAC;IAC7E,CAAC;IAED,YACmB,kBAAsC,EACtC,uBAAgD;QADhD,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,4BAAuB,GAAvB,uBAAuB,CAAyB;QAdlD,WAAM,GAAG,IAAI,eAAM,CAAC,8BAA4B,CAAC,IAAI,CAAC,CAAC;QAEhE,qBAAgB,GAAoC,IAAI,CAAC;QAEhD,iCAA4B,GAAG,EAAE,GAAG,IAAI,CAAC;QAEzC,4CAAuC,GAAG,EAAE,CAAC;IAS3D,CAAC;IAEJ,KAAK,CAAC,YAAY;QAChB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,gBAAgB,EAAE,QAAQ,EAAE,CAAC;YACpC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;YACxD,OAAO;QACT,CAAC;QAED,IAAI,CAAC,gBAAgB,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,aAAa,CACtE;YACE,UAAU,EAAE,IAAI,CAAC,4BAA4B;YAC7C,IAAI,EAAE,yBAAyB;SAChC,EACD,GAAG,EAAE,CAAC,IAAI,CAAC,wBAAwB,EAAE,CACtC,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,wBAAwB;QACpC,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,QAAQ,EAAE,CAAC;YACrC,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,aAAa,EAAE,CAAC;QAE9D,KAAK,MAAM,UAAU,IAAI,OAAO,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;YAE3E,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,MAAM,IAAI,CAAC,6BAA6B,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACnE,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,6BAA6B,CACzC,UAAkB,EAClB,SAAiB;QAEjB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QAEpF,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,IAAI,QAAQ,CAAC,IAAI,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;gBACvC,SAAS;YACX,CAAC;YAED,IAAI,QAAQ,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,CAAC,iCAAiC,CAAC,UAAU,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;YAChF,CAAC;YAED,MAAM,IAAI,CAAC,kBAAkB,CAAC,cAAc,CAAC,UAAU,EAAE,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;YAEnF,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,6BAA6B,QAAQ,CAAC,IAAI,eAAe,SAAS,cAAc,UAAU,EAAE,CAC7F,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,iCAAiC,CAC7C,UAAkB,EAClB,SAAiB,EACjB,QAAyD;QAEzD,IAAI,qBAAqB,GAAG,CAAC,CAAC;QAE9B,GAAG,CAAC;YACF,MAAM,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,GACjD,MAAM,IAAI,CAAC,kBAAkB,CAAC,iCAAiC,CAC7D,UAAU,EACV,SAAS,EACT,QAAQ,CAAC,IAAI,CACd,CAAC;YAEJ,qBAAqB,IAAI,uBAAuB,CAAC;QACnD,CAAC,QAAQ,qBAAqB,GAAG,QAAQ,CAAC,OAAO,EAAE;QAEnD,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,eAAe,qBAAqB,oCAAoC,QAAQ,CAAC,IAAI,aAAa,SAAS,eAAe,UAAU,EAAE,CACvI,CAAC;QACF,OAAO,qBAAqB,CAAC;IAC/B,CAAC;CACF,CAAA;AApGY,oEAA4B;uCAA5B,4BAA4B;IADxC,IAAA,mBAAU,GAAE;qCAe4B,yCAAkB;QACb,mDAAuB;GAfxD,4BAA4B,CAoGxC"}

package/dist/stream/stream-consumer.service.d.ts

@@ -0,0 +1,25 @@
+import { PeriodicExecutorService } from './periodic-executor.service';
+import { RedisStreamService, StreamMessage } from './redis-stream.service';
+import { StreamMessageHandlerService } from './stream-message-handler.service';
+import { StreamRetryService } from './stream-retry.service';
+export interface ConsumerConfig {
+    streamName: string;
+    groupName: string;
+    consumerName: string;
+    maxRetries?: number;
+    retryDelay?: number;
+    blockTime?: number;
+    maxConcurrency?: number;
+    prefetchCount?: number;
+}
+export declare const CONSUMER_INTERVAL_TIME = 1000;
+export declare class StreamConsumerService {
+    private readonly redisStreamService;
+    private readonly messageHandlerService;
+    private readonly streamRetryService;
+    private readonly periodicExecutorService;
+    private readonly logger;
+    constructor(redisStreamService: RedisStreamService, messageHandlerService: StreamMessageHandlerService, streamRetryService: StreamRetryService, periodicExecutorService: PeriodicExecutorService);
+    consumeMessages(config: ConsumerConfig, messageHandler: (entry: StreamMessage) => Promise<void>): Promise<void>;
+}
+//# sourceMappingURL=stream-consumer.service.d.ts.map

package/dist/stream/stream-consumer.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stream-consumer.service.d.ts","sourceRoot":"","sources":["../../src/stream/stream-consumer.service.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,uBAAuB,EAAE,MAAM,6BAA6B,CAAC;AACtE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAC3E,OAAO,EAEL,2BAA2B,EAC5B,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAI5D,MAAM,WAAW,cAAc;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,eAAO,MAAM,sBAAsB,OAAO,CAAC;AAE3C,qBACa,qBAAqB;IAI9B,OAAO,CAAC,QAAQ,CAAC,kBAAkB;IACnC,OAAO,CAAC,QAAQ,CAAC,qBAAqB;IACtC,OAAO,CAAC,QAAQ,CAAC,kBAAkB;IACnC,OAAO,CAAC,QAAQ,CAAC,uBAAuB;IAN1C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0C;gBAG9C,kBAAkB,EAAE,kBAAkB,EACtC,qBAAqB,EAAE,2BAA2B,EAClD,kBAAkB,EAAE,kBAAkB,EACtC,uBAAuB,EAAE,uBAAuB;IAG7D,eAAe,CACnB,MAAM,EAAE,cAAc,EACtB,cAAc,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,OAAO,CAAC,IAAI,CAAC,GACtD,OAAO,CAAC,IAAI,CAAC;CAmEjB"}

package/dist/stream/stream-consumer.service.js

@@ -0,0 +1,78 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var StreamConsumerService_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StreamConsumerService = exports.CONSUMER_INTERVAL_TIME = void 0;
+const common_1 = require("@nestjs/common");
+const periodic_executor_service_1 = require("./periodic-executor.service");
+const redis_stream_service_1 = require("./redis-stream.service");
+const stream_message_handler_service_1 = require("./stream-message-handler.service");
+const stream_retry_service_1 = require("./stream-retry.service");
+const generateConsumerId = () => process.hrtime.bigint().toString();
+exports.CONSUMER_INTERVAL_TIME = 1000;
+let StreamConsumerService = StreamConsumerService_1 = class StreamConsumerService {
+    constructor(redisStreamService, messageHandlerService, streamRetryService, periodicExecutorService) {
+        this.redisStreamService = redisStreamService;
+        this.messageHandlerService = messageHandlerService;
+        this.streamRetryService = streamRetryService;
+        this.periodicExecutorService = periodicExecutorService;
+        this.logger = new common_1.Logger(StreamConsumerService_1.name);
+    }
+    async consumeMessages(config, messageHandler) {
+        const { streamName, groupName, consumerName, maxRetries = 3, retryDelay = 60 * 5000, blockTime = 100, prefetchCount = 100, maxConcurrency = 25, } = {
+            ...config,
+            ...{ consumerName: `${config.consumerName}:${generateConsumerId()}` },
+        };
+        await this.streamRetryService.addDelayedQueueKey(streamName);
+        await this.redisStreamService.createConsumerGroup(streamName, groupName);
+        this.logger.debug(`Starting consumer ${consumerName} for stream ${streamName} with concurrency ${maxConcurrency}`);
+        const processBatchRequest = {
+            messages: [],
+            streamName,
+            groupName,
+            messageHandler,
+            maxRetries,
+            retryDelay,
+            maxConcurrency,
+        };
+        const processMessages = async () => {
+            this.logger.debug(`Consumer ${consumerName} is running, fetching messages from stream ${streamName}`);
+            const messages = await this.redisStreamService.readFromConsumerGroup({
+                streamName,
+                groupName,
+                consumerName,
+                count: prefetchCount,
+                blockTime,
+            });
+            if (!messages || messages.length === 0) {
+                return;
+            }
+            this.logger.debug(`Consumer ${consumerName} fetched ${messages.length} messages from stream ${streamName}`);
+            await this.messageHandlerService.processMessages({
+                ...processBatchRequest,
+                messages,
+            });
+        };
+        await this.periodicExecutorService.startExecutor({
+            intervalMs: exports.CONSUMER_INTERVAL_TIME,
+            name: `consumer-${consumerName}`,
+        }, processMessages);
+    }
+};
+exports.StreamConsumerService = StreamConsumerService;
+exports.StreamConsumerService = StreamConsumerService = StreamConsumerService_1 = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [redis_stream_service_1.RedisStreamService,
+        stream_message_handler_service_1.StreamMessageHandlerService,
+        stream_retry_service_1.StreamRetryService,
+        periodic_executor_service_1.PeriodicExecutorService])
+], StreamConsumerService);
+//# sourceMappingURL=stream-consumer.service.js.map

package/dist/stream/stream-consumer.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stream-consumer.service.js","sourceRoot":"","sources":["../../src/stream/stream-consumer.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAAoD;AACpD,2EAAsE;AACtE,iEAA2E;AAC3E,qFAG0C;AAC1C,iEAA4D;AAE5D,MAAM,kBAAkB,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC;AAavD,QAAA,sBAAsB,GAAG,IAAI,CAAC;AAGpC,IAAM,qBAAqB,6BAA3B,MAAM,qBAAqB;IAGhC,YACmB,kBAAsC,EACtC,qBAAkD,EAClD,kBAAsC,EACtC,uBAAgD;QAHhD,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,0BAAqB,GAArB,qBAAqB,CAA6B;QAClD,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,4BAAuB,GAAvB,uBAAuB,CAAyB;QANlD,WAAM,GAAG,IAAI,eAAM,CAAC,uBAAqB,CAAC,IAAI,CAAC,CAAC;IAO9D,CAAC;IAEJ,KAAK,CAAC,eAAe,CACnB,MAAsB,EACtB,cAAuD;QAEvD,MAAM,EACJ,UAAU,EACV,SAAS,EACT,YAAY,EACZ,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,EAAE,GAAG,IAAI,EACtB,SAAS,GAAG,GAAG,EACf,aAAa,GAAG,GAAG,EACnB,cAAc,GAAG,EAAE,GACpB,GAAG;YACF,GAAG,MAAM;YACT,GAAG,EAAE,YAAY,EAAE,GAAG,MAAM,CAAC,YAAY,IAAI,kBAAkB,EAAE,EAAE,EAAE;SACtE,CAAC;QAEF,MAAM,IAAI,CAAC,kBAAkB,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;QAC7D,MAAM,IAAI,CAAC,kBAAkB,CAAC,mBAAmB,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QAEzE,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,qBAAqB,YAAY,eAAe,UAAU,qBAAqB,cAAc,EAAE,CAChG,CAAC;QAEF,MAAM,mBAAmB,GAAyB;YAChD,QAAQ,EAAE,EAAE;YACZ,UAAU;YACV,SAAS;YACT,cAAc;YACd,UAAU;YACV,UAAU;YACV,cAAc;SACf,CAAC;QAEF,MAAM,eAAe,GAAG,KAAK,IAAI,EAAE;YACjC,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,YAAY,YAAY,8CAA8C,UAAU,EAAE,CACnF,CAAC;YAEF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,qBAAqB,CAAC;gBACnE,UAAU;gBACV,SAAS;gBACT,YAAY;gBACZ,KAAK,EAAE,aAAa;gBACpB,SAAS;aACV,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvC,OAAO;YACT,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,YAAY,YAAY,YAAY,QAAQ,CAAC,MAAM,yBAAyB,UAAU,EAAE,CACzF,CAAC;YAEF,MAAM,IAAI,CAAC,qBAAqB,CAAC,eAAe,CAAC;gBAC/C,GAAG,mBAAmB;gBACtB,QAAQ;aACT,CAAC,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,IAAI,CAAC,uBAAuB,CAAC,aAAa,CAC9C;YACE,UAAU,EAAE,8BAAsB;YAClC,IAAI,EAAE,YAAY,YAAY,EAAE;SACjC,EACD,eAAe,CAChB,CAAC;IACJ,CAAC;CACF,CAAA;AAhFY,sDAAqB;gCAArB,qBAAqB;IADjC,IAAA,mBAAU,GAAE;qCAK4B,yCAAkB;QACf,4DAA2B;QAC9B,yCAAkB;QACb,mDAAuB;GAPxD,qBAAqB,CAgFjC"}

package/dist/stream/stream-message-handler.service.d.ts

@@ -0,0 +1,28 @@
+import { ContextService } from '@sprout-idws/sprout-context';
+import { RedisStreamService, StreamMessage } from './redis-stream.service';
+import { StreamRetryService } from './stream-retry.service';
+export interface ProcessMessagesInput {
+    streamName: string;
+    groupName: string;
+    maxRetries: number;
+    retryDelay: number;
+    maxConcurrency: number;
+    messages: StreamMessage[];
+    messageHandler: (entry: StreamMessage) => Promise<void>;
+}
+export declare class StreamMessageHandlerService {
+    private readonly redisStreamService;
+    private readonly retryService;
+    private readonly contextService;
+    private readonly logger;
+    private processingMessages;
+    constructor(redisStreamService: RedisStreamService, retryService: StreamRetryService, contextService: ContextService);
+    processMessages(processBatchInput: ProcessMessagesInput): Promise<void>;
+    private processMessage;
+    private handleFailedMessage;
+    private sendToDlq;
+    private sendRetryMessage;
+    getProcessingMessagesCount(): number;
+    isMessageProcessing(messageId: string): boolean;
+}
+//# sourceMappingURL=stream-message-handler.service.d.ts.map

package/dist/stream/stream-message-handler.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stream-message-handler.service.d.ts","sourceRoot":"","sources":["../../src/stream/stream-message-handler.service.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAC3E,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAE5D,MAAM,WAAW,oBAAoB;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,aAAa,EAAE,CAAC;IAC1B,cAAc,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACzD;AAWD,qBACa,2BAA2B;IAKpC,OAAO,CAAC,QAAQ,CAAC,kBAAkB;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,cAAc;IANjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgD;IACvE,OAAO,CAAC,kBAAkB,CAAqB;gBAG5B,kBAAkB,EAAE,kBAAkB,EACtC,YAAY,EAAE,kBAAkB,EAChC,cAAc,EAAE,cAAc;IAG3C,eAAe,CAAC,iBAAiB,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC;YA6C/D,cAAc;YAiCd,mBAAmB;YAuBnB,SAAS;YAKT,gBAAgB;IAW9B,0BAA0B,IAAI,MAAM;IAIpC,mBAAmB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO;CAGhD"}

package/dist/stream/stream-message-handler.service.js

@@ -0,0 +1,145 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var StreamMessageHandlerService_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StreamMessageHandlerService = void 0;
+const common_1 = require("@nestjs/common");
+const sprout_context_1 = require("@sprout-idws/sprout-context");
+const redis_stream_service_1 = require("./redis-stream.service");
+const stream_retry_service_1 = require("./stream-retry.service");
+let StreamMessageHandlerService = StreamMessageHandlerService_1 = class StreamMessageHandlerService {
+    constructor(redisStreamService, retryService, contextService) {
+        this.redisStreamService = redisStreamService;
+        this.retryService = retryService;
+        this.contextService = contextService;
+        this.logger = new common_1.Logger(StreamMessageHandlerService_1.name);
+        this.processingMessages = new Set();
+    }
+    async processMessages(processBatchInput) {
+        const { messages, streamName, groupName, maxRetries, retryDelay, maxConcurrency, messageHandler, } = processBatchInput;
+        const semaphore = new Semaphore(Math.min(maxConcurrency, messages.length));
+        const processMessageInput = {
+            streamName,
+            groupName,
+            messageHandler,
+            maxRetries,
+            retryDelay,
+        };
+        const promises = messages.map(async (message) => {
+            if (this.processingMessages.has(message.id)) {
+                return;
+            }
+            this.processingMessages.add(message.id);
+            const release = await semaphore.acquire();
+            try {
+                const startTime = Date.now();
+                await this.processMessage({
+                    ...processMessageInput,
+                    message: message,
+                });
+            }
+            finally {
+                this.processingMessages.delete(message.id);
+                release();
+            }
+        });
+        await Promise.allSettled(promises);
+    }
+    async processMessage(request) {
+        try {
+            const encodedContext = request.message.context;
+            const runCallback = async () => {
+                await request.messageHandler(request.message);
+                await this.redisStreamService.ackDelete(request.streamName, request.groupName, request.message.id);
+            };
+            if (encodedContext) {
+                const context = this.contextService.getDecodedContext(encodedContext);
+                await this.contextService.run(context, runCallback);
+            }
+            else {
+                await runCallback();
+            }
+        }
+        catch (error) {
+            const typedError = error;
+            this.logger.error(`Error processing message ${request.message.id}: ${typedError.message}`, typedError.stack);
+            this.logger.error(`Message payload for ${request.message.id}: ${JSON.stringify(request.message.data)}`);
+            await this.handleFailedMessage(request, error);
+            throw error;
+        }
+    }
+    async handleFailedMessage(inFlightMessage, error) {
+        const { maxRetries, message, streamName, groupName } = inFlightMessage;
+        const retryCount = (inFlightMessage.message.metadata?.retryCount || 0) + 1;
+        if (retryCount < maxRetries) {
+            this.logger.warn(`Message ${message.id} in stream ${streamName} and group ${groupName} failed with error: ${error.message}, retrying...`);
+            await this.sendRetryMessage({
+                ...inFlightMessage,
+                message: { ...message, metadata: { ...message.metadata, retryCount } },
+            });
+        }
+        else {
+            this.logger.error(`Message ${message.id} in stream ${streamName} and group ${groupName} exceeded max retries (${maxRetries}), moving to DLQ. Final error:`, error.stack || error.message);
+            await this.sendToDlq(inFlightMessage);
+        }
+    }
+    async sendToDlq(request) {
+        const { message, streamName, groupName } = request;
+        await this.redisStreamService.sendToDlq(streamName, groupName, message.id);
+    }
+    async sendRetryMessage(inFlightMessage) {
+        const { message, maxRetries } = inFlightMessage;
+        const retryCount = message.metadata?.retryCount || 0;
+        this.logger.warn(`Scheduling retry for message ${message.id} (attempt ${retryCount}/${maxRetries})`);
+        await this.retryService.delayedRetry(inFlightMessage);
+    }
+    getProcessingMessagesCount() {
+        return this.processingMessages.size;
+    }
+    isMessageProcessing(messageId) {
+        return this.processingMessages.has(messageId);
+    }
+};
+exports.StreamMessageHandlerService = StreamMessageHandlerService;
+exports.StreamMessageHandlerService = StreamMessageHandlerService = StreamMessageHandlerService_1 = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [redis_stream_service_1.RedisStreamService,
+        stream_retry_service_1.StreamRetryService,
+        sprout_context_1.ContextService])
+], StreamMessageHandlerService);
+class Semaphore {
+    constructor(permits) {
+        this.waitQueue = [];
+        this.permits = permits;
+    }
+    async acquire() {
+        if (this.permits > 0) {
+            this.permits--;
+            return () => this.release();
+        }
+        return new Promise((resolve) => {
+            this.waitQueue.push(() => {
+                this.permits--;
+                resolve(() => this.release());
+            });
+        });
+    }
+    release() {
+        this.permits++;
+        if (this.waitQueue.length > 0) {
+            const next = this.waitQueue.shift();
+            if (next) {
+                next();
+            }
+        }
+    }
+}
+//# sourceMappingURL=stream-message-handler.service.js.map

package/dist/stream/stream-message-handler.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stream-message-handler.service.js","sourceRoot":"","sources":["../../src/stream/stream-message-handler.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAAoD;AACpD,gEAA6D;AAC7D,iEAA2E;AAC3E,iEAA4D;AAsBrD,IAAM,2BAA2B,mCAAjC,MAAM,2BAA2B;IAItC,YACmB,kBAAsC,EACtC,YAAgC,EAChC,cAA8B;QAF9B,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,iBAAY,GAAZ,YAAY,CAAoB;QAChC,mBAAc,GAAd,cAAc,CAAgB;QANhC,WAAM,GAAG,IAAI,eAAM,CAAC,6BAA2B,CAAC,IAAI,CAAC,CAAC;QAC/D,uBAAkB,GAAG,IAAI,GAAG,EAAU,CAAC;IAM5C,CAAC;IAEJ,KAAK,CAAC,eAAe,CAAC,iBAAuC;QAC3D,MAAM,EACJ,QAAQ,EACR,UAAU,EACV,SAAS,EACT,UAAU,EACV,UAAU,EACV,cAAc,EACd,cAAc,GACf,GAAG,iBAAiB,CAAC;QAEtB,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QAE3E,MAAM,mBAAmB,GAA6B;YACpD,UAAU;YACV,SAAS;YACT,cAAc;YACd,UAAU;YACV,UAAU;SACX,CAAC;QAEF,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;YAC9C,IAAI,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAG,CAAC,EAAE,CAAC;gBAC7C,OAAO;YACT,CAAC;YAED,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAG,CAAC,CAAC;YACzC,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;YAE1C,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAE7B,MAAM,IAAI,CAAC,cAAc,CAAC;oBACxB,GAAG,mBAAmB;oBACtB,OAAO,EAAE,OAAO;iBACE,CAAC,CAAC;YACxB,CAAC;oBAAS,CAAC;gBACT,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAG,CAAC,CAAC;gBAC5C,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACrC,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,OAAwB;QACnD,IAAI,CAAC;YACH,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC;YAE/C,MAAM,WAAW,GAAG,KAAK,IAAI,EAAE;gBAC7B,MAAM,OAAO,CAAC,cAAc,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC9C,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CACrC,OAAO,CAAC,UAAU,EAClB,OAAO,CAAC,SAAS,EACjB,OAAO,CAAC,OAAO,CAAC,EAAG,CACpB,CAAC;YACJ,CAAC,CAAC;YAEF,IAAI,cAAc,EAAE,CAAC;gBACnB,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAAC,cAAc,CAAC,CAAC;gBACtE,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;YACtD,CAAC;iBAAM,CAAC;gBACN,MAAM,WAAW,EAAE,CAAC;YACtB,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,UAAU,GAAG,KAAc,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,4BAA4B,OAAO,CAAC,OAAO,CAAC,EAAE,KAAK,UAAU,CAAC,OAAO,EAAE,EACvE,UAAU,CAAC,KAAK,CACjB,CAAC;YACF,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,uBAAuB,OAAO,CAAC,OAAO,CAAC,EAAE,KAAK,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CACrF,CAAC;YACF,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,KAAc,CAAC,CAAC;YACxD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,eAAgC,EAAE,KAAY;QAC9E,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,eAAe,CAAC;QACvE,MAAM,UAAU,GAAG,CAAC,eAAe,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAE3E,IAAI,UAAU,GAAG,UAAU,EAAE,CAAC;YAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,WAAW,OAAO,CAAC,EAAE,cAAc,UAAU,cAAc,SAAS,uBAAuB,KAAK,CAAC,OAAO,eAAe,CACxH,CAAC;YAEF,MAAM,IAAI,CAAC,gBAAgB,CAAC;gBAC1B,GAAG,eAAe;gBAClB,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,QAAQ,EAAE,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,EAAE;aACvE,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,WAAW,OAAO,CAAC,EAAE,cAAc,UAAU,cAAc,SAAS,0BAA0B,UAAU,gCAAgC,EACxI,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,CAC7B,CAAC;YAEF,MAAM,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,OAAwB;QAC9C,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QACnD,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,UAAU,EAAE,SAAS,EAAE,OAAO,CAAC,EAAG,CAAC,CAAC;IAC9E,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,eAAgC;QAC7D,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,eAAe,CAAC;QAChD,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,EAAE,UAAU,IAAI,CAAC,CAAC;QAErD,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,gCAAgC,OAAO,CAAC,EAAE,aAAa,UAAU,IAAI,UAAU,GAAG,CACnF,CAAC;QAEF,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC;IACxD,CAAC;IAED,0BAA0B;QACxB,OAAO,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC;IACtC,CAAC;IAED,mBAAmB,CAAC,SAAiB;QACnC,OAAO,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAChD,CAAC;CACF,CAAA;AAtIY,kEAA2B;sCAA3B,2BAA2B;IADvC,IAAA,mBAAU,GAAE;qCAM4B,yCAAkB;QACxB,yCAAkB;QAChB,+BAAc;GAPtC,2BAA2B,CAsIvC;AAED,MAAM,SAAS;IAIb,YAAY,OAAe;QAFnB,cAAS,GAAsB,EAAE,CAAC;QAGxC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACrB,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QAC9B,CAAC;QAED,OAAO,IAAI,OAAO,CAAa,CAAC,OAAO,EAAE,EAAE;YACzC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE;gBACvB,IAAI,CAAC,OAAO,EAAE,CAAC;gBACf,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;YAChC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,OAAO;QACb,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;YACpC,IAAI,IAAI,EAAE,CAAC;gBACT,IAAI,EAAE,CAAC;YACT,CAAC;QACH,CAAC;IACH,CAAC;CACF"}

package/dist/stream/stream-publisher.service.d.ts

@@ -0,0 +1,9 @@
+import { RedisStreamService } from './redis-stream.service';
+import { ContextService } from '@sprout-idws/sprout-context';
+export declare class StreamPublisherService {
+    private readonly redisStreamService;
+    private readonly contextService;
+    constructor(redisStreamService: RedisStreamService, contextService: ContextService);
+    publishToStream(streamName: string, data: object): Promise<string>;
+}
+//# sourceMappingURL=stream-publisher.service.d.ts.map