@loipv/nestjs-kafka 0.0.1

package/dist/services/consumer-registry.service.js

@@ -0,0 +1,195 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var ConsumerRegistryService_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConsumerRegistryService = void 0;
+const common_1 = require("@nestjs/common");
+const kafka_core_service_1 = require("./kafka-core.service");
+const batch_processor_service_1 = require("./batch-processor.service");
+const idempotency_service_1 = require("./idempotency.service");
+const pressure_manager_service_1 = require("./pressure-manager.service");
+const dlq_service_1 = require("./dlq.service");
+let ConsumerRegistryService = ConsumerRegistryService_1 = class ConsumerRegistryService {
+    kafkaCore;
+    batchProcessor;
+    idempotencyService;
+    pressureManager;
+    dlqService;
+    logger = new common_1.Logger(ConsumerRegistryService_1.name);
+    registeredConsumers = new Map();
+    isShuttingDown = false;
+    constructor(kafkaCore, batchProcessor, idempotencyService, pressureManager, dlqService) {
+        this.kafkaCore = kafkaCore;
+        this.batchProcessor = batchProcessor;
+        this.idempotencyService = idempotencyService;
+        this.pressureManager = pressureManager;
+        this.dlqService = dlqService;
+    }
+    registerConsumers(consumers) {
+        for (const metadata of consumers) {
+            this.registerConsumer(metadata);
+        }
+    }
+    registerConsumer(metadata) {
+        const { topic, options } = metadata;
+        const consumerId = `${topic}-${options.groupId || 'default'}`;
+        const consumer = this.kafkaCore.getKafka().consumer({
+            groupId: options.groupId || `${topic}-group`,
+            sessionTimeout: options.sessionTimeout,
+            heartbeatInterval: options.heartbeatInterval,
+            rebalanceTimeout: options.rebalanceTimeout,
+            maxBytesPerPartition: 1048576,
+        });
+        this.pressureManager.register(consumerId, consumer, {
+            backPressureThreshold: options.backPressureThreshold || 80,
+            resumeThreshold: 60,
+            maxQueueSize: options.maxQueueSize || 1000,
+            checkIntervalMs: 1000,
+        });
+        this.registeredConsumers.set(consumerId, {
+            metadata,
+            consumer,
+            isRunning: false,
+        });
+        this.logger.log(`Registered consumer: ${consumerId}`);
+    }
+    async startAll() {
+        const startPromises = Array.from(this.registeredConsumers.values()).map((registered) => this.startConsumer(registered));
+        await Promise.all(startPromises);
+    }
+    async startConsumer(registered) {
+        const { metadata, consumer } = registered;
+        const { topic, options } = metadata;
+        try {
+            await consumer.connect();
+            await consumer.subscribe({
+                topic,
+                fromBeginning: options.fromBeginning,
+            });
+            if (options.batch) {
+                await this.startBatchConsumer(registered);
+            }
+            else {
+                await this.startMessageConsumer(registered);
+            }
+            registered.isRunning = true;
+            this.logger.log(`Started consumer for topic: ${topic}`);
+        }
+        catch (error) {
+            this.logger.error(`Failed to start consumer for topic: ${topic}`, error);
+            throw error;
+        }
+    }
+    async startMessageConsumer(registered) {
+        const { metadata, consumer } = registered;
+        const { topic, options, target, methodName } = metadata;
+        const handler = target[methodName].bind(target);
+        await consumer.run({
+            autoCommit: options.autoCommit !== false,
+            autoCommitInterval: options.autoCommitInterval,
+            autoCommitThreshold: options.autoCommitThreshold,
+            eachMessage: async (payload) => {
+                if (this.isShuttingDown)
+                    return;
+                const { message, partition } = payload;
+                if (options.idempotencyKey) {
+                    if (this.idempotencyService.isProcessed(message, options.idempotencyKey)) {
+                        this.logger.debug('Skipping duplicate message');
+                        return;
+                    }
+                }
+                try {
+                    await handler(message);
+                    if (options.idempotencyKey) {
+                        this.idempotencyService.markProcessed(message, options.idempotencyKey);
+                    }
+                    this.dlqService.clearRetryState(message, topic, partition);
+                }
+                catch (error) {
+                    await this.handleError(message, error, metadata, partition);
+                }
+            },
+        });
+    }
+    async startBatchConsumer(registered) {
+        const { metadata, consumer } = registered;
+        const { options, target, methodName } = metadata;
+        const handler = target[methodName].bind(target);
+        const eachBatchHandler = this.batchProcessor.createEachBatchHandler(options, async (messages) => {
+            if (this.isShuttingDown)
+                return;
+            let processableMessages = messages;
+            if (options.idempotencyKey) {
+                processableMessages = this.idempotencyService.filterDuplicates(messages, options.idempotencyKey);
+            }
+            if (options.groupByKey) {
+                const grouped = this.batchProcessor.groupMessagesByKey(processableMessages);
+                await handler(grouped);
+            }
+            else {
+                await handler(processableMessages);
+            }
+            if (options.idempotencyKey) {
+                for (const msg of processableMessages) {
+                    this.idempotencyService.markProcessed(msg, options.idempotencyKey);
+                }
+            }
+        });
+        await consumer.run({
+            autoCommit: false,
+            eachBatch: eachBatchHandler,
+        });
+    }
+    async handleError(message, error, metadata, partition) {
+        const { topic, options } = metadata;
+        this.logger.error(`Error processing message from ${topic}`, error);
+        if (options.dlq) {
+            const shouldRetry = await this.dlqService.handleFailure(message, error, options.dlq, topic, partition);
+            if (shouldRetry) {
+                throw error;
+            }
+        }
+        else {
+            throw error;
+        }
+    }
+    async gracefulShutdown() {
+        this.isShuttingDown = true;
+        this.logger.log('Starting graceful shutdown of consumers...');
+        const shutdownPromises = Array.from(this.registeredConsumers.values()).map(async (registered) => {
+            try {
+                if (registered.isRunning) {
+                    await registered.consumer.stop();
+                    await registered.consumer.disconnect();
+                }
+            }
+            catch (error) {
+                this.logger.error('Error during consumer shutdown', error);
+            }
+        });
+        await Promise.all(shutdownPromises);
+        this.idempotencyService.stopCleanup();
+        this.logger.log('All consumers shut down gracefully');
+    }
+    async onApplicationShutdown() {
+        await this.gracefulShutdown();
+    }
+};
+exports.ConsumerRegistryService = ConsumerRegistryService;
+exports.ConsumerRegistryService = ConsumerRegistryService = ConsumerRegistryService_1 = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [kafka_core_service_1.KafkaCoreService,
+        batch_processor_service_1.BatchProcessorService,
+        idempotency_service_1.IdempotencyService,
+        pressure_manager_service_1.PressureManagerService,
+        dlq_service_1.DlqService])
+], ConsumerRegistryService);
+//# sourceMappingURL=consumer-registry.service.js.map

package/dist/services/consumer-registry.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"consumer-registry.service.js","sourceRoot":"","sources":["../../lib/services/consumer-registry.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAA2E;AAE3E,6DAAwD;AACxD,uEAAkE;AAClE,+DAA2D;AAC3D,yEAAoE;AACpE,+CAA2C;AAUpC,IAAM,uBAAuB,+BAA7B,MAAM,uBAAuB;IAMf;IACA;IACA;IACA;IACA;IATF,MAAM,GAAG,IAAI,eAAM,CAAC,yBAAuB,CAAC,IAAI,CAAC,CAAC;IAC3D,mBAAmB,GAAG,IAAI,GAAG,EAA8B,CAAC;IAC5D,cAAc,GAAG,KAAK,CAAC;IAE/B,YACmB,SAA2B,EAC3B,cAAqC,EACrC,kBAAsC,EACtC,eAAuC,EACvC,UAAsB;QAJtB,cAAS,GAAT,SAAS,CAAkB;QAC3B,mBAAc,GAAd,cAAc,CAAuB;QACrC,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,oBAAe,GAAf,eAAe,CAAwB;QACvC,eAAU,GAAV,UAAU,CAAY;IACtC,CAAC;IAEJ,iBAAiB,CAAC,SAA6B;QAC7C,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAEO,gBAAgB,CAAC,QAA0B;QACjD,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC;QACpC,MAAM,UAAU,GAAG,GAAG,KAAK,IAAI,OAAO,CAAC,OAAO,IAAI,SAAS,EAAE,CAAC;QAE9D,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC;YAClD,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,GAAG,KAAK,QAAQ;YAC5C,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;YAC5C,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;YAC1C,oBAAoB,EAAE,OAAO;SAC9B,CAAC,CAAC;QAEH,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,UAAU,EAAE,QAAQ,EAAE;YAClD,qBAAqB,EAAE,OAAO,CAAC,qBAAqB,IAAI,EAAE;YAC1D,eAAe,EAAE,EAAE;YACnB,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI;YAC1C,eAAe,EAAE,IAAI;SACtB,CAAC,CAAC;QAEH,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,EAAE;YACvC,QAAQ;YACR,QAAQ;YACR,SAAS,EAAE,KAAK;SACjB,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,wBAAwB,UAAU,EAAE,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CACrE,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAC/C,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IACnC,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,UAA8B;QACxD,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,UAAU,CAAC;QAC1C,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC;QAEpC,IAAI,CAAC;YACH,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;YAEzB,MAAM,QAAQ,CAAC,SAAS,CAAC;gBACvB,KAAK;gBACL,aAAa,EAAE,OAAO,CAAC,aAAa;aACrC,CAAC,CAAC;YAEH,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBAClB,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;YAC5C,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,oBAAoB,CAAC,UAAU,CAAC,CAAC;YAC9C,CAAC;YAED,UAAU,CAAC,SAAS,GAAG,IAAI,CAAC;YAC5B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,+BAA+B,KAAK,EAAE,CAAC,CAAC;QAC1D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,uCAAuC,KAAK,EAAE,EAAE,KAAK,CAAC,CAAC;YACzE,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAChC,UAA8B;QAE9B,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,UAAU,CAAC;QAE1C,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,QAAQ,CAAC;QAExD,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhD,MAAM,QAAQ,CAAC,GAAG,CAAC;YACjB,UAAU,EAAE,OAAO,CAAC,UAAU,KAAK,KAAK;YACxC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;YAC9C,mBAAmB,EAAE,OAAO,CAAC,mBAAmB;YAChD,WAAW,EAAE,KAAK,EAAE,OAA2B,EAAE,EAAE;gBACjD,IAAI,IAAI,CAAC,cAAc;oBAAE,OAAO;gBAEhC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;gBAEvC,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;oBAC3B,IACE,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,cAAc,CAAC,EACpE,CAAC;wBACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;wBAChD,OAAO;oBACT,CAAC;gBACH,CAAC;gBAED,IAAI,CAAC;oBAEH,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;oBAEvB,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;wBAC3B,IAAI,CAAC,kBAAkB,CAAC,aAAa,CACnC,OAAO,EACP,OAAO,CAAC,cAAc,CACvB,CAAC;oBACJ,CAAC;oBAED,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;gBAC7D,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,KAAc,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;gBACvE,CAAC;YACH,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,UAA8B;QAE9B,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,UAAU,CAAC;QAE1C,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,QAAQ,CAAC;QAEjD,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhD,MAAM,gBAAgB,GAAG,IAAI,CAAC,cAAc,CAAC,sBAAsB,CACjE,OAAO,EACP,KAAK,EAAE,QAAQ,EAAE,EAAE;YACjB,IAAI,IAAI,CAAC,cAAc;gBAAE,OAAO;YAEhC,IAAI,mBAAmB,GAAG,QAAQ,CAAC;YACnC,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;gBAC3B,mBAAmB,GAAG,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAC5D,QAAQ,EACR,OAAO,CAAC,cAAc,CACvB,CAAC;YACJ,CAAC;YAED,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACvB,MAAM,OAAO,GACX,IAAI,CAAC,cAAc,CAAC,kBAAkB,CAAC,mBAAmB,CAAC,CAAC;gBAE9D,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;YACzB,CAAC;iBAAM,CAAC;gBAEN,MAAM,OAAO,CAAC,mBAAmB,CAAC,CAAC;YACrC,CAAC;YAED,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;gBAC3B,KAAK,MAAM,GAAG,IAAI,mBAAmB,EAAE,CAAC;oBACtC,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC;gBACrE,CAAC;YACH,CAAC;QACH,CAAC,CACF,CAAC;QAEF,MAAM,QAAQ,CAAC,GAAG,CAAC;YACjB,UAAU,EAAE,KAAK;YACjB,SAAS,EAAE,gBAAgB;SAC5B,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,OAAY,EACZ,KAAY,EACZ,QAA0B,EAC1B,SAAkB;QAElB,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC;QAEpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iCAAiC,KAAK,EAAE,EAAE,KAAK,CAAC,CAAC;QAEnE,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;YAChB,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CACrD,OAAO,EACP,KAAK,EACL,OAAO,CAAC,GAAG,EACX,KAAK,EACL,SAAS,CACV,CAAC;YAEF,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,gBAAgB;QACpB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;QAE9D,MAAM,gBAAgB,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CACxE,KAAK,EAAE,UAAU,EAAE,EAAE;YACnB,IAAI,CAAC;gBACH,IAAI,UAAU,CAAC,SAAS,EAAE,CAAC;oBACzB,MAAM,UAAU,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;oBACjC,MAAM,UAAU,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;gBACzC,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC,CACF,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QACpC,IAAI,CAAC,kBAAkB,CAAC,WAAW,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAChC,CAAC;CACF,CAAA;AAjOY,0DAAuB;kCAAvB,uBAAuB;IADnC,IAAA,mBAAU,GAAE;qCAOmB,qCAAgB;QACX,+CAAqB;QACjB,wCAAkB;QACrB,iDAAsB;QAC3B,wBAAU;GAV9B,uBAAuB,CAiOnC"}

package/dist/services/dlq.service.d.ts

@@ -0,0 +1,14 @@
+import { KafkaMessage } from 'kafkajs';
+import { KafkaClient } from './kafka-client.service';
+import { DlqOptions } from '../interfaces';
+export declare class DlqService {
+    private readonly kafkaClient;
+    private readonly logger;
+    private retryStates;
+    constructor(kafkaClient: KafkaClient);
+    handleFailure(message: KafkaMessage, error: Error, options: DlqOptions, originalTopic: string, partition?: number): Promise<boolean>;
+    private sendToDlq;
+    private getMessageKey;
+    private sleep;
+    clearRetryState(message: KafkaMessage, topic: string, partition?: number): void;
+}

package/dist/services/dlq.service.js

@@ -0,0 +1,90 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+var DlqService_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DlqService = void 0;
+const common_1 = require("@nestjs/common");
+const kafka_client_service_1 = require("./kafka-client.service");
+let DlqService = DlqService_1 = class DlqService {
+    kafkaClient;
+    logger = new common_1.Logger(DlqService_1.name);
+    retryStates = new Map();
+    constructor(kafkaClient) {
+        this.kafkaClient = kafkaClient;
+    }
+    async handleFailure(message, error, options, originalTopic, partition) {
+        const messageKey = this.getMessageKey(message, originalTopic, partition);
+        let state = this.retryStates.get(messageKey);
+        if (!state) {
+            state = { retryCount: 0 };
+            this.retryStates.set(messageKey, state);
+        }
+        state.retryCount++;
+        state.lastError = error;
+        const maxRetries = options.maxRetries ?? 3;
+        if (state.retryCount <= maxRetries) {
+            const baseDelay = options.retryDelay ?? 1000;
+            const multiplier = options.retryBackoffMultiplier ?? 2;
+            const delay = baseDelay * Math.pow(multiplier, state.retryCount - 1);
+            this.logger.warn(`Retry ${state.retryCount}/${maxRetries} for message, waiting ${delay}ms`);
+            await this.sleep(delay);
+            return true;
+        }
+        await this.sendToDlq(message, error, options, originalTopic, state.retryCount);
+        this.retryStates.delete(messageKey);
+        return false;
+    }
+    async sendToDlq(message, error, options, originalTopic, retryCount) {
+        const headers = {};
+        if (options.includeOriginalHeaders !== false && message.headers) {
+            Object.assign(headers, message.headers);
+        }
+        headers['x-dlq-original-topic'] = originalTopic;
+        headers['x-dlq-retry-count'] = String(retryCount);
+        headers['x-dlq-timestamp'] = new Date().toISOString();
+        if (options.includeErrorInfo !== false) {
+            headers['x-dlq-error-message'] = error.message;
+            headers['x-dlq-error-stack'] = error.stack || '';
+        }
+        try {
+            await this.kafkaClient.send(options.topic, {
+                key: message.key,
+                value: message.value,
+                headers,
+            });
+            this.logger.warn(`Message sent to DLQ: ${options.topic} after ${retryCount} retries`);
+        }
+        catch (dlqError) {
+            this.logger.error('Failed to send message to DLQ', dlqError);
+            throw dlqError;
+        }
+    }
+    getMessageKey(message, topic, partition) {
+        return `${topic}:${partition ?? 0}:${message.offset}`;
+    }
+    sleep(ms) {
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    }
+    clearRetryState(message, topic, partition) {
+        const key = this.getMessageKey(message, topic, partition);
+        this.retryStates.delete(key);
+    }
+};
+exports.DlqService = DlqService;
+exports.DlqService = DlqService = DlqService_1 = __decorate([
+    (0, common_1.Injectable)(),
+    __param(0, (0, common_1.Inject)((0, common_1.forwardRef)(() => kafka_client_service_1.KafkaClient))),
+    __metadata("design:paramtypes", [kafka_client_service_1.KafkaClient])
+], DlqService);
+//# sourceMappingURL=dlq.service.js.map

package/dist/services/dlq.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dlq.service.js","sourceRoot":"","sources":["../../lib/services/dlq.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,2CAAwE;AAExE,iEAAqD;AAS9C,IAAM,UAAU,kBAAhB,MAAM,UAAU;IAMF;IALF,MAAM,GAAG,IAAI,eAAM,CAAC,YAAU,CAAC,IAAI,CAAC,CAAC;IAC9C,WAAW,GAAG,IAAI,GAAG,EAAsB,CAAC;IAEpD,YAEmB,WAAwB;QAAxB,gBAAW,GAAX,WAAW,CAAa;IACxC,CAAC;IAEJ,KAAK,CAAC,aAAa,CACjB,OAAqB,EACrB,KAAY,EACZ,OAAmB,EACnB,aAAqB,EACrB,SAAkB;QAElB,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,aAAa,EAAE,SAAS,CAAC,CAAC;QACzE,IAAI,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAE7C,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;YAC1B,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAC1C,CAAC;QAED,KAAK,CAAC,UAAU,EAAE,CAAC;QACnB,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC;QAExB,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC;QAE3C,IAAI,KAAK,CAAC,UAAU,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC;YAC7C,MAAM,UAAU,GAAG,OAAO,CAAC,sBAAsB,IAAI,CAAC,CAAC;YACvD,MAAM,KAAK,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;YAErE,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,SAAS,KAAK,CAAC,UAAU,IAAI,UAAU,yBAAyB,KAAK,IAAI,CAC1E,CAAC;YAEF,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACxB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,IAAI,CAAC,SAAS,CAClB,OAAO,EACP,KAAK,EACL,OAAO,EACP,aAAa,EACb,KAAK,CAAC,UAAU,CACjB,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAEpC,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,SAAS,CACrB,OAAqB,EACrB,KAAY,EACZ,OAAmB,EACnB,aAAqB,EACrB,UAAkB;QAElB,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,IAAI,OAAO,CAAC,sBAAsB,KAAK,KAAK,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YAChE,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QAC1C,CAAC;QAED,OAAO,CAAC,sBAAsB,CAAC,GAAG,aAAa,CAAC;QAChD,OAAO,CAAC,mBAAmB,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;QAClD,OAAO,CAAC,iBAAiB,CAAC,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAEtD,IAAI,OAAO,CAAC,gBAAgB,KAAK,KAAK,EAAE,CAAC;YACvC,OAAO,CAAC,qBAAqB,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC;YAC/C,OAAO,CAAC,mBAAmB,CAAC,GAAG,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;QACnD,CAAC;QAED,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE;gBACzC,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,OAAO;aACR,CAAC,CAAC;YAEH,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,wBAAwB,OAAO,CAAC,KAAK,UAAU,UAAU,UAAU,CACpE,CAAC;QACJ,CAAC;QAAC,OAAO,QAAQ,EAAE,CAAC;YAClB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,QAAQ,CAAC,CAAC;YAC7D,MAAM,QAAQ,CAAC;QACjB,CAAC;IACH,CAAC;IAEO,aAAa,CACnB,OAAqB,EACrB,KAAa,EACb,SAAkB;QAElB,OAAO,GAAG,KAAK,IAAI,SAAS,IAAI,CAAC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IACxD,CAAC;IAEO,KAAK,CAAC,EAAU;QACtB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,eAAe,CACb,OAAqB,EACrB,KAAa,EACb,SAAkB;QAElB,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QAC1D,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC/B,CAAC;CACF,CAAA;AAhHY,gCAAU;qBAAV,UAAU;IADtB,IAAA,mBAAU,GAAE;IAMR,WAAA,IAAA,eAAM,EAAC,IAAA,mBAAU,EAAC,GAAG,EAAE,CAAC,kCAAW,CAAC,CAAC,CAAA;qCACR,kCAAW;GANhC,UAAU,CAgHtB"}

package/dist/services/idempotency.service.d.ts

@@ -0,0 +1,16 @@
+import { OnModuleDestroy } from '@nestjs/common';
+import { KafkaMessage } from 'kafkajs';
+export declare class IdempotencyService implements OnModuleDestroy {
+    private readonly logger;
+    private processedKeys;
+    private cleanupInterval;
+    private readonly defaultTtl;
+    constructor();
+    onModuleDestroy(): void;
+    isProcessed(message: KafkaMessage, keyExtractor?: (msg: KafkaMessage) => string | undefined): boolean;
+    markProcessed(message: KafkaMessage, keyExtractor?: (msg: KafkaMessage) => string | undefined): void;
+    filterDuplicates(messages: KafkaMessage[], keyExtractor?: (msg: KafkaMessage) => string | undefined): KafkaMessage[];
+    private extractKey;
+    private startCleanup;
+    stopCleanup(): void;
+}

package/dist/services/idempotency.service.js

@@ -0,0 +1,76 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var IdempotencyService_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IdempotencyService = void 0;
+const common_1 = require("@nestjs/common");
+let IdempotencyService = IdempotencyService_1 = class IdempotencyService {
+    logger = new common_1.Logger(IdempotencyService_1.name);
+    processedKeys = new Map();
+    cleanupInterval = null;
+    defaultTtl = 3600000;
+    constructor() {
+        this.startCleanup();
+    }
+    onModuleDestroy() {
+        this.stopCleanup();
+    }
+    isProcessed(message, keyExtractor) {
+        const key = this.extractKey(message, keyExtractor);
+        if (!key)
+            return false;
+        return this.processedKeys.has(key);
+    }
+    markProcessed(message, keyExtractor) {
+        const key = this.extractKey(message, keyExtractor);
+        if (!key)
+            return;
+        this.processedKeys.set(key, {
+            key,
+            timestamp: Date.now(),
+        });
+    }
+    filterDuplicates(messages, keyExtractor) {
+        return messages.filter((msg) => !this.isProcessed(msg, keyExtractor));
+    }
+    extractKey(message, keyExtractor) {
+        if (keyExtractor) {
+            return keyExtractor(message);
+        }
+        const header = message.headers?.['idempotency-key'];
+        if (header) {
+            return Buffer.isBuffer(header) ? header.toString() : String(header);
+        }
+        return undefined;
+    }
+    startCleanup() {
+        this.cleanupInterval = setInterval(() => {
+            const now = Date.now();
+            for (const [key, entry] of this.processedKeys.entries()) {
+                if (now - entry.timestamp > this.defaultTtl) {
+                    this.processedKeys.delete(key);
+                }
+            }
+        }, 60000);
+    }
+    stopCleanup() {
+        if (this.cleanupInterval) {
+            clearInterval(this.cleanupInterval);
+            this.cleanupInterval = null;
+        }
+    }
+};
+exports.IdempotencyService = IdempotencyService;
+exports.IdempotencyService = IdempotencyService = IdempotencyService_1 = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [])
+], IdempotencyService);
+//# sourceMappingURL=idempotency.service.js.map

package/dist/services/idempotency.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"idempotency.service.js","sourceRoot":"","sources":["../../lib/services/idempotency.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAAqE;AAS9D,IAAM,kBAAkB,0BAAxB,MAAM,kBAAkB;IACZ,MAAM,GAAG,IAAI,eAAM,CAAC,oBAAkB,CAAC,IAAI,CAAC,CAAC;IAEtD,aAAa,GAAG,IAAI,GAAG,EAA4B,CAAC;IACpD,eAAe,GAA0B,IAAI,CAAC;IACrC,UAAU,GAAG,OAAO,CAAC;IAEtC;QACE,IAAI,CAAC,YAAY,EAAE,CAAC;IACtB,CAAC;IAED,eAAe;QACb,IAAI,CAAC,WAAW,EAAE,CAAC;IACrB,CAAC;IAED,WAAW,CACT,OAAqB,EACrB,YAAwD;QAExD,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACnD,IAAI,CAAC,GAAG;YAAE,OAAO,KAAK,CAAC;QAEvB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACrC,CAAC;IAED,aAAa,CACX,OAAqB,EACrB,YAAwD;QAExD,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACnD,IAAI,CAAC,GAAG;YAAE,OAAO;QAEjB,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE;YAC1B,GAAG;YACH,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC,CAAC;IACL,CAAC;IAED,gBAAgB,CACd,QAAwB,EACxB,YAAwD;QAExD,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC,CAAC;IACxE,CAAC;IAEO,UAAU,CAChB,OAAqB,EACrB,YAAwD;QAExD,IAAI,YAAY,EAAE,CAAC;YACjB,OAAO,YAAY,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,iBAAiB,CAAC,CAAC;QACpD,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACtE,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,YAAY;QAClB,IAAI,CAAC,eAAe,GAAG,WAAW,CAAC,GAAG,EAAE;YACtC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEvB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;gBACxD,IAAI,GAAG,GAAG,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;oBAC5C,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACjC,CAAC;YACH,CAAC;QACH,CAAC,EAAE,KAAK,CAAC,CAAC;IACZ,CAAC;IAED,WAAW;QACT,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACpC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QAC9B,CAAC;IACH,CAAC;CACF,CAAA;AA/EY,gDAAkB;6BAAlB,kBAAkB;IAD9B,IAAA,mBAAU,GAAE;;GACA,kBAAkB,CA+E9B"}

package/dist/services/index.d.ts

@@ -0,0 +1,7 @@
+export * from './kafka-core.service';
+export * from './kafka-client.service';
+export * from './consumer-registry.service';
+export * from './batch-processor.service';
+export * from './idempotency.service';
+export * from './pressure-manager.service';
+export * from './dlq.service';

package/dist/services/index.js

@@ -0,0 +1,24 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./kafka-core.service"), exports);
+__exportStar(require("./kafka-client.service"), exports);
+__exportStar(require("./consumer-registry.service"), exports);
+__exportStar(require("./batch-processor.service"), exports);
+__exportStar(require("./idempotency.service"), exports);
+__exportStar(require("./pressure-manager.service"), exports);
+__exportStar(require("./dlq.service"), exports);
+//# sourceMappingURL=index.js.map

package/dist/services/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../lib/services/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,uDAAqC;AACrC,yDAAuC;AACvC,8DAA4C;AAC5C,4DAA0C;AAC1C,wDAAsC;AACtC,6DAA2C;AAC3C,gDAA8B"}

package/dist/services/kafka-client.service.d.ts

@@ -0,0 +1,31 @@
+import { OnModuleInit, OnApplicationShutdown } from '@nestjs/common';
+import { KafkaModuleOptions, ProducerMessage, SendOptions } from '../interfaces';
+import { KafkaCoreService } from './kafka-core.service';
+export declare class KafkaClient implements OnModuleInit, OnApplicationShutdown {
+    private readonly options;
+    private readonly kafkaCore;
+    private readonly logger;
+    private producer;
+    private isConnected;
+    private batchBuffer;
+    private batchTimer;
+    private readonly defaultBatchSize;
+    private readonly defaultBatchTimeout;
+    constructor(options: KafkaModuleOptions, kafkaCore: KafkaCoreService);
+    onModuleInit(): Promise<void>;
+    onApplicationShutdown(): Promise<void>;
+    connect(): Promise<void>;
+    disconnect(): Promise<void>;
+    send(topic: string, message: ProducerMessage, options?: SendOptions): Promise<void>;
+    sendBatch(topic: string, messages: ProducerMessage[], options?: SendOptions): Promise<void>;
+    sendMultiTopicBatch(topicMessages: Array<{
+        topic: string;
+        messages: ProducerMessage[];
+    }>, options?: SendOptions): Promise<void>;
+    sendQueued(topic: string, message: ProducerMessage): Promise<void>;
+    private serializeMessage;
+    private scheduleBatchFlush;
+    private flushBatch;
+    private flushAllBatches;
+    isHealthy(): boolean;
+}