@message-queue-toolkit/kafka 0.9.1-beta.1 → 0.9.1-beta.11

package/dist/AbstractKafkaConsumer.d.ts

@@ -3,7 +3,7 @@ import { type ConsumeOptions, type ConsumerOptions } from '@platformatic/kafka';
 import { AbstractKafkaService, type BaseKafkaOptions } from './AbstractKafkaService.ts';
 import type { KafkaHandlerRouting } from './handler-routing/KafkaHandlerRoutingBuilder.ts';
 import type { KafkaConfig, KafkaDependencies, TopicConfig } from './types.ts';
-import type { KafkaMessageBatchOptions } from './utils/KafkaMessageBatchStream.ts';
+import { type KafkaMessageBatchOptions } from './utils/KafkaMessageBatchStream.ts';
 export type KafkaConsumerDependencies = KafkaDependencies & Pick<QueueConsumerDependencies, 'transactionObservabilityManager'>;
 export type KafkaBatchProcessingOptions<BatchProcessingEnabled> = BatchProcessingEnabled extends true ? {
     batchProcessingEnabled: true;
@@ -32,8 +32,8 @@ export declare abstract class AbstractKafkaConsumer<TopicsConfig extends TopicCo
      */
     get isActive(): boolean;
     init(): Promise<void>;
-    private handleAsyncBatch;
     private handleSyncStream;
+    private handleSyncStreamBatch;
     close(): Promise<void>;
     private resolveHandler;
     private consume;

package/dist/AbstractKafkaConsumer.js

@@ -1,10 +1,11 @@
 import { randomUUID } from 'node:crypto';
-import { setInterval } from 'node:timers';
+import { pipeline } from 'node:stream/promises';
 import { setTimeout } from 'node:timers/promises';
 import { InternalError, stringValueSerializer, } from '@lokalise/node-core';
 import { Consumer, ProtocolError, ResponseError, stringDeserializer, } from '@platformatic/kafka';
 import { AbstractKafkaService, } from "./AbstractKafkaService.js";
 import { ILLEGAL_GENERATION, REBALANCE_IN_PROGRESS, UNKNOWN_MEMBER_ID } from "./utils/errorCodes.js";
+import { KafkaMessageBatchStream, } from "./utils/KafkaMessageBatchStream.js";
 import { safeJsonDeserializer } from "./utils/safeJsonDeserializer.js";
 const commitErrorCodesToIgnore = new Set([
     ILLEGAL_GENERATION,
@@ -95,46 +96,16 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
                 heartbeatInterval: consumeOptions.heartbeatInterval,
             });
             this.consumerStream = await this.consumer.consume({ ...consumeOptions, topics });
-            this.consumerStream.on('error', (error) => this.handlerError(error));
             if (this.options.batchProcessingEnabled && this.options.batchProcessingOptions) {
-                this.consumerStream.on('end', () => this.logger.debug('stream ended'));
-                this.consumerStream.on('close', () => this.logger.debug('stream closed'));
-                this.consumerStream.on('pause', () => this.logger.debug('stream paused'));
-                this.consumerStream.on('resume', () => this.logger.debug('stream resumed'));
-                /*this.messageBatchStream = new KafkaMessageBatchStream<
-                  DeserializedMessage<SupportedMessageValues<TopicsConfig>>
-                >(
-                  async (batch) => {
-                    const messagesMapped: Record<string, DeserializedMessage<SupportedMessageValues<TopicsConfig>>[]> = {};
-        
-                    for (const item of batch) {
-                      if (!messagesMapped[item.topic])
-                        messagesMapped[item.topic] = [];
-        
-                      messagesMapped[item.topic]!.push(item);
-                    }
-        
-                    const promises: Promise<void>[] = [];
-        
-                    for (const topic in messagesMapped) {
-                      promises.push(this.consume(topic, messagesMapped[topic]!));
-                    }
-        
-                    try {
-                      await Promise.all(promises)
-                    } catch (error) {
-                      console.error("failed to await all consumes", error);
-                      //this.handlerError(error);
-                    }
-                  },
-                  this.options.batchProcessingOptions,
-                )
-        
-                this.consumerStream.pipe(this.messageBatchStream)*/
-                this.handleAsyncBatch(this.consumerStream);
-            }
-            else {
-                this.handleSyncStream(this.consumerStream).catch((error) => this.handlerError(error));
+                this.messageBatchStream = new KafkaMessageBatchStream({
+                    batchSize: this.options.batchProcessingOptions.batchSize,
+                    timeoutMilliseconds: this.options.batchProcessingOptions.timeoutMilliseconds,
+                });
+                // Use pipeline for better error handling and backpressure management
+                pipeline(this.consumerStream, this.messageBatchStream).catch((error) => {
+                    this.logger.error('Stream pipeline failed');
+                    this.handlerError(error);
+                });
             }
         }
         catch (error) {
@@ -144,58 +115,53 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
                 cause: error,
             });
         }
-    }
-    handleAsyncBatch(stream) {
-        let messages = [];
-        let processingMessages = false;
-        const processMessages = async () => {
-            if (processingMessages)
-                return;
-            processingMessages = true;
-            const messagesMapped = {};
-            for (const message of messages) {
-                const key = `${message.topic}.${message.partition}`;
-                if (!messagesMapped[key])
-                    messagesMapped[key] = [];
-                messagesMapped[key]?.push(message);
-            }
-            const promises = [];
-            for (const key in messagesMapped) {
-                // @ts-expect-error
-                promises.push(this.consume(messagesMapped[key][0].topic, messagesMapped[key]));
-            }
-            messages = [];
-            try {
-                await Promise.all(promises);
-            }
-            finally {
-                processingMessages = false;
-            }
-        };
-        setInterval(() => processMessages(), 2000);
-        stream.on('data', (message) => {
-            messages.push(message);
-            if (messages.length < 2000)
-                return;
-            stream.pause();
-            // biome-ignore lint/suspicious/noAsyncPromiseExecutor: it is for testing purposes
-            const promise = new Promise(async (resolve, reject) => {
-                try {
-                    await processMessages();
-                    resolve(undefined);
-                }
-                catch (error) {
-                    reject(error);
-                }
-            });
-            promise.finally(() => stream.resume());
-        });
+        if (this.options.batchProcessingEnabled && this.messageBatchStream) {
+            this.handleSyncStreamBatch(this.messageBatchStream).catch((error) => this.handlerError(error));
+        }
+        else {
+            this.handleSyncStream(this.consumerStream).catch((error) => this.handlerError(error));
+            this.consumerStream.on('error', (error) => this.handlerError(error));
+        }
+        // To be removed
+        // Add logging for stream lifecycle events to diagnose backpressure issues
+        this.consumerStream.on('pause', () => this.logger.debug('Consumer stream PAUSED'));
+        this.consumerStream.on('resume', () => this.logger.debug('Consumer stream RESUMED'));
+        this.consumerStream.on('close', () => this.logger.warn('Consumer stream CLOSE'));
+        this.consumerStream.on('end', () => this.logger.warn('Consumer stream END'));
+        this.consumerStream.on('error', () => this.logger.error('Consumer stream ERROR'));
+        this.messageBatchStream?.on('pause', () => this.logger.debug('Message batch stream PAUSED'));
+        this.messageBatchStream?.on('resume', () => this.logger.debug('Message batch stream RESUMED'));
+        this.messageBatchStream?.on('close', () => this.logger.warn('Message batch stream CLOSE'));
+        this.messageBatchStream?.on('end', () => this.logger.warn('Message batch stream END'));
+        this.messageBatchStream?.on('error', () => this.logger.warn('Message batch stream ERROR'));
     }
     async handleSyncStream(stream) {
         for await (const message of stream) {
             await this.consume(message.topic, message);
         }
     }
+    async handleSyncStreamBatch(stream) {
+        const getTopicPartitionKey = (topic, partition) => `${topic}:${partition}`;
+        const splitTopicPartitionKey = (key) => {
+            const [topic, partition] = key.split(':');
+            /* v8 ignore start */
+            if (!topic || !partition)
+                throw new Error('Invalid topic-partition key format');
+            /* v8 ignore stop */
+            return { topic, partition: Number.parseInt(partition, 10) };
+        };
+        for await (const messageBatch of stream) {
+            const messagesByTopicPartition = {};
+            for (const message of messageBatch) {
+                const key = getTopicPartitionKey(message.topic, message.partition);
+                if (!messagesByTopicPartition[key]) {
+                    messagesByTopicPartition[key] = [];
+                }
+                messagesByTopicPartition[key].push(message);
+            }
+            await Promise.allSettled(Object.entries(messagesByTopicPartition).map(([key, messages]) => this.consume(splitTopicPartitionKey(key).topic, messages)));
+        }
+    }
     async close() {
         if (!this.consumerStream && !this.messageBatchStream) {
             // Leaving the group in case consumer joined but streams were not created
@@ -344,9 +310,9 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
         }
         catch (error) {
             this.logger.debug(logDetails, 'Message commit failed');
-            if (error instanceof ResponseError)
+            if (error instanceof ResponseError) {
                 return this.handleResponseErrorOnCommit(error);
-            throw error;
+            }
         }
     }
     handleResponseErrorOnCommit(responseError) {
@@ -355,7 +321,7 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
             if (error instanceof ProtocolError &&
                 error.apiCode &&
                 commitErrorCodesToIgnore.has(error.apiCode)) {
-                this.logger.error({
+                this.logger.warn({
                     apiCode: error.apiCode,
                     apiId: error.apiId,
                     responseErrorMessage: responseError.message,
@@ -363,10 +329,6 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
                     error: responseError,
                 }, `Failed to commit message: ${error.message}`);
             }
-            else {
-                // If error is not recognized, rethrow it
-                throw responseError;
-            }
         }
     }
     buildTransactionName(topic) {

package/dist/AbstractKafkaConsumer.js.map

@@ -1 +1 @@
-{"version":3,"file":"AbstractKafkaConsumer.js","sourceRoot":"","sources":["../lib/AbstractKafkaConsumer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EACL,aAAa,EACb,qBAAqB,GAEtB,MAAM,qBAAqB,CAAA;AAK5B,OAAO,EAEL,QAAQ,EAIR,aAAa,EACb,aAAa,EACb,kBAAkB,GACnB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACL,oBAAoB,GAGrB,MAAM,2BAA2B,CAAA;AAYlC,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAA;AAKpG,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAA;AAkCtE,MAAM,wBAAwB,GAAG,IAAI,GAAG,CAAC;IACvC,kBAAkB;IAClB,iBAAiB;IACjB,qBAAqB;CACtB,CAAC,CAAA;AAEF;;;GAGG;AACH,MAAM,qBAAqB,GAAG,CAAC,CAAA;AAE/B,MAAM,OAAgB,qBAIpB,SAAQ,oBAGT;IACkB,QAAQ,CAA0C;IAC3D,cAAc,CAAiD;IAC/D,kBAAkB,CAEzB;IAEgB,+BAA+B,CAAiC;IAChE,gBAAgB,CAAkB;IAEnD,YACE,YAAuC,EACvC,OAAqF,EACrF,gBAAkC;QAElC,KAAK,CAAC,YAAY,EAAE,OAAO,CAAC,CAAA;QAE5B,IAAI,CAAC,+BAA+B,GAAG,YAAY,CAAC,+BAA+B,CAAA;QACnF,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAA;QAExC,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC;YAC3B,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK;YACrB,GAAG,IAAI,CAAC,OAAO;YACf,UAAU,EAAE,KAAK,EAAE,4BAA4B;YAC/C,aAAa,EAAE;gBACb,GAAG,EAAE,kBAAkB;gBACvB,KAAK,EAAE,oBAAoB;gBAC3B,SAAS,EAAE,kBAAkB;gBAC7B,WAAW,EAAE,kBAAkB;aAChC;SACF,CAAC,CAAA;QAEF,MAAM,UAAU,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAA;QACnF,qBAAqB;QACrB,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,EAAE,CAC5C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,6BAA6B,CAAC,CAC7D,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,iBAAiB,EAAE,CAAC,CAAC,EAAE,EAAE,CACxC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,kDAAkD,CAAC,CAClF,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,sBAAsB,EAAE,CAAC,CAAC,EAAE,EAAE,CAC7C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,+BAA+B,CAAC,CAC/D,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,0BAA0B,EAAE,CAAC,CAAC,EAAE,EAAE,CACjD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,sBAAsB,CAAC,CACtD,CAAA;QACD,oBAAoB;IACtB,CAAC;IAED;;OAEG;IACH,IAAI,WAAW;QACb,uDAAuD;QACvD,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO,KAAK,CAAA;QAClE,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAA;YAClC,qBAAqB;QACvB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,yEAAyE;YACzE,OAAO,KAAK,CAAA;QACd,CAAC;QACD,oBAAoB;IACtB,CAAC;IAED;;;OAGG;IACH,IAAI,QAAQ;QACV,uDAAuD;QACvD,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO,KAAK,CAAA;QAClE,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAA;YAC/B,qBAAqB;QACvB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,yEAAyE;YACzE,OAAO,KAAK,CAAA;QACd,CAAC;QACD,oBAAoB;IACtB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;QACjD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QACjD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QAE9E,IAAI,CAAC;YACH,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC,OAAO,CAAA,CAAC,8CAA8C;YAEtG,0IAA0I;YAC1I,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC5B,cAAc,EAAE,cAAc,CAAC,cAAc;gBAC7C,gBAAgB,EAAE,cAAc,CAAC,gBAAgB;gBACjD,iBAAiB,EAAE,cAAc,CAAC,iBAAiB;aACpD,CAAC,CAAA;YAEF,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,GAAG,cAAc,EAAE,MAAM,EAAE,CAAC,CAAA;YAChF,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;YAEpE,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;gBAC/E,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAA;gBACtE,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAA;gBACzE,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAA;gBACzE,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAA;gBAE3E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mEA6BmD;gBAEnD,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;YAC5C,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;YACvF,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,aAAa,CAAC;gBACtB,OAAO,EAAE,sBAAsB;gBAC/B,SAAS,EAAE,2BAA2B;gBACtC,KAAK,EAAE,KAAK;aACb,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAEO,gBAAgB,CAAC,MAAsD;QAC7E,IAAI,QAAQ,GAA8C,EAAE,CAAA;QAC5D,IAAI,kBAAkB,GAAY,KAAK,CAAA;QAEvC,MAAM,eAAe,GAAG,KAAK,IAAI,EAAE;YACjC,IAAI,kBAAkB;gBAAE,OAAM;YAE9B,kBAAkB,GAAG,IAAI,CAAA;YAEzB,MAAM,cAAc,GAA8D,EAAE,CAAA;YAEpF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,MAAM,GAAG,GAAG,GAAG,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,SAAS,EAAE,CAAA;gBAEnD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC;oBAAE,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,CAAA;gBAElD,cAAc,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;YACpC,CAAC;YAED,MAAM,QAAQ,GAAuB,EAAE,CAAA;YAEvC,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;gBACjC,mBAAmB;gBACnB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YAChF,CAAC;YAED,QAAQ,GAAG,EAAE,CAAA;YAEb,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;YAC7B,CAAC;oBAAS,CAAC;gBACT,kBAAkB,GAAG,KAAK,CAAA;YAC5B,CAAC;QACH,CAAC,CAAA;QAED,WAAW,CAAC,GAAG,EAAE,CAAC,eAAe,EAAE,EAAE,IAAI,CAAC,CAAA;QAE1C,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE;YAC5B,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAEtB,IAAI,QAAQ,CAAC,MAAM,GAAG,IAAI;gBAAE,OAAM;YAElC,MAAM,CAAC,KAAK,EAAE,CAAA;YAEd,kFAAkF;YAClF,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;gBACpD,IAAI,CAAC;oBACH,MAAM,eAAe,EAAE,CAAA;oBACvB,OAAO,CAAC,SAAS,CAAC,CAAA;gBACpB,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,CAAC,KAAK,CAAC,CAAA;gBACf,CAAC;YACH,CAAC,CAAC,CAAA;YAEF,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;QACxC,CAAC,CAAC,CAAA;IACJ,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,MAAsD;QAEtD,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;YACnC,MAAM,IAAI,CAAC,OAAO,CAChB,OAAO,CAAC,KAAK,EACb,OAAoE,CACrE,CAAA;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACrD,yEAAyE;YACzE,IAAI,IAAI,CAAC,QAAQ;gBAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAA;YAC7C,OAAM;QACR,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAA;YACjC,IAAI,CAAC,cAAc,GAAG,SAAS,CAAA;QACjC,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAA;YACrE,IAAI,CAAC,kBAAkB,GAAG,SAAS,CAAA;QACrC,CAAC;QAED,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;IAC7B,CAAC;IAEO,cAAc,CAAC,KAAoC;QACzD,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IACrC,CAAC;IAEO,KAAK,CAAC,OAAO,CACnB,KAAa,EACb,cAAoE;QAEpE,MAAM,+BAA+B,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAClD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,sBAAsB,CAAC,CAAA;QAEnF,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;QAEhD,iFAAiF;QACjF,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;QAEtD,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CACtC,aAAa,EACb,cAAc,EACd,+BAA+B,CAChC,CAAA;QAED,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,+BAA+B,CAAC,CAAA;YAC5F,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;QACpC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,kBAAkB,EAAE,aAAa,CAAC,MAAM,EAAE,EAClF,sCAAsC,CACvC,CAAA;QACH,CAAC;QAED,kFAAkF;QAClF,MAAM,YAAY,GAAG,aAAa,CAAC,CAAC,CAAE,CAAA;QACtC,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAA;QAE3D,MAAM,aAAa,GAAG,UAAU,EAAE,CAAA;QAClC,IAAI,CAAC,+BAA+B,EAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,EAAE,aAAa,CAAC,CAAA;QAE5F,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,uBAAuB,CACzD,KAAK;QACL,0EAA0E;QAC1E,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,EAClE,aAAa,CAAC,OAAO,EACrB,cAAc,CACf,CAAA;QAED,IAAI,CAAC,uBAAuB,CAAC,aAAa,EAAE,gBAAgB,EAAE,+BAA+B,CAAC,CAAA;QAE9F,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,aAAa,CAAC,CAAA;QAEzD,+EAA+E;QAC/E,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;IACpC,CAAC;IAEO,aAAa,CACnB,aAIC,EACD,cAAoE,EACpE,+BAAuC;QAEvC,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAA;QAEzF,MAAM,aAAa,GAAgE,EAAE,CAAA;QAErF,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,yEAAyE;YACzE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACnB,SAAQ;YACV,CAAC;YAED,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAEjE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;gBACzB,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,EAAE;oBACnC,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC;iBAC9C,CAAC,CAAA;gBACF,IAAI,CAAC,sBAAsB,CAAC;oBAC1B,OAAO,EAAE,OAAO;oBAChB,gBAAgB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE;oBACpE,+BAA+B;iBAChC,CAAC,CAAA;gBACF,SAAQ;YACV,CAAC;YAED,aAAa,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,IAAI,EAAE,CAAC,CAAA;QAC7D,CAAC;QAED,OAAO,aAAa,CAAA;IACtB,CAAC;IAEO,KAAK,CAAC,uBAAuB,CACnC,KAAa,EACb,cAA4C,EAC5C,OAA6E,EAC7E,cAA8B;QAE9B,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,gBAAgB,GAA4B;YAC9C,MAAM,EAAE,OAAO;YACf,WAAW,EAAE,cAAc;SAC5B,CAAA;QACD,GAAG,CAAC;YACF,qCAAqC;YACrC,IAAI,OAAO,GAAG,CAAC;gBAAE,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAA;YAE3D,gBAAgB,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,cAAc,CAAC,CAAA;YAC1F,IAAI,gBAAgB,CAAC,MAAM,KAAK,UAAU;gBAAE,MAAK;YAEjD,OAAO,EAAE,CAAA;QACX,CAAC,QAAQ,OAAO,GAAG,qBAAqB,EAAC;QAEzC,OAAO,gBAAgB,CAAA;IACzB,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,KAAa,EACb,cAA4C,EAC5C,OAA6E,EAC7E,cAA8B;QAE9B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;YAC7C,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,CAAC,OAAO,EAAE,CAAC;gBACpD,MAAM,IAAI,KAAK,CACb,6EAA6E,CAC9E,CAAA;YACH,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,OAAO,EAAE,CAAC;gBACpD,MAAM,IAAI,KAAK,CACb,iFAAiF,CAClF,CAAA;YACH,CAAC;YAED,MAAM,OAAO;YACX,mGAAmG;YACnG,cAEuC,EACvC,IAAI,CAAC,gBAAgB,EACrB,cAAc,CACf,CAAA;YACD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAA;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC;gBAChD,CAAC,CAAC,EAAE,SAAS,EAAE,cAAc,CAAC,MAAM,EAAE;gBACtC,CAAC,CAAC,EAAE,OAAO,EAAE,qBAAqB,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,CAAA;YAC5D,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE;gBACvB,KAAK;gBACL,GAAG,YAAY;aAChB,CAAC,CAAA;QACJ,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,CAAA;IACzD,CAAC;IAEO,uBAAuB,CAC7B,QAA0C,EAC1C,gBAAyC,EACzC,+BAAuC;QAEvC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,sBAAsB,CAAC;gBAC1B,OAAO;gBACP,gBAAgB;gBAChB,+BAA+B;aAChC,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,cAAoE;QACjF,IAAI,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;YAClC,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;YAEzD,wEAAwE;YACxE,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAA;QACvE,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,CAAA;QAC3C,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAkE;QAC5F,MAAM,UAAU,GAAG;YACjB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,SAAS,EAAE,OAAO,CAAC,SAAS;SAC7B,CAAA;QACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,0BAA0B,CAAC,CAAA;QAEzD,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,MAAM,EAAE,CAAA;YACtB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,gCAAgC,CAAC,CAAA;QACjE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAA;YACtD,IAAI,KAAK,YAAY,aAAa;gBAAE,OAAO,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAA;YAClF,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAEO,2BAA2B,CAAC,aAA4B;QAC9D,kFAAkF;QAClF,KAAK,MAAM,KAAK,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YACzC,IACE,KAAK,YAAY,aAAa;gBAC9B,KAAK,CAAC,OAAO;gBACb,wBAAwB,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAC3C,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,KAAK,CACf;oBACE,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,oBAAoB,EAAE,aAAa,CAAC,OAAO;oBAC3C,oBAAoB,EAAE,KAAK,CAAC,OAAO;oBACnC,KAAK,EAAE,aAAa;iBACrB,EACD,6BAA6B,KAAK,CAAC,OAAO,EAAE,CAC7C,CAAA;YACH,CAAC;iBAAM,CAAC;gBACN,yCAAyC;gBACzC,MAAM,aAAa,CAAA;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IAEO,oBAAoB,CAAC,KAAa;QACxC,MAAM,mBAAmB,GAAG,SAAS,IAAI,CAAC,WAAW,CAAC,IAAI,IAAI,KAAK,EAAE,CAAA;QACrE,OAAO,IAAI,CAAC,OAAO,CAAC,sBAAsB;YACxC,CAAC,CAAC,GAAG,mBAAmB,QAAQ;YAChC,CAAC,CAAC,mBAAmB,CAAA;IACzB,CAAC;IAEO,iBAAiB,CACvB,OAAkE;QAElE,IAAI,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC,CAAA;QACnE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,KAAK,GAAG,UAAU,EAAE,CAAA;QAE7D,OAAO;YACL,KAAK;YACL,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;gBACxB,cAAc,EAAE,KAAK;gBACrB,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;gBAC7B,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,UAAU,EAAE,OAAO,CAAC,GAAG;aACxB,CAAC;SACH,CAAA;IACH,CAAC;CACF"}
+{"version":3,"file":"AbstractKafkaConsumer.js","sourceRoot":"","sources":["../lib/AbstractKafkaConsumer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EACL,aAAa,EACb,qBAAqB,GAEtB,MAAM,qBAAqB,CAAA;AAK5B,OAAO,EAEL,QAAQ,EAGR,aAAa,EACb,aAAa,EACb,kBAAkB,GACnB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACL,oBAAoB,GAGrB,MAAM,2BAA2B,CAAA;AAYlC,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAA;AACpG,OAAO,EAEL,uBAAuB,GACxB,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAA;AAkCtE,MAAM,wBAAwB,GAAG,IAAI,GAAG,CAAC;IACvC,kBAAkB;IAClB,iBAAiB;IACjB,qBAAqB;CACtB,CAAC,CAAA;AAEF;;;GAGG;AACH,MAAM,qBAAqB,GAAG,CAAC,CAAA;AAE/B,MAAM,OAAgB,qBAIpB,SAAQ,oBAGT;IACkB,QAAQ,CAA0C;IAC3D,cAAc,CAAiD;IAC/D,kBAAkB,CAEzB;IAEgB,+BAA+B,CAAiC;IAChE,gBAAgB,CAAkB;IAEnD,YACE,YAAuC,EACvC,OAAqF,EACrF,gBAAkC;QAElC,KAAK,CAAC,YAAY,EAAE,OAAO,CAAC,CAAA;QAE5B,IAAI,CAAC,+BAA+B,GAAG,YAAY,CAAC,+BAA+B,CAAA;QACnF,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAA;QAExC,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC;YAC3B,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK;YACrB,GAAG,IAAI,CAAC,OAAO;YACf,UAAU,EAAE,KAAK,EAAE,4BAA4B;YAC/C,aAAa,EAAE;gBACb,GAAG,EAAE,kBAAkB;gBACvB,KAAK,EAAE,oBAAoB;gBAC3B,SAAS,EAAE,kBAAkB;gBAC7B,WAAW,EAAE,kBAAkB;aAChC;SACF,CAAC,CAAA;QAEF,MAAM,UAAU,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAA;QACnF,qBAAqB;QACrB,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,EAAE,CAC5C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,6BAA6B,CAAC,CAC7D,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,iBAAiB,EAAE,CAAC,CAAC,EAAE,EAAE,CACxC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,kDAAkD,CAAC,CAClF,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,sBAAsB,EAAE,CAAC,CAAC,EAAE,EAAE,CAC7C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,+BAA+B,CAAC,CAC/D,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,0BAA0B,EAAE,CAAC,CAAC,EAAE,EAAE,CACjD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,sBAAsB,CAAC,CACtD,CAAA;QACD,oBAAoB;IACtB,CAAC;IAED;;OAEG;IACH,IAAI,WAAW;QACb,uDAAuD;QACvD,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO,KAAK,CAAA;QAClE,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAA;YAClC,qBAAqB;QACvB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,yEAAyE;YACzE,OAAO,KAAK,CAAA;QACd,CAAC;QACD,oBAAoB;IACtB,CAAC;IAED;;;OAGG;IACH,IAAI,QAAQ;QACV,uDAAuD;QACvD,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO,KAAK,CAAA;QAClE,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAA;YAC/B,qBAAqB;QACvB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,yEAAyE;YACzE,OAAO,KAAK,CAAA;QACd,CAAC;QACD,oBAAoB;IACtB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;QACjD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QACjD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QAE9E,IAAI,CAAC;YACH,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC,OAAO,CAAA,CAAC,8CAA8C;YAEtG,0IAA0I;YAC1I,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC5B,cAAc,EAAE,cAAc,CAAC,cAAc;gBAC7C,gBAAgB,EAAE,cAAc,CAAC,gBAAgB;gBACjD,iBAAiB,EAAE,cAAc,CAAC,iBAAiB;aACpD,CAAC,CAAA;YAEF,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,GAAG,cAAc,EAAE,MAAM,EAAE,CAAC,CAAA;YAChF,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;gBAC/E,IAAI,CAAC,kBAAkB,GAAG,IAAI,uBAAuB,CAEnD;oBACA,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,SAAS;oBACxD,mBAAmB,EAAE,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,mBAAmB;iBAC7E,CAAC,CAAA;gBAEF,qEAAqE;gBACrE,QAAQ,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;oBACrE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAA;oBAC3C,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAA;gBAC1B,CAAC,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,aAAa,CAAC;gBACtB,OAAO,EAAE,sBAAsB;gBAC/B,SAAS,EAAE,2BAA2B;gBACtC,KAAK,EAAE,KAAK;aACb,CAAC,CAAA;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACnE,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;QAChG,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;YACrF,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;QACtE,CAAC;QAED,gBAAgB;QAChB,0EAA0E;QAC1E,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,CAAA;QAClF,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC,CAAA;QACpF,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC,CAAA;QAChF,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAA;QAC5E,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC,CAAA;QACjF,IAAI,CAAC,kBAAkB,EAAE,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC,CAAA;QAC5F,IAAI,CAAC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC,CAAA;QAC9F,IAAI,CAAC,kBAAkB,EAAE,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC,CAAA;QAC1F,IAAI,CAAC,kBAAkB,EAAE,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC,CAAA;QACtF,IAAI,CAAC,kBAAkB,EAAE,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC,CAAA;IAC5F,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,MAAsD;QAEtD,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;YACnC,MAAM,IAAI,CAAC,OAAO,CAChB,OAAO,CAAC,KAAK,EACb,OAAoE,CACrE,CAAA;QACH,CAAC;IACH,CAAC;IACO,KAAK,CAAC,qBAAqB,CACjC,MAA0F;QAE1F,MAAM,oBAAoB,GAAG,CAAC,KAAa,EAAE,SAAiB,EAAU,EAAE,CACxE,GAAG,KAAK,IAAI,SAAS,EAAE,CAAA;QACzB,MAAM,sBAAsB,GAAG,CAAC,GAAW,EAAwC,EAAE;YACnF,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;YACzC,qBAAqB;YACrB,IAAI,CAAC,KAAK,IAAI,CAAC,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;YAC/E,oBAAoB;YAEpB,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,EAAE,CAAA;QAC7D,CAAC,CAAA;QAED,IAAI,KAAK,EAAE,MAAM,YAAY,IAAI,MAAM,EAAE,CAAC;YACxC,MAAM,wBAAwB,GAG1B,EAAE,CAAA;YACN,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;gBACnC,MAAM,GAAG,GAAG,oBAAoB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,CAAA;gBAClE,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,EAAE,CAAC;oBACnC,wBAAwB,CAAC,GAAG,CAAC,GAAG,EAAE,CAAA;gBACpC,CAAC;gBACD,wBAAwB,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAC7C,CAAC;YAED,MAAM,OAAO,CAAC,UAAU,CACtB,MAAM,CAAC,OAAO,CAAC,wBAAwB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,EAAE,CAC/D,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,QAAQ,CAAC,CAC1D,CACF,CAAA;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACrD,yEAAyE;YACzE,IAAI,IAAI,CAAC,QAAQ;gBAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAA;YAC7C,OAAM;QACR,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAA;YACjC,IAAI,CAAC,cAAc,GAAG,SAAS,CAAA;QACjC,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAA;YACrE,IAAI,CAAC,kBAAkB,GAAG,SAAS,CAAA;QACrC,CAAC;QAED,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;IAC7B,CAAC;IAEO,cAAc,CAAC,KAAoC;QACzD,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IACrC,CAAC;IAEO,KAAK,CAAC,OAAO,CACnB,KAAa,EACb,cAAoE;QAEpE,MAAM,+BAA+B,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAClD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,sBAAsB,CAAC,CAAA;QAEnF,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;QAEhD,iFAAiF;QACjF,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;QAEtD,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CACtC,aAAa,EACb,cAAc,EACd,+BAA+B,CAChC,CAAA;QAED,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,+BAA+B,CAAC,CAAA;YAC5F,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;QACpC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,kBAAkB,EAAE,aAAa,CAAC,MAAM,EAAE,EAClF,sCAAsC,CACvC,CAAA;QACH,CAAC;QAED,kFAAkF;QAClF,MAAM,YAAY,GAAG,aAAa,CAAC,CAAC,CAAE,CAAA;QACtC,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAA;QAE3D,MAAM,aAAa,GAAG,UAAU,EAAE,CAAA;QAClC,IAAI,CAAC,+BAA+B,EAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,EAAE,aAAa,CAAC,CAAA;QAE5F,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,uBAAuB,CACzD,KAAK;QACL,0EAA0E;QAC1E,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,EAClE,aAAa,CAAC,OAAO,EACrB,cAAc,CACf,CAAA;QAED,IAAI,CAAC,uBAAuB,CAAC,aAAa,EAAE,gBAAgB,EAAE,+BAA+B,CAAC,CAAA;QAE9F,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,aAAa,CAAC,CAAA;QAEzD,+EAA+E;QAC/E,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;IACpC,CAAC;IAEO,aAAa,CACnB,aAIC,EACD,cAAoE,EACpE,+BAAuC;QAEvC,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAA;QAEzF,MAAM,aAAa,GAAgE,EAAE,CAAA;QAErF,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,yEAAyE;YACzE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACnB,SAAQ;YACV,CAAC;YAED,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAEjE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;gBACzB,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,EAAE;oBACnC,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC;iBAC9C,CAAC,CAAA;gBACF,IAAI,CAAC,sBAAsB,CAAC;oBAC1B,OAAO,EAAE,OAAO;oBAChB,gBAAgB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE;oBACpE,+BAA+B;iBAChC,CAAC,CAAA;gBACF,SAAQ;YACV,CAAC;YAED,aAAa,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,IAAI,EAAE,CAAC,CAAA;QAC7D,CAAC;QAED,OAAO,aAAa,CAAA;IACtB,CAAC;IAEO,KAAK,CAAC,uBAAuB,CACnC,KAAa,EACb,cAA4C,EAC5C,OAA6E,EAC7E,cAA8B;QAE9B,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,gBAAgB,GAA4B;YAC9C,MAAM,EAAE,OAAO;YACf,WAAW,EAAE,cAAc;SAC5B,CAAA;QACD,GAAG,CAAC;YACF,qCAAqC;YACrC,IAAI,OAAO,GAAG,CAAC;gBAAE,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAA;YAE3D,gBAAgB,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,cAAc,CAAC,CAAA;YAC1F,IAAI,gBAAgB,CAAC,MAAM,KAAK,UAAU;gBAAE,MAAK;YAEjD,OAAO,EAAE,CAAA;QACX,CAAC,QAAQ,OAAO,GAAG,qBAAqB,EAAC;QAEzC,OAAO,gBAAgB,CAAA;IACzB,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,KAAa,EACb,cAA4C,EAC5C,OAA6E,EAC7E,cAA8B;QAE9B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;YAC7C,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,CAAC,OAAO,EAAE,CAAC;gBACpD,MAAM,IAAI,KAAK,CACb,6EAA6E,CAC9E,CAAA;YACH,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,OAAO,EAAE,CAAC;gBACpD,MAAM,IAAI,KAAK,CACb,iFAAiF,CAClF,CAAA;YACH,CAAC;YAED,MAAM,OAAO;YACX,mGAAmG;YACnG,cAEuC,EACvC,IAAI,CAAC,gBAAgB,EACrB,cAAc,CACf,CAAA;YACD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAA;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC;gBAChD,CAAC,CAAC,EAAE,SAAS,EAAE,cAAc,CAAC,MAAM,EAAE;gBACtC,CAAC,CAAC,EAAE,OAAO,EAAE,qBAAqB,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,CAAA;YAC5D,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE;gBACvB,KAAK;gBACL,GAAG,YAAY;aAChB,CAAC,CAAA;QACJ,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,CAAA;IACzD,CAAC;IAEO,uBAAuB,CAC7B,QAA0C,EAC1C,gBAAyC,EACzC,+BAAuC;QAEvC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,sBAAsB,CAAC;gBAC1B,OAAO;gBACP,gBAAgB;gBAChB,+BAA+B;aAChC,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,cAAoE;QACjF,IAAI,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;YAClC,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;YAEzD,wEAAwE;YACxE,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAA;QACvE,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,CAAA;QAC3C,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAkE;QAC5F,MAAM,UAAU,GAAG;YACjB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,SAAS,EAAE,OAAO,CAAC,SAAS;SAC7B,CAAA;QACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,0BAA0B,CAAC,CAAA;QAEzD,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,MAAM,EAAE,CAAA;YACtB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,gCAAgC,CAAC,CAAA;QACjE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAA;YACtD,IAAI,KAAK,YAAY,aAAa,EAAE,CAAC;gBACnC,OAAO,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAA;YAChD,CAAC;QACH,CAAC;IACH,CAAC;IAEO,2BAA2B,CAAC,aAA4B;QAC9D,kFAAkF;QAClF,KAAK,MAAM,KAAK,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YACzC,IACE,KAAK,YAAY,aAAa;gBAC9B,KAAK,CAAC,OAAO;gBACb,wBAAwB,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAC3C,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,IAAI,CACd;oBACE,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,oBAAoB,EAAE,aAAa,CAAC,OAAO;oBAC3C,oBAAoB,EAAE,KAAK,CAAC,OAAO;oBACnC,KAAK,EAAE,aAAa;iBACrB,EACD,6BAA6B,KAAK,CAAC,OAAO,EAAE,CAC7C,CAAA;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAEO,oBAAoB,CAAC,KAAa;QACxC,MAAM,mBAAmB,GAAG,SAAS,IAAI,CAAC,WAAW,CAAC,IAAI,IAAI,KAAK,EAAE,CAAA;QACrE,OAAO,IAAI,CAAC,OAAO,CAAC,sBAAsB;YACxC,CAAC,CAAC,GAAG,mBAAmB,QAAQ;YAChC,CAAC,CAAC,mBAAmB,CAAA;IACzB,CAAC;IAEO,iBAAiB,CACvB,OAAkE;QAElE,IAAI,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC,CAAA;QACnE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,KAAK,GAAG,UAAU,EAAE,CAAA;QAE7D,OAAO;YACL,KAAK;YACL,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;gBACxB,cAAc,EAAE,KAAK;gBACrB,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;gBAC7B,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,UAAU,EAAE,OAAO,CAAC,GAAG;aACxB,CAAC;SACH,CAAA;IACH,CAAC;CACF"}

package/dist/utils/KafkaMessageBatchStream.d.ts

@@ -1,18 +1,9 @@
-import { Transform } from 'node:stream';
-type MessageWithTopicAndPartition = {
-    topic: string;
-    partition: number;
-};
+import { Duplex } from 'node:stream';
+type CallbackFunction = (error?: Error | null) => void;
 export type KafkaMessageBatchOptions = {
     batchSize: number;
     timeoutMilliseconds: number;
 };
-export type MessageBatch<TMessage> = {
-    topic: string;
-    partition: number;
-    messages: TMessage[];
-};
-export type OnMessageBatchCallback<TMessage> = (batch: MessageBatch<TMessage>) => Promise<void>;
 /**
  * Collects messages in batches based on provided batchSize and flushes them when messages amount or timeout is reached.
  *
@@ -20,20 +11,20 @@ export type OnMessageBatchCallback<TMessage> = (batch: MessageBatch<TMessage>) =
  * When the downstream consumer is slow, the stream will automatically pause accepting new messages
  * until the consumer catches up, preventing memory leaks and OOM errors.
  */
-export declare class KafkaMessageBatchStream<TMessage extends MessageWithTopicAndPartition> extends Transform {
-    private readonly onBatch;
+export declare class KafkaMessageBatchStream<TMessage> extends Duplex {
     private readonly batchSize;
     private readonly timeout;
-    private readonly currentBatchPerTopicPartition;
-    private readonly batchTimeoutPerTopicPartition;
-    private readonly timeoutProcessingPromises;
-    constructor(onBatch: OnMessageBatchCallback<TMessage>, options: {
+    private readonly messages;
+    private existingTimeout;
+    private pendingCallback;
+    constructor(options: {
         batchSize: number;
         timeoutMilliseconds: number;
     });
-    _transform(message: TMessage, _encoding: BufferEncoding, callback: () => void): Promise<void>;
-    _flush(callback: () => void): Promise<void>;
-    private flushAllBatches;
-    private flushCurrentBatchMessages;
+    _read(): void;
+    _write(message: TMessage, _encoding: BufferEncoding, callback: CallbackFunction): void;
+    _final(callback: CallbackFunction): void;
+    private flushMessages;
+    push(chunk: TMessage[] | null, encoding?: BufferEncoding): boolean;
 }
 export {};

package/dist/utils/KafkaMessageBatchStream.js

@@ -1,4 +1,4 @@
-import { Transform } from 'node:stream';
+import { Duplex } from 'node:stream';
 /**
  * Collects messages in batches based on provided batchSize and flushes them when messages amount or timeout is reached.
  *
@@ -6,77 +6,60 @@ import { Transform } from 'node:stream';
  * When the downstream consumer is slow, the stream will automatically pause accepting new messages
  * until the consumer catches up, preventing memory leaks and OOM errors.
  */
-export class KafkaMessageBatchStream extends Transform {
-    onBatch;
+export class KafkaMessageBatchStream extends Duplex {
     batchSize;
     timeout;
-    currentBatchPerTopicPartition;
-    batchTimeoutPerTopicPartition;
-    timeoutProcessingPromises = new Map();
-    constructor(onBatch, options) {
+    messages;
+    existingTimeout;
+    pendingCallback;
+    constructor(options) {
         super({ objectMode: true });
-        this.onBatch = onBatch;
         this.batchSize = options.batchSize;
         this.timeout = options.timeoutMilliseconds;
-        this.currentBatchPerTopicPartition = {};
-        this.batchTimeoutPerTopicPartition = {};
+        this.messages = [];
     }
-    async _transform(message, _encoding, callback) {
-        const key = getTopicPartitionKey(message.topic, message.partition);
-        // Wait for all pending timeout flushes to complete to maintain backpressure
-        if (this.timeoutProcessingPromises.size > 0) {
-            // Capture a snapshot of current promises to avoid race conditions with new timeouts
-            const promiseEntries = Array.from(this.timeoutProcessingPromises.entries());
-            // Wait for all to complete and then clean up from the map
-            await Promise.all(promiseEntries.map(([k, p]) => p.finally(() => this.timeoutProcessingPromises.delete(k))));
-        }
-        // Accumulate the message
-        if (!this.currentBatchPerTopicPartition[key])
-            this.currentBatchPerTopicPartition[key] = [];
-        this.currentBatchPerTopicPartition[key].push(message);
-        // Check if the batch is complete by size
-        if (this.currentBatchPerTopicPartition[key].length >= this.batchSize) {
-            await this.flushCurrentBatchMessages(message.topic, message.partition);
-            callback();
+    _read() {
+        // When _read is called, it means the downstream consumer is ready for more data
+        // This is when we should resume the writable side by calling the pending callback if it exists
+        if (!this.pendingCallback)
             return;
+        const cb = this.pendingCallback;
+        this.pendingCallback = undefined;
+        cb();
+    }
+    _write(message, _encoding, callback) {
+        let canContinue = true;
+        try {
+            this.messages.push(message);
+            if (this.messages.length >= this.batchSize) {
+                canContinue = this.flushMessages();
+            }
+            else {
+                // If backpressure happens, we don't have a callback to hold
+                // The next _write will handle backpressure
+                this.existingTimeout ??= setTimeout(() => this.flushMessages(), this.timeout);
+            }
         }
-        // Start timeout for this partition if not already started
-        if (!this.batchTimeoutPerTopicPartition[key]) {
-            this.batchTimeoutPerTopicPartition[key] = setTimeout(() => this.timeoutProcessingPromises.set(key, this.flushCurrentBatchMessages(message.topic, message.partition)), this.timeout);
+        finally {
+            if (!canContinue)
+                this.pendingCallback = callback;
+            else
+                callback();
         }
-        callback();
     }
-    // Flush all remaining batches when stream is closing
-    async _flush(callback) {
-        await this.flushAllBatches();
+    _final(callback) {
+        this.flushMessages();
+        this.push(null); // End readable side
         callback();
     }
-    async flushAllBatches() {
-        for (const key of Object.keys(this.currentBatchPerTopicPartition)) {
-            const { topic, partition } = splitTopicPartitionKey(key);
-            await this.flushCurrentBatchMessages(topic, partition);
-        }
+    flushMessages() {
+        clearTimeout(this.existingTimeout);
+        this.existingTimeout = undefined;
+        const messages = this.messages.splice(0, this.messages.length);
+        return this.push(messages);
     }
-    async flushCurrentBatchMessages(topic, partition) {
-        const key = getTopicPartitionKey(topic, partition);
-        // Clear timeout
-        if (this.batchTimeoutPerTopicPartition[key]) {
-            clearTimeout(this.batchTimeoutPerTopicPartition[key]);
-            this.batchTimeoutPerTopicPartition[key] = undefined;
-        }
-        const messages = this.currentBatchPerTopicPartition[key] ?? [];
-        // Push the batch downstream
-        await this.onBatch({ topic, partition, messages });
-        this.currentBatchPerTopicPartition[key] = [];
+    push(chunk, encoding) {
+        return super.push(chunk, encoding);
     }
 }
-const getTopicPartitionKey = (topic, partition) => `${topic}:${partition}`;
-const splitTopicPartitionKey = (key) => {
-    const [topic, partition] = key.split(':');
-    /* v8 ignore start */
-    if (!topic || !partition)
-        throw new Error('Invalid topic-partition key format');
-    /* v8 ignore stop */
-    return { topic, partition: Number.parseInt(partition, 10) };
-};
 //# sourceMappingURL=KafkaMessageBatchStream.js.map

package/dist/utils/KafkaMessageBatchStream.js.map

@@ -1 +1 @@
-{"version":3,"file":"KafkaMessageBatchStream.js","sourceRoot":"","sources":["../../lib/utils/KafkaMessageBatchStream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAavC;;;;;;GAMG;AACH,MAAM,OAAO,uBAEX,SAAQ,SAAS;IACA,OAAO,CAAkC;IACzC,SAAS,CAAQ;IACjB,OAAO,CAAQ;IAEf,6BAA6B,CAA4B;IACzD,6BAA6B,CAA4C;IAEzE,yBAAyB,GAA+B,IAAI,GAAG,EAAE,CAAA;IAElF,YACE,OAAyC,EACzC,OAA2D;QAE3D,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAA;QAC3B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAA;QACtB,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAA;QAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,mBAAmB,CAAA;QAC1C,IAAI,CAAC,6BAA6B,GAAG,EAAE,CAAA;QACvC,IAAI,CAAC,6BAA6B,GAAG,EAAE,CAAA;IACzC,CAAC;IAEQ,KAAK,CAAC,UAAU,CAAC,OAAiB,EAAE,SAAyB,EAAE,QAAoB;QAC1F,MAAM,GAAG,GAAG,oBAAoB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,CAAA;QAElE,4EAA4E;QAC5E,IAAI,IAAI,CAAC,yBAAyB,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC5C,oFAAoF;YACpF,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,yBAAyB,CAAC,OAAO,EAAE,CAAC,CAAA;YAC3E,0DAA0D;YAC1D,MAAM,OAAO,CAAC,GAAG,CACf,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAC1F,CAAA;QACH,CAAC;QAED,yBAAyB;QACzB,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,GAAG,EAAE,CAAA;QAC1F,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAErD,yCAAyC;QACzC,IAAI,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrE,MAAM,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,CAAA;YACtE,QAAQ,EAAE,CAAA;YACV,OAAM;QACR,CAAC;QAED,0DAA0D;QAC1D,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,GAAG,UAAU,CAClD,GAAG,EAAE,CACH,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAChC,GAAG,EACH,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,CACjE,EACH,IAAI,CAAC,OAAO,CACb,CAAA;QACH,CAAC;QAED,QAAQ,EAAE,CAAA;IACZ,CAAC;IAED,qDAAqD;IAC5C,KAAK,CAAC,MAAM,CAAC,QAAoB;QACxC,MAAM,IAAI,CAAC,eAAe,EAAE,CAAA;QAC5B,QAAQ,EAAE,CAAA;IACZ,CAAC;IAEO,KAAK,CAAC,eAAe;QAC3B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,EAAE,CAAC;YAClE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,sBAAsB,CAAC,GAAG,CAAC,CAAA;YACxD,MAAM,IAAI,CAAC,yBAAyB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;QACxD,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,yBAAyB,CAAC,KAAa,EAAE,SAAiB;QACtE,MAAM,GAAG,GAAG,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;QAElD,gBAAgB;QAChB,IAAI,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5C,YAAY,CAAC,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,CAAC,CAAA;YACrD,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,GAAG,SAAS,CAAA;QACrD,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,IAAI,EAAE,CAAA;QAE9D,4BAA4B;QAC5B,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAA;QAClD,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,GAAG,EAAE,CAAA;IAC9C,CAAC;CACF;AAED,MAAM,oBAAoB,GAAG,CAAC,KAAa,EAAE,SAAiB,EAAU,EAAE,CAAC,GAAG,KAAK,IAAI,SAAS,EAAE,CAAA;AAClG,MAAM,sBAAsB,GAAG,CAAC,GAAW,EAAwC,EAAE;IACnF,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IACzC,qBAAqB;IACrB,IAAI,CAAC,KAAK,IAAI,CAAC,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;IAC/E,oBAAoB;IAEpB,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,EAAE,CAAA;AAC7D,CAAC,CAAA"}
+{"version":3,"file":"KafkaMessageBatchStream.js","sourceRoot":"","sources":["../../lib/utils/KafkaMessageBatchStream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AASpC;;;;;;GAMG;AACH,MAAM,OAAO,uBAAkC,SAAQ,MAAM;IAC1C,SAAS,CAAQ;IACjB,OAAO,CAAQ;IAEf,QAAQ,CAAY;IAC7B,eAAe,CAA4B;IAC3C,eAAe,CAA8B;IAErD,YAAY,OAA2D;QACrE,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAA;QAC3B,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAA;QAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,mBAAmB,CAAA;QAE1C,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAA;IACpB,CAAC;IAEQ,KAAK;QACZ,gFAAgF;QAChF,+FAA+F;QAC/F,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,OAAM;QAEjC,MAAM,EAAE,GAAG,IAAI,CAAC,eAAe,CAAA;QAC/B,IAAI,CAAC,eAAe,GAAG,SAAS,CAAA;QAChC,EAAE,EAAE,CAAA;IACN,CAAC;IAEQ,MAAM,CAAC,OAAiB,EAAE,SAAyB,EAAE,QAA0B;QACtF,IAAI,WAAW,GAAG,IAAI,CAAA;QAEtB,IAAI,CAAC;YACH,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAE3B,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC3C,WAAW,GAAG,IAAI,CAAC,aAAa,EAAE,CAAA;YACpC,CAAC;iBAAM,CAAC;gBACN,4DAA4D;gBAC5D,2CAA2C;gBAC3C,IAAI,CAAC,eAAe,KAAK,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;YAC/E,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,WAAW;gBAAE,IAAI,CAAC,eAAe,GAAG,QAAQ,CAAA;;gBAC5C,QAAQ,EAAE,CAAA;QACjB,CAAC;IACH,CAAC;IAEQ,MAAM,CAAC,QAA0B;QACxC,IAAI,CAAC,aAAa,EAAE,CAAA;QACpB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA,CAAC,oBAAoB;QACpC,QAAQ,EAAE,CAAA;IACZ,CAAC;IAEO,aAAa;QACnB,YAAY,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;QAClC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAA;QAEhC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;QAC9D,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IAC5B,CAAC;IAEQ,IAAI,CAAC,KAAwB,EAAE,QAAyB;QAC/D,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;IACpC,CAAC;CACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@message-queue-toolkit/kafka",
-    "version": "0.9.1-beta.1",
+    "version": "0.9.1-beta.11",
     "engines": {
         "node": ">= 22.14.0"
     },
@@ -53,7 +53,7 @@
     "dependencies": {
         "@lokalise/node-core": "^14.2.0",
         "@lokalise/universal-ts-utils": "^4.5.1",
-        "@platformatic/kafka": "^1.22.0"
+        "@platformatic/kafka": "^1.23.0"
     },
     "peerDependencies": {
         "@message-queue-toolkit/core": ">=23.0.0",