@message-queue-toolkit/kafka 0.9.1 → 0.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. package/README.md +2 -1
  2. package/dist/AbstractKafkaConsumer.d.ts +1 -0
  3. package/dist/AbstractKafkaConsumer.js +27 -12
  4. package/dist/AbstractKafkaConsumer.js.map +1 -1
  5. package/dist/AbstractKafkaService.js +4 -1
  6. package/dist/AbstractKafkaService.js.map +1 -1
  7. package/dist/utils/KafkaMessageBatchStream.d.ts +59 -24
  8. package/dist/utils/KafkaMessageBatchStream.js +95 -62
  9. package/dist/utils/KafkaMessageBatchStream.js.map +1 -1
  10. package/package.json +2 -2
package/README.md CHANGED
@@ -20,7 +20,8 @@ See [test consumer](test/consumer/PermissionConsumer.ts) for an example of imple
20
20
 
21
21
  ## Batch Processing
22
22
 
23
- Kafka supports batch processing for improved throughput. To enable it, set `batchProcessingEnabled` to `true` and configure `batchProcessingOptions`.
23
+ Kafka supports batch processing for improved throughput. To enable it, set `batchProcessingEnabled` to `true` and
24
+ configure `batchProcessingOptions`.
24
25
 
25
26
  When batch processing is enabled, message handlers receive an array of messages instead of a single message.
26
27
 
package/dist/AbstractKafkaConsumer.d.ts CHANGED
@@ -33,6 +33,7 @@ export declare abstract class AbstractKafkaConsumer<TopicsConfig extends TopicCo
33
33
  get isActive(): boolean;
34
34
  init(): Promise<void>;
35
35
  private handleSyncStream;
36
+ private handleSyncStreamBatch;
36
37
  close(): Promise<void>;
37
38
  private resolveHandler;
38
39
  private consume;
package/dist/AbstractKafkaConsumer.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import { randomUUID } from 'node:crypto';
2
+ import { pipeline } from 'node:stream/promises';
2
3
  import { setTimeout } from 'node:timers/promises';
3
4
  import { InternalError, stringValueSerializer, } from '@lokalise/node-core';
4
5
  import { Consumer, ProtocolError, ResponseError, stringDeserializer, } from '@platformatic/kafka';
@@ -95,13 +96,18 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
95
96
  heartbeatInterval: consumeOptions.heartbeatInterval,
96
97
  });
97
98
  this.consumerStream = await this.consumer.consume({ ...consumeOptions, topics });
98
- this.consumerStream.on('error', (error) => this.handlerError(error));
99
99
  if (this.options.batchProcessingEnabled && this.options.batchProcessingOptions) {
100
- this.messageBatchStream = new KafkaMessageBatchStream((batch) => this.consume(batch.topic, batch.messages).catch((error) => this.handlerError(error)), this.options.batchProcessingOptions);
101
- this.consumerStream.pipe(this.messageBatchStream);
100
+ this.messageBatchStream = new KafkaMessageBatchStream({
101
+ batchSize: this.options.batchProcessingOptions.batchSize,
102
+ timeoutMilliseconds: this.options.batchProcessingOptions.timeoutMilliseconds,
103
+ readableHighWaterMark: this.options.batchProcessingOptions.readableHighWaterMark,
104
+ });
105
+ // Use pipeline for better error handling and backpressure management.
106
+ // pipeline() internally listens for errors on all streams
107
+ pipeline(this.consumerStream, this.messageBatchStream).catch((error) => this.handlerError(error));
102
108
  }
103
109
  else {
104
- this.handleSyncStream(this.consumerStream).catch((error) => this.handlerError(error));
110
+ this.consumerStream.on('error', (error) => this.handlerError(error));
105
111
  }
106
112
  }
107
113
  catch (error) {
@@ -111,12 +117,23 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
111
117
  cause: error,
112
118
  });
113
119
  }
120
+ if (this.messageBatchStream) {
121
+ this.handleSyncStreamBatch(this.messageBatchStream).catch((error) => this.handlerError(error));
122
+ }
123
+ else {
124
+ this.handleSyncStream(this.consumerStream).catch((error) => this.handlerError(error));
125
+ }
114
126
  }
115
127
  async handleSyncStream(stream) {
116
128
  for await (const message of stream) {
117
129
  await this.consume(message.topic, message);
118
130
  }
119
131
  }
132
+ async handleSyncStreamBatch(stream) {
133
+ for await (const messageBatch of stream) {
134
+ await this.consume(messageBatch[0].topic, messageBatch);
135
+ }
136
+ }
120
137
  async close() {
121
138
  if (!this.consumerStream && !this.messageBatchStream) {
122
139
  // Leaving the group in case consumer joined but streams were not created
@@ -210,12 +227,14 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
210
227
  async tryToConsume(topic, messageOrBatch, handler, requestContext) {
211
228
  try {
212
229
  const isBatch = Array.isArray(messageOrBatch);
230
+ /* v8 ignore start */
213
231
  if (this.options.batchProcessingEnabled && !isBatch) {
214
232
  throw new Error('Batch processing is enabled, but a single message was passed to the handler');
215
233
  }
216
234
  if (!this.options.batchProcessingEnabled && isBatch) {
217
235
  throw new Error('Batch processing is disabled, but a batch of messages was passed to the handler');
218
236
  }
237
+ /* v8 ignore stop */
219
238
  await handler(
220
239
  // We need casting to match message type with handler type - it is safe as we verify the type above
221
240
  messageOrBatch, this.executionContext, requestContext);
@@ -225,10 +244,7 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
225
244
  const errorContext = Array.isArray(messageOrBatch)
226
245
  ? { batchSize: messageOrBatch.length }
227
246
  : { message: stringValueSerializer(messageOrBatch.value) };
228
- this.handlerError(error, {
229
- topic,
230
- ...errorContext,
231
- });
247
+ this.handlerError(error, { topic, ...errorContext });
232
248
  }
233
249
  return { status: 'error', errorReason: 'handlerError' };
234
250
  }
@@ -267,7 +283,7 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
267
283
  this.logger.debug(logDetails, 'Message commit failed');
268
284
  if (error instanceof ResponseError)
269
285
  return this.handleResponseErrorOnCommit(error);
270
- throw error;
286
+ this.handlerError(error);
271
287
  }
272
288
  }
273
289
  handleResponseErrorOnCommit(responseError) {
@@ -276,7 +292,7 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
276
292
  if (error instanceof ProtocolError &&
277
293
  error.apiCode &&
278
294
  commitErrorCodesToIgnore.has(error.apiCode)) {
279
- this.logger.error({
295
+ this.logger.warn({
280
296
  apiCode: error.apiCode,
281
297
  apiId: error.apiId,
282
298
  responseErrorMessage: responseError.message,
@@ -285,8 +301,7 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
285
301
  }, `Failed to commit message: ${error.message}`);
286
302
  }
287
303
  else {
288
- // If error is not recognized, rethrow it
289
- throw responseError;
304
+ this.handlerError(error);
290
305
  }
291
306
  }
292
307
  }
package/dist/AbstractKafkaConsumer.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"AbstractKafkaConsumer.js","sourceRoot":"","sources":["../lib/AbstractKafkaConsumer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EACL,aAAa,EACb,qBAAqB,GAEtB,MAAM,qBAAqB,CAAA;AAK5B,OAAO,EAEL,QAAQ,EAGR,aAAa,EACb,aAAa,EACb,kBAAkB,GACnB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACL,oBAAoB,GAGrB,MAAM,2BAA2B,CAAA;AAYlC,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAA;AACpG,OAAO,EAEL,uBAAuB,GACxB,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAA;AAkCtE,MAAM,wBAAwB,GAAG,IAAI,GAAG,CAAC;IACvC,kBAAkB;IAClB,iBAAiB;IACjB,qBAAqB;CACtB,CAAC,CAAA;AAEF;;;GAGG;AACH,MAAM,qBAAqB,GAAG,CAAC,CAAA;AAE/B,MAAM,OAAgB,qBAIpB,SAAQ,oBAGT;IACkB,QAAQ,CAA0C;IAC3D,cAAc,CAAiD;IAC/D,kBAAkB,CAEzB;IAEgB,+BAA+B,CAAiC;IAChE,gBAAgB,CAAkB;IAEnD,YACE,YAAuC,EACvC,OAAqF,EACrF,gBAAkC;QAElC,KAAK,CAAC,YAAY,EAAE,OAAO,CAAC,CAAA;QAE5B,IAAI,CAAC,+BAA+B,GAAG,YAAY,CAAC,+BAA+B,CAAA;QACnF,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAA;QAExC,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC;YAC3B,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK;YACrB,GAAG,IAAI,CAAC,OAAO;YACf,UAAU,EAAE,KAAK,EAAE,4BAA4B;YAC/C,aAAa,EAAE;gBACb,GAAG,EAAE,kBAAkB;gBACvB,KAAK,EAAE,oBAAoB;gBAC3B,SAAS,EAAE,kBAAkB;gBAC7B,WAAW,EAAE,kBAAkB;aAChC;SACF,CAAC,CAAA;QAEF,MAAM,UAAU,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAA;QACnF,qBAAqB;QACrB,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,EAAE,CAC5C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,6BAA6B,CAAC,CAC7D,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE,CAC7C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,kDAAkD,CAAC,CAClF,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,sBAAsB,EAAE,CAAC,CAAC,EAAE,EAAE,CAC7C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,+BAA+B,CAAC,CAC/D,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,0BAA0B,EAAE,CAAC,CAAC,EAAE,EAAE,CACjD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,sBAAsB,CAAC,CACtD,CAAA;QACD,oBAAoB;IACtB,CAAC;IAED;;OAEG;IACH,IAAI,WAAW;QACb,uDAAuD;QACvD,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO,KAAK,CAAA;QAClE,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAA;YAClC,qBAAqB;QACvB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,yEAAyE;YACzE,OAAO,KAAK,CAAA;QACd,CAAC;QACD,oBAAoB;IACtB,CAAC;IAED;;;OAGG;IACH,IAAI,QAAQ;QACV,uDAAuD;QACvD,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO,KAAK,CAAA;QAClE,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAA;YAC/B,qBAAqB;QACvB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,yEAAyE;YACzE,OAAO,KAAK,CAAA;QACd,CAAC;QACD,oBAAoB;IACtB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;QACjD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QACjD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QAE9E,IAAI,CAAC;YACH,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC,OAAO,CAAA,CAAC,8CAA8C;YAEtG,0IAA0I;YAC1I,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC5B,cAAc,EAAE,cAAc,CAAC,cAAc;gBAC7C,gBAAgB,EAAE,cAAc,CAAC,gBAAgB;gBACjD,iBAAiB,EAAE,cAAc,CAAC,iBAAiB;aACpD,CAAC,CAAA;YAEF,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,GAAG,cAAc,EAAE,MAAM,EAAE,CAAC,CAAA;YAChF,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;YAEpE,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;gBAC/E,IAAI,CAAC,kBAAkB,GAAG,IAAI,uBAAuB,CAGnD,CAAC,KAAK,EAAE,EAAE,CACR,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EACtF,IAAI,CAAC,OAAO,CAAC,sBAAsB,CACpC,CAAA;gBACD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAA;YACnD,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;YACvF,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,aAAa,CAAC;gBACtB,OAAO,EAAE,sBAAsB;gBAC/B,SAAS,EAAE,2BAA2B;gBACtC,KAAK,EAAE,KAAK;aACb,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,MAAsD;QAEtD,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;YACnC,MAAM,IAAI,CAAC,OAAO,CAChB,OAAO,CAAC,KAAK,EACb,OAAoE,CACrE,CAAA;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACrD,yEAAyE;YACzE,IAAI,IAAI,CAAC,QAAQ;gBAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAA;YAC7C,OAAM;QACR,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAA;YACjC,IAAI,CAAC,cAAc,GAAG,SAAS,CAAA;QACjC,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAA;YACrE,IAAI,CAAC,kBAAkB,GAAG,SAAS,CAAA;QACrC,CAAC;QAED,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;IAC7B,CAAC;IAEO,cAAc,CAAC,KAAoC;QACzD,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IACrC,CAAC;IAEO,KAAK,CAAC,OAAO,CACnB,KAAa,EACb,cAAoE;QAEpE,MAAM,+BAA+B,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAClD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,sBAAsB,CAAC,CAAA;QAEnF,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;QAEhD,iFAAiF;QACjF,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;QAEtD,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CACtC,aAAa,EACb,cAAc,EACd,+BAA+B,CAChC,CAAA;QAED,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,+BAA+B,CAAC,CAAA;YAC5F,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;QACpC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,kBAAkB,EAAE,aAAa,CAAC,MAAM,EAAE,EAClF,sCAAsC,CACvC,CAAA;QACH,CAAC;QAED,kFAAkF;QAClF,MAAM,YAAY,GAAG,aAAa,CAAC,CAAC,CAAE,CAAA;QACtC,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAA;QAE3D,MAAM,aAAa,GAAG,UAAU,EAAE,CAAA;QAClC,IAAI,CAAC,+BAA+B,EAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,EAAE,aAAa,CAAC,CAAA;QAE5F,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,uBAAuB,CACzD,KAAK;QACL,0EAA0E;QAC1E,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,EAClE,aAAa,CAAC,OAAO,EACrB,cAAc,CACf,CAAA;QAED,IAAI,CAAC,uBAAuB,CAAC,aAAa,EAAE,gBAAgB,EAAE,+BAA+B,CAAC,CAAA;QAE9F,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,aAAa,CAAC,CAAA;QAEzD,+EAA+E;QAC/E,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;IACpC,CAAC;IAEO,aAAa,CACnB,aAIC,EACD,cAAoE,EACpE,+BAAuC;QAEvC,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAA;QAEzF,MAAM,aAAa,GAAgE,EAAE,CAAA;QAErF,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,yEAAyE;YACzE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACnB,SAAQ;YACV,CAAC;YAED,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAEjE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;gBACzB,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,EAAE;oBACnC,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC;iBAC9C,CAAC,CAAA;gBACF,IAAI,CAAC,sBAAsB,CAAC;oBAC1B,OAAO,EAAE,OAAO;oBAChB,gBAAgB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE;oBACpE,+BAA+B;iBAChC,CAAC,CAAA;gBACF,SAAQ;YACV,CAAC;YAED,aAAa,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,IAAI,EAAE,CAAC,CAAA;QAC7D,CAAC;QAED,OAAO,aAAa,CAAA;IACtB,CAAC;IAEO,KAAK,CAAC,uBAAuB,CACnC,KAAa,EACb,cAA4C,EAC5C,OAA6E,EAC7E,cAA8B;QAE9B,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,gBAAgB,GAA4B;YAC9C,MAAM,EAAE,OAAO;YACf,WAAW,EAAE,cAAc;SAC5B,CAAA;QACD,GAAG,CAAC;YACF,qCAAqC;YACrC,IAAI,OAAO,GAAG,CAAC;gBAAE,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAA;YAE3D,gBAAgB,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,cAAc,CAAC,CAAA;YAC1F,IAAI,gBAAgB,CAAC,MAAM,KAAK,UAAU;gBAAE,MAAK;YAEjD,OAAO,EAAE,CAAA;QACX,CAAC,QAAQ,OAAO,GAAG,qBAAqB,EAAC;QAEzC,OAAO,gBAAgB,CAAA;IACzB,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,KAAa,EACb,cAA4C,EAC5C,OAA6E,EAC7E,cAA8B;QAE9B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;YAC7C,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,CAAC,OAAO,EAAE,CAAC;gBACpD,MAAM,IAAI,KAAK,CACb,6EAA6E,CAC9E,CAAA;YACH,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,OAAO,EAAE,CAAC;gBACpD,MAAM,IAAI,KAAK,CACb,iFAAiF,CAClF,CAAA;YACH,CAAC;YAED,MAAM,OAAO;YACX,mGAAmG;YACnG,cAEuC,EACvC,IAAI,CAAC,gBAAgB,EACrB,cAAc,CACf,CAAA;YACD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAA;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC;gBAChD,CAAC,CAAC,EAAE,SAAS,EAAE,cAAc,CAAC,MAAM,EAAE;gBACtC,CAAC,CAAC,EAAE,OAAO,EAAE,qBAAqB,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,CAAA;YAC5D,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE;gBACvB,KAAK;gBACL,GAAG,YAAY;aAChB,CAAC,CAAA;QACJ,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,CAAA;IACzD,CAAC;IAEO,uBAAuB,CAC7B,QAA0C,EAC1C,gBAAyC,EACzC,+BAAuC;QAEvC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,sBAAsB,CAAC;gBAC1B,OAAO;gBACP,gBAAgB;gBAChB,+BAA+B;aAChC,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,cAAoE;QACjF,IAAI,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;YAClC,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;YAEzD,wEAAwE;YACxE,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAA;QACvE,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,CAAA;QAC3C,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAkE;QAC5F,MAAM,UAAU,GAAG;YACjB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,SAAS,EAAE,OAAO,CAAC,SAAS;SAC7B,CAAA;QACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,0BAA0B,CAAC,CAAA;QAEzD,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,MAAM,EAAE,CAAA;YACtB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,gCAAgC,CAAC,CAAA;QACjE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAA;YACtD,IAAI,KAAK,YAAY,aAAa;gBAAE,OAAO,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAA;YAClF,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAEO,2BAA2B,CAAC,aAA4B;QAC9D,kFAAkF;QAClF,KAAK,MAAM,KAAK,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YACzC,IACE,KAAK,YAAY,aAAa;gBAC9B,KAAK,CAAC,OAAO;gBACb,wBAAwB,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAC3C,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,KAAK,CACf;oBACE,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,oBAAoB,EAAE,aAAa,CAAC,OAAO;oBAC3C,oBAAoB,EAAE,KAAK,CAAC,OAAO;oBACnC,KAAK,EAAE,aAAa;iBACrB,EACD,6BAA6B,KAAK,CAAC,OAAO,EAAE,CAC7C,CAAA;YACH,CAAC;iBAAM,CAAC;gBACN,yCAAyC;gBACzC,MAAM,aAAa,CAAA;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IAEO,oBAAoB,CAAC,KAAa;QACxC,MAAM,mBAAmB,GAAG,SAAS,IAAI,CAAC,WAAW,CAAC,IAAI,IAAI,KAAK,EAAE,CAAA;QACrE,OAAO,IAAI,CAAC,OAAO,CAAC,sBAAsB;YACxC,CAAC,CAAC,GAAG,mBAAmB,QAAQ;YAChC,CAAC,CAAC,mBAAmB,CAAA;IACzB,CAAC;IAEO,iBAAiB,CACvB,OAAkE;QAElE,IAAI,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC,CAAA;QACnE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,KAAK,GAAG,UAAU,EAAE,CAAA;QAE7D,OAAO;YACL,KAAK;YACL,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;gBACxB,cAAc,EAAE,KAAK;gBACrB,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;gBAC7B,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,UAAU,EAAE,OAAO,CAAC,GAAG;aACxB,CAAC;SACH,CAAA;IACH,CAAC;CACF"}
1
+ {"version":3,"file":"AbstractKafkaConsumer.js","sourceRoot":"","sources":["../lib/AbstractKafkaConsumer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EACL,aAAa,EACb,qBAAqB,GAEtB,MAAM,qBAAqB,CAAA;AAK5B,OAAO,EAEL,QAAQ,EAGR,aAAa,EACb,aAAa,EACb,kBAAkB,GACnB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACL,oBAAoB,GAGrB,MAAM,2BAA2B,CAAA;AAYlC,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAA;AACpG,OAAO,EAEL,uBAAuB,GACxB,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAA;AAkCtE,MAAM,wBAAwB,GAAG,IAAI,GAAG,CAAC;IACvC,kBAAkB;IAClB,iBAAiB;IACjB,qBAAqB;CACtB,CAAC,CAAA;AAEF;;;GAGG;AACH,MAAM,qBAAqB,GAAG,CAAC,CAAA;AAE/B,MAAM,OAAgB,qBAIpB,SAAQ,oBAGT;IACkB,QAAQ,CAA0C;IAC3D,cAAc,CAAiD;IAC/D,kBAAkB,CAEzB;IAEgB,+BAA+B,CAAiC;IAChE,gBAAgB,CAAkB;IAEnD,YACE,YAAuC,EACvC,OAAqF,EACrF,gBAAkC;QAElC,KAAK,CAAC,YAAY,EAAE,OAAO,CAAC,CAAA;QAE5B,IAAI,CAAC,+BAA+B,GAAG,YAAY,CAAC,+BAA+B,CAAA;QACnF,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAA;QAExC,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC;YAC3B,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK;YACrB,GAAG,IAAI,CAAC,OAAO;YACf,UAAU,EAAE,KAAK,EAAE,4BAA4B;YAC/C,aAAa,EAAE;gBACb,GAAG,EAAE,kBAAkB;gBACvB,KAAK,EAAE,oBAAoB;gBAC3B,SAAS,EAAE,kBAAkB;gBAC7B,WAAW,EAAE,kBAAkB;aAChC;SACF,CAAC,CAAA;QAEF,MAAM,UAAU,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAA;QACnF,qBAAqB;QACrB,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,EAAE,CAC5C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,6BAA6B,CAAC,CAC7D,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE,CAC7C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,kDAAkD,CAAC,CAClF,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,sBAAsB,EAAE,CAAC,CAAC,EAAE,EAAE,CAC7C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,+BAA+B,CAAC,CAC/D,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,0BAA0B,EAAE,CAAC,CAAC,EAAE,EAAE,CACjD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,sBAAsB,CAAC,CACtD,CAAA;QACD,oBAAoB;IACtB,CAAC;IAED;;OAEG;IACH,IAAI,WAAW;QACb,uDAAuD;QACvD,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO,KAAK,CAAA;QAClE,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAA;YAClC,qBAAqB;QACvB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,yEAAyE;YACzE,OAAO,KAAK,CAAA;QACd,CAAC;QACD,oBAAoB;IACtB,CAAC;IAED;;;OAGG;IACH,IAAI,QAAQ;QACV,uDAAuD;QACvD,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO,KAAK,CAAA;QAClE,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAA;YAC/B,qBAAqB;QACvB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,yEAAyE;YACzE,OAAO,KAAK,CAAA;QACd,CAAC;QACD,oBAAoB;IACtB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;QACjD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QACjD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QAE9E,IAAI,CAAC;YACH,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC,OAAO,CAAA,CAAC,8CAA8C;YAEtG,0IAA0I;YAC1I,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC5B,cAAc,EAAE,cAAc,CAAC,cAAc;gBAC7C,gBAAgB,EAAE,cAAc,CAAC,gBAAgB;gBACjD,iBAAiB,EAAE,cAAc,CAAC,iBAAiB;aACpD,CAAC,CAAA;YAEF,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,GAAG,cAAc,EAAE,MAAM,EAAE,CAAC,CAAA;YAEhF,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;gBAC/E,IAAI,CAAC,kBAAkB,GAAG,IAAI,uBAAuB,CAEnD;oBACA,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,SAAS;oBACxD,mBAAmB,EAAE,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,mBAAmB;oBAC5E,qBAAqB,EAAE,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,qBAAqB;iBACjF,CAAC,CAAA;gBAEF,sEAAsE;gBACtE,0DAA0D;gBAC1D,QAAQ,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CACrE,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CACzB,CAAA;YACH,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;YACtE,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,aAAa,CAAC;gBACtB,OAAO,EAAE,sBAAsB;gBAC/B,SAAS,EAAE,2BAA2B;gBACtC,KAAK,EAAE,KAAK;aACb,CAAC,CAAA;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;QAChG,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;QACvF,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,MAAsD;QAEtD,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;YACnC,MAAM,IAAI,CAAC,OAAO,CAChB,OAAO,CAAC,KAAK,EACb,OAAoE,CACrE,CAAA;QACH,CAAC;IACH,CAAC;IACO,KAAK,CAAC,qBAAqB,CACjC,MAA0F;QAE1F,IAAI,KAAK,EAAE,MAAM,YAAY,IAAI,MAAM,EAAE,CAAC;YACxC,MAAM,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,YAAY,CAAC,CAAA;QACzD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACrD,yEAAyE;YACzE,IAAI,IAAI,CAAC,QAAQ;gBAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAA;YAC7C,OAAM;QACR,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAA;YACjC,IAAI,CAAC,cAAc,GAAG,SAAS,CAAA;QACjC,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAA;YACrE,IAAI,CAAC,kBAAkB,GAAG,SAAS,CAAA;QACrC,CAAC;QAED,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;IAC7B,CAAC;IAEO,cAAc,CAAC,KAAoC;QACzD,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IACrC,CAAC;IAEO,KAAK,CAAC,OAAO,CACnB,KAAa,EACb,cAAoE;QAEpE,MAAM,+BAA+B,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAClD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,sBAAsB,CAAC,CAAA;QAEnF,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;QAEhD,iFAAiF;QACjF,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;QAEtD,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CACtC,aAAa,EACb,cAAc,EACd,+BAA+B,CAChC,CAAA;QAED,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,+BAA+B,CAAC,CAAA;YAC5F,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;QACpC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,kBAAkB,EAAE,aAAa,CAAC,MAAM,EAAE,EAClF,sCAAsC,CACvC,CAAA;QACH,CAAC;QAED,kFAAkF;QAClF,MAAM,YAAY,GAAG,aAAa,CAAC,CAAC,CAAE,CAAA;QACtC,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAA;QAE3D,MAAM,aAAa,GAAG,UAAU,EAAE,CAAA;QAClC,IAAI,CAAC,+BAA+B,EAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,EAAE,aAAa,CAAC,CAAA;QAE5F,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,uBAAuB,CACzD,KAAK;QACL,0EAA0E;QAC1E,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,EAClE,aAAa,CAAC,OAAO,EACrB,cAAc,CACf,CAAA;QAED,IAAI,CAAC,uBAAuB,CAAC,aAAa,EAAE,gBAAgB,EAAE,+BAA+B,CAAC,CAAA;QAE9F,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,aAAa,CAAC,CAAA;QAEzD,+EAA+E;QAC/E,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;IACpC,CAAC;IAEO,aAAa,CACnB,aAIC,EACD,cAAoE,EACpE,+BAAuC;QAEvC,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAA;QAEzF,MAAM,aAAa,GAAgE,EAAE,CAAA;QAErF,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,yEAAyE;YACzE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACnB,SAAQ;YACV,CAAC;YAED,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAEjE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;gBACzB,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,EAAE;oBACnC,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC;iBAC9C,CAAC,CAAA;gBACF,IAAI,CAAC,sBAAsB,CAAC;oBAC1B,OAAO,EAAE,OAAO;oBAChB,gBAAgB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE;oBACpE,+BAA+B;iBAChC,CAAC,CAAA;gBACF,SAAQ;YACV,CAAC;YAED,aAAa,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,IAAI,EAAE,CAAC,CAAA;QAC7D,CAAC;QAED,OAAO,aAAa,CAAA;IACtB,CAAC;IAEO,KAAK,CAAC,uBAAuB,CACnC,KAAa,EACb,cAA4C,EAC5C,OAA6E,EAC7E,cAA8B;QAE9B,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,gBAAgB,GAA4B;YAC9C,MAAM,EAAE,OAAO;YACf,WAAW,EAAE,cAAc;SAC5B,CAAA;QACD,GAAG,CAAC;YACF,qCAAqC;YACrC,IAAI,OAAO,GAAG,CAAC;gBAAE,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAA;YAE3D,gBAAgB,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,cAAc,CAAC,CAAA;YAC1F,IAAI,gBAAgB,CAAC,MAAM,KAAK,UAAU;gBAAE,MAAK;YAEjD,OAAO,EAAE,CAAA;QACX,CAAC,QAAQ,OAAO,GAAG,qBAAqB,EAAC;QAEzC,OAAO,gBAAgB,CAAA;IACzB,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,KAAa,EACb,cAA4C,EAC5C,OAA6E,EAC7E,cAA8B;QAE9B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;YAC7C,qBAAqB;YACrB,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,CAAC,OAAO,EAAE,CAAC;gBACpD,MAAM,IAAI,KAAK,CACb,6EAA6E,CAC9E,CAAA;YACH,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,OAAO,EAAE,CAAC;gBACpD,MAAM,IAAI,KAAK,CACb,iFAAiF,CAClF,CAAA;YACH,CAAC;YACD,oBAAoB;YAEpB,MAAM,OAAO;YACX,mGAAmG;YACnG,cAEuC,EACvC,IAAI,CAAC,gBAAgB,EACrB,cAAc,CACf,CAAA;YACD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAA;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC;gBAChD,CAAC,CAAC,EAAE,SAAS,EAAE,cAAc,CAAC,MAAM,EAAE;gBACtC,CAAC,CAAC,EAAE,OAAO,EAAE,qBAAqB,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,CAAA;YAC5D,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,GAAG,YAAY,EAAE,CAAC,CAAA;QACtD,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,CAAA;IACzD,CAAC;IAEO,uBAAuB,CAC7B,QAA0C,EAC1C,gBAAyC,EACzC,+BAAuC;QAEvC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,sBAAsB,CAAC;gBAC1B,OAAO;gBACP,gBAAgB;gBAChB,+BAA+B;aAChC,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,cAAoE;QACjF,IAAI,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;YAClC,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;YAEzD,wEAAwE;YACxE,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAA;QACvE,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,CAAA;QAC3C,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAkE;QAC5F,MAAM,UAAU,GAAG;YACjB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,SAAS,EAAE,OAAO,CAAC,SAAS;SAC7B,CAAA;QACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,0BAA0B,CAAC,CAAA;QAEzD,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,MAAM,EAAE,CAAA;YACtB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,gCAAgC,CAAC,CAAA;QACjE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAA;YACtD,IAAI,KAAK,YAAY,aAAa;gBAAE,OAAO,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAA;YAClF,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAA;QAC1B,CAAC;IACH,CAAC;IAEO,2BAA2B,CAAC,aAA4B;QAC9D,kFAAkF;QAClF,KAAK,MAAM,KAAK,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YACzC,IACE,KAAK,YAAY,aAAa;gBAC9B,KAAK,CAAC,OAAO;gBACb,wBAAwB,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAC3C,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,IAAI,CACd;oBACE,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,oBAAoB,EAAE,aAAa,CAAC,OAAO;oBAC3C,oBAAoB,EAAE,KAAK,CAAC,OAAO;oBACnC,KAAK,EAAE,aAAa;iBACrB,EACD,6BAA6B,KAAK,CAAC,OAAO,EAAE,CAC7C,CAAA;YACH,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAA;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAEO,oBAAoB,CAAC,KAAa;QACxC,MAAM,mBAAmB,GAAG,SAAS,IAAI,CAAC,WAAW,CAAC,IAAI,IAAI,KAAK,EAAE,CAAA;QACrE,OAAO,IAAI,CAAC,OAAO,CAAC,sBAAsB;YACxC,CAAC,CAAC,GAAG,mBAAmB,QAAQ;YAChC,CAAC,CAAC,mBAAmB,CAAA;IACzB,CAAC;IAEO,iBAAiB,CACvB,OAAkE;QAElE,IAAI,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC,CAAA;QACnE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,KAAK,GAAG,UAAU,EAAE,CAAA;QAE7D,OAAO;YACL,KAAK;YACL,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;gBACxB,cAAc,EAAE,KAAK;gBACrB,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;gBAC7B,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,UAAU,EAAE,OAAO,CAAC,GAAG;aACxB,CAAC;SACH,CAAA;IACH,CAAC;CACF"}
package/dist/AbstractKafkaService.js CHANGED
@@ -54,7 +54,10 @@ export class AbstractKafkaService {
54
54
  handlerError(error, context = {}) {
55
55
  this.logger.error({ ...resolveGlobalErrorLogObject(error), ...context });
56
56
  if (isError(error))
57
- this.errorReporter.report({ error, context });
57
+ this.errorReporter.report({
58
+ error,
59
+ context: context,
60
+ });
58
61
  }
59
62
  }
60
63
  //# sourceMappingURL=AbstractKafkaService.js.map
package/dist/AbstractKafkaService.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"AbstractKafkaService.js","sourceRoot":"","sources":["../lib/AbstractKafkaService.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,OAAO,EACP,2BAA2B,EAC3B,qBAAqB,GACtB,MAAM,qBAAqB,CAAA;AAE5B,OAAO,EAML,iBAAiB,EACjB,iBAAiB,GAClB,MAAM,6BAA6B,CAAA;AA4BpC,MAAM,OAAgB,oBAAoB;IAIrB,aAAa,CAAe;IAC5B,MAAM,CAAc;IACpB,qBAAqB,CAEvC;IAEkB,OAAO,CAA8C;IACrD,WAAW,CAAmD;IAEjF,YAAY,YAA+B,EAAE,OAAqB;QAChE,IAAI,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAA;QACjC,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC,aAAa,CAAA;QAC/C,IAAI,CAAC,qBAAqB,GAAG,YAAY,CAAC,qBAAqB,CAAA;QAC/D,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,IAAI,EAAE,CAAA;QAE7E,IAAI,CAAC,WAAW,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAA;IAC/C,CAAC;IAKD,IAAI,UAAU;QACZ,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC,WAAW,CAAA;QAE7C,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;IACH,CAAC;IAES,gBAAgB,CAAC,OAA6C;QACtE,mBAAmB;QACnB,OAAO,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAuB,CAAA;IACnE,CAAC;IAES,2BAA2B;QACnC,OAAO,IAAI,CAAC,OAAO,CAAC,oBAAoB,IAAI,cAAc,CAAA;IAC5D,CAAC;IAES,sBAAsB,CAAC,MAIhC;QACC,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,GAAG,MAAM,CAAA;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAEtD,gFAAgF;QAChF,IAAI,CAAC,WAAW,EAAE,mBAAmB,CACnC,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,EAAE,gBAAgB,EAAE,EAC5C,SAAS,EACT,iBAAiB,CAClB,CAAA;QAED,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,CACf;gBACE,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC;gBAC7C,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,gBAAgB;gBAChB,SAAS;aACV,EACD,+BAA+B,SAAS,EAAE,CAC3C,CAAA;QACH,CAAC;QAED,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC/B,IAAI,CAAC,qBAAqB,CAAC,wBAAwB,CAAC;gBAClD,OAAO,EAAE,OAAO,CAAC,KAAK;gBACtB,gBAAgB;gBAChB,SAAS,EAAE,OAAO,CAAC,KAAK;gBACxB,SAAS,EAAE,SAAS,IAAI,SAAS;gBACjC,WAAW,EAAE,SAAS;gBACtB,gBAAgB,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;gBAC3E,+BAA+B,EAAE,MAAM,CAAC,+BAA+B;gBACvE,6BAA6B,EAAE,IAAI,CAAC,GAAG,EAAE;aAC1C,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAES,YAAY,CAAC,KAAc,EAAE,UAAmC,EAAE;QAC1E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,2BAA2B,CAAC,KAAK,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC,CAAA;QACxE,IAAI,OAAO,CAAC,KAAK,CAAC;YAAE,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAA;IACnE,CAAC;CACF"}
1
+ {"version":3,"file":"AbstractKafkaService.js","sourceRoot":"","sources":["../lib/AbstractKafkaService.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,OAAO,EACP,2BAA2B,EAC3B,qBAAqB,GACtB,MAAM,qBAAqB,CAAA;AAE5B,OAAO,EAML,iBAAiB,EACjB,iBAAiB,GAClB,MAAM,6BAA6B,CAAA;AA4BpC,MAAM,OAAgB,oBAAoB;IAIrB,aAAa,CAAe;IAC5B,MAAM,CAAc;IACpB,qBAAqB,CAEvC;IAEkB,OAAO,CAA8C;IACrD,WAAW,CAAmD;IAEjF,YAAY,YAA+B,EAAE,OAAqB;QAChE,IAAI,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAA;QACjC,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC,aAAa,CAAA;QAC/C,IAAI,CAAC,qBAAqB,GAAG,YAAY,CAAC,qBAAqB,CAAA;QAC/D,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,IAAI,EAAE,CAAA;QAE7E,IAAI,CAAC,WAAW,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAA;IAC/C,CAAC;IAKD,IAAI,UAAU;QACZ,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC,WAAW,CAAA;QAE7C,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;IACH,CAAC;IAES,gBAAgB,CAAC,OAA6C;QACtE,mBAAmB;QACnB,OAAO,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAuB,CAAA;IACnE,CAAC;IAES,2BAA2B;QACnC,OAAO,IAAI,CAAC,OAAO,CAAC,oBAAoB,IAAI,cAAc,CAAA;IAC5D,CAAC;IAES,sBAAsB,CAAC,MAIhC;QACC,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,GAAG,MAAM,CAAA;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAEtD,gFAAgF;QAChF,IAAI,CAAC,WAAW,EAAE,mBAAmB,CACnC,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,EAAE,gBAAgB,EAAE,EAC5C,SAAS,EACT,iBAAiB,CAClB,CAAA;QAED,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,CACf;gBACE,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC;gBAC7C,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,gBAAgB;gBAChB,SAAS;aACV,EACD,+BAA+B,SAAS,EAAE,CAC3C,CAAA;QACH,CAAC;QAED,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC/B,IAAI,CAAC,qBAAqB,CAAC,wBAAwB,CAAC;gBAClD,OAAO,EAAE,OAAO,CAAC,KAAK;gBACtB,gBAAgB;gBAChB,SAAS,EAAE,OAAO,CAAC,KAAK;gBACxB,SAAS,EAAE,SAAS,IAAI,SAAS;gBACjC,WAAW,EAAE,SAAS;gBACtB,gBAAgB,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;gBAC3E,+BAA+B,EAAE,MAAM,CAAC,+BAA+B;gBACvE,6BAA6B,EAAE,IAAI,CAAC,GAAG,EAAE;aAC1C,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAES,YAAY,CAAC,KAAc,EAAE,UAAmC,EAAE;QAC1E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,2BAA2B,CAAC,KAAK,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC,CAAA;QACxE,IAAI,OAAO,CAAC,KAAK,CAAC;YAChB,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC;gBACxB,KAAK;gBACL,OAAO,EAAE,OAAO;aACjB,CAAC,CAAA;IACN,CAAC;CACF"}
package/dist/utils/KafkaMessageBatchStream.d.ts CHANGED
@@ -1,39 +1,74 @@
1
- import { Transform } from 'node:stream';
1
+ import { Duplex } from 'node:stream';
2
+ type CallbackFunction = (error?: Error | null) => void;
2
3
  type MessageWithTopicAndPartition = {
3
4
  topic: string;
4
5
  partition: number;
5
6
  };
7
+ /**
8
+ * Options for configuring the KafkaMessageBatchStream behavior.
9
+ */
6
10
  export type KafkaMessageBatchOptions = {
11
+ /** Maximum number of messages to accumulate across all partitions before flushing */
7
12
  batchSize: number;
13
+ /** Time in milliseconds to wait before flushing incomplete batches */
8
14
  timeoutMilliseconds: number;
15
+ /**
16
+ * Maximum number of topic-partition batches to buffer on the readable side before signaling backpressure.
17
+ * Each unit represents one array of messages belonging to the same topic-partition, produced per flush.
18
+ * A single flush may push multiple such arrays (one per distinct topic-partition in the accumulated batch).
19
+ * Defaults to Node.js object-mode default (16). Lower values trigger backpressure sooner,
20
+ * reducing downstream memory pressure at the cost of more frequent flow-control cycles.
21
+ */
22
+ readableHighWaterMark?: number;
9
23
  };
10
- export type MessageBatch<TMessage> = {
11
- topic: string;
12
- partition: number;
13
- messages: TMessage[];
14
- };
15
- export type OnMessageBatchCallback<TMessage> = (batch: MessageBatch<TMessage>) => Promise<void>;
16
24
  /**
17
- * Collects messages in batches based on provided batchSize and flushes them when messages amount or timeout is reached.
25
+ * Interface extending Duplex to provide strong typing for the 'data' event.
26
+ * The stream emits arrays of messages grouped by topic-partition.
27
+ */
28
+ export interface KafkaMessageBatchStream<TMessage extends MessageWithTopicAndPartition> extends Duplex {
29
+ on(event: string | symbol, listener: (...args: any[]) => void): this;
30
+ /** Listen for batches of messages from the same topic-partition */
31
+ on(event: 'data', listener: (chunk: TMessage[]) => void): this;
32
+ push(chunk: TMessage[] | null): boolean;
33
+ }
34
+ /**
35
+ * A Duplex stream that batches Kafka messages based on size and timeout constraints.
36
+ *
37
+ * Key features:
38
+ * - Accumulates messages across all partitions up to `batchSize` for true memory control
39
+ * - Groups messages by topic-partition when flushing
40
+ * - Implements backpressure: pauses input when downstream consumers are overwhelmed
41
+ * - Auto-flushes on timeout to prevent messages from waiting indefinitely
18
42
  *
19
- * This implementation uses Transform stream which properly handles backpressure by design.
20
- * When the downstream consumer is slow, the stream will automatically pause accepting new messages
21
- * until the consumer catches up, preventing memory leaks and OOM errors.
43
+ * @example
44
+ * ```typescript
45
+ * const batchStream = new KafkaMessageBatchStream({ batchSize: 100, timeoutMilliseconds: 1000 })
46
+ * batchStream.on('data', (batch) => {
47
+ * console.log(`Received ${batch.length} messages from ${batch[0].topic}:${batch[0].partition}`)
48
+ * })
49
+ * ```
22
50
  */
23
- export declare class KafkaMessageBatchStream<TMessage extends MessageWithTopicAndPartition> extends Transform {
24
- private readonly onBatch;
51
+ export declare class KafkaMessageBatchStream<TMessage extends MessageWithTopicAndPartition> extends Duplex {
25
52
  private readonly batchSize;
26
53
  private readonly timeout;
27
- private readonly currentBatchPerTopicPartition;
28
- private readonly batchTimeoutPerTopicPartition;
29
- private readonly timeoutProcessingPromises;
30
- constructor(onBatch: OnMessageBatchCallback<TMessage>, options: {
31
- batchSize: number;
32
- timeoutMilliseconds: number;
33
- });
34
- _transform(message: TMessage, _encoding: BufferEncoding, callback: () => void): Promise<void>;
35
- _flush(callback: () => void): Promise<void>;
36
- private flushAllBatches;
37
- private flushCurrentBatchMessages;
54
+ private messages;
55
+ private existingTimeout;
56
+ private pendingCallback;
57
+ private isBackPressured;
58
+ constructor(options: KafkaMessageBatchOptions);
59
+ /**
60
+ * Called when the downstream consumer is ready to receive more data.
61
+ * This is the backpressure release mechanism: we resume the writable side
62
+ * by calling the pending callback that was held during backpressure.
63
+ */
64
+ _read(): void;
65
+ /**
66
+ * Writes a message to the stream.
67
+ * Messages accumulate until batchSize is reached or timeout expires.
68
+ * Implements backpressure by holding the callback when downstream cannot consume.
69
+ */
70
+ _write(message: TMessage, _encoding: BufferEncoding, callback: CallbackFunction): void;
71
+ _final(callback: CallbackFunction): void;
72
+ private flushMessages;
38
73
  }
39
74
  export {};
package/dist/utils/KafkaMessageBatchStream.js CHANGED
@@ -1,82 +1,115 @@
1
- import { Transform } from 'node:stream';
1
+ import { Duplex } from 'node:stream';
2
2
  /**
3
- * Collects messages in batches based on provided batchSize and flushes them when messages amount or timeout is reached.
3
+ * A Duplex stream that batches Kafka messages based on size and timeout constraints.
4
4
  *
5
- * This implementation uses Transform stream which properly handles backpressure by design.
6
- * When the downstream consumer is slow, the stream will automatically pause accepting new messages
7
- * until the consumer catches up, preventing memory leaks and OOM errors.
5
+ * Key features:
6
+ * - Accumulates messages across all partitions up to `batchSize` for true memory control
7
+ * - Groups messages by topic-partition when flushing
8
+ * - Implements backpressure: pauses input when downstream consumers are overwhelmed
9
+ * - Auto-flushes on timeout to prevent messages from waiting indefinitely
10
+ *
11
+ * @example
12
+ * ```typescript
13
+ * const batchStream = new KafkaMessageBatchStream({ batchSize: 100, timeoutMilliseconds: 1000 })
14
+ * batchStream.on('data', (batch) => {
15
+ * console.log(`Received ${batch.length} messages from ${batch[0].topic}:${batch[0].partition}`)
16
+ * })
17
+ * ```
8
18
  */
9
- export class KafkaMessageBatchStream extends Transform {
10
- onBatch;
19
+ // biome-ignore lint/suspicious/noUnsafeDeclarationMerging: merging interface with class to add strong typing for 'data' event
20
+ export class KafkaMessageBatchStream extends Duplex {
11
21
  batchSize;
12
22
  timeout;
13
- currentBatchPerTopicPartition;
14
- batchTimeoutPerTopicPartition;
15
- timeoutProcessingPromises = new Map();
16
- constructor(onBatch, options) {
17
- super({ objectMode: true });
18
- this.onBatch = onBatch;
23
+ messages;
24
+ existingTimeout;
25
+ pendingCallback;
26
+ isBackPressured;
27
+ constructor(options) {
28
+ super({ objectMode: true, readableHighWaterMark: options.readableHighWaterMark });
19
29
  this.batchSize = options.batchSize;
20
30
  this.timeout = options.timeoutMilliseconds;
21
- this.currentBatchPerTopicPartition = {};
22
- this.batchTimeoutPerTopicPartition = {};
31
+ this.messages = [];
32
+ this.isBackPressured = false;
23
33
  }
24
- async _transform(message, _encoding, callback) {
25
- const key = getTopicPartitionKey(message.topic, message.partition);
26
- // Wait for all pending timeout flushes to complete to maintain backpressure
27
- if (this.timeoutProcessingPromises.size > 0) {
28
- // Capture a snapshot of current promises to avoid race conditions with new timeouts
29
- const promiseEntries = Array.from(this.timeoutProcessingPromises.entries());
30
- // Wait for all to complete and then clean up from the map
31
- await Promise.all(promiseEntries.map(([k, p]) => p.finally(() => this.timeoutProcessingPromises.delete(k))));
32
- }
33
- // Accumulate the message
34
- if (!this.currentBatchPerTopicPartition[key])
35
- this.currentBatchPerTopicPartition[key] = [];
36
- this.currentBatchPerTopicPartition[key].push(message);
37
- // Check if the batch is complete by size
38
- if (this.currentBatchPerTopicPartition[key].length >= this.batchSize) {
39
- await this.flushCurrentBatchMessages(message.topic, message.partition);
40
- callback();
34
+ /**
35
+ * Called when the downstream consumer is ready to receive more data.
36
+ * This is the backpressure release mechanism: we resume the writable side
37
+ * by calling the pending callback that was held during backpressure.
38
+ */
39
+ _read() {
40
+ this.isBackPressured = false;
41
+ if (!this.pendingCallback)
41
42
  return;
43
+ const cb = this.pendingCallback;
44
+ this.pendingCallback = undefined;
45
+ cb(); // Resume the writable side
46
+ }
47
+ /**
48
+ * Writes a message to the stream.
49
+ * Messages accumulate until batchSize is reached or timeout expires.
50
+ * Implements backpressure by holding the callback when downstream cannot consume.
51
+ */
52
+ _write(message, _encoding, callback) {
53
+ let canContinue = true;
54
+ try {
55
+ this.messages.push(message);
56
+ if (this.messages.length >= this.batchSize) {
57
+ // Batch is full, flush immediately
58
+ canContinue = this.flushMessages();
59
+ }
60
+ else {
61
+ // Start/continue the timeout for partial batches
62
+ // Using ??= ensures we only set one timeout at a time
63
+ this.existingTimeout ??= setTimeout(() => this.flushMessages(), this.timeout);
64
+ }
42
65
  }
43
- // Start timeout for this partition if not already started
44
- if (!this.batchTimeoutPerTopicPartition[key]) {
45
- this.batchTimeoutPerTopicPartition[key] = setTimeout(() => this.timeoutProcessingPromises.set(key, this.flushCurrentBatchMessages(message.topic, message.partition)), this.timeout);
66
+ finally {
67
+ // Backpressure handling: hold the callback if push() returned false
68
+ if (!canContinue)
69
+ this.pendingCallback = callback;
70
+ else
71
+ callback();
46
72
  }
47
- callback();
48
73
  }
49
- // Flush all remaining batches when stream is closing
50
- async _flush(callback) {
51
- await this.flushAllBatches();
74
+ _final(callback) {
75
+ // Clean timeout
76
+ clearTimeout(this.existingTimeout);
77
+ this.existingTimeout = undefined;
78
+ // If there are remaining messages -> skip them
79
+ // As they are not committed, the next consumer will process them
80
+ this.messages = [];
81
+ this.push(null);
52
82
  callback();
53
83
  }
54
- async flushAllBatches() {
55
- for (const key of Object.keys(this.currentBatchPerTopicPartition)) {
56
- const { topic, partition } = splitTopicPartitionKey(key);
57
- await this.flushCurrentBatchMessages(topic, partition);
84
+ flushMessages() {
85
+ clearTimeout(this.existingTimeout);
86
+ this.existingTimeout = undefined;
87
+ if (this.isBackPressured) {
88
+ this.existingTimeout = setTimeout(() => this.flushMessages(), this.timeout);
89
+ return false;
58
90
  }
59
- }
60
- async flushCurrentBatchMessages(topic, partition) {
61
- const key = getTopicPartitionKey(topic, partition);
62
- // Clear timeout
63
- if (this.batchTimeoutPerTopicPartition[key]) {
64
- clearTimeout(this.batchTimeoutPerTopicPartition[key]);
65
- this.batchTimeoutPerTopicPartition[key] = undefined;
91
+ // Extract all accumulated messages and clear the array
92
+ const messageBatch = this.messages.splice(0, this.messages.length);
93
+ // Group by topic-partition to maintain commit guarantees
94
+ const messagesByTopicPartition = {};
95
+ for (const message of messageBatch) {
96
+ const key = getTopicPartitionKey(message.topic, message.partition);
97
+ if (!messagesByTopicPartition[key])
98
+ messagesByTopicPartition[key] = [];
99
+ messagesByTopicPartition[key].push(message);
100
+ }
101
+ // Push each topic-partition batch and track backpressure.
102
+ // All batches must be pushed regardless: messages were already splice'd from the buffer,
103
+ // so breaking early would lose them. Once push() returns false, subsequent calls in the
104
+ // same tick also return false, so the last value correctly reflects backpressure.
105
+ let canContinue = true;
106
+ for (const messagesForKey of Object.values(messagesByTopicPartition)) {
107
+ canContinue = this.push(messagesForKey);
66
108
  }
67
- const messages = this.currentBatchPerTopicPartition[key] ?? [];
68
- // Push the batch downstream
69
- await this.onBatch({ topic, partition, messages });
70
- this.currentBatchPerTopicPartition[key] = [];
109
+ if (!canContinue)
110
+ this.isBackPressured = true;
111
+ return canContinue;
71
112
  }
72
113
  }
73
114
  const getTopicPartitionKey = (topic, partition) => `${topic}:${partition}`;
74
- const splitTopicPartitionKey = (key) => {
75
- const [topic, partition] = key.split(':');
76
- /* v8 ignore start */
77
- if (!topic || !partition)
78
- throw new Error('Invalid topic-partition key format');
79
- /* v8 ignore stop */
80
- return { topic, partition: Number.parseInt(partition, 10) };
81
- };
82
115
  //# sourceMappingURL=KafkaMessageBatchStream.js.map
package/dist/utils/KafkaMessageBatchStream.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"KafkaMessageBatchStream.js","sourceRoot":"","sources":["../../lib/utils/KafkaMessageBatchStream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAavC;;;;;;GAMG;AACH,MAAM,OAAO,uBAEX,SAAQ,SAAS;IACA,OAAO,CAAkC;IACzC,SAAS,CAAQ;IACjB,OAAO,CAAQ;IAEf,6BAA6B,CAA4B;IACzD,6BAA6B,CAA4C;IAEzE,yBAAyB,GAA+B,IAAI,GAAG,EAAE,CAAA;IAElF,YACE,OAAyC,EACzC,OAA2D;QAE3D,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAA;QAC3B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAA;QACtB,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAA;QAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,mBAAmB,CAAA;QAC1C,IAAI,CAAC,6BAA6B,GAAG,EAAE,CAAA;QACvC,IAAI,CAAC,6BAA6B,GAAG,EAAE,CAAA;IACzC,CAAC;IAEQ,KAAK,CAAC,UAAU,CAAC,OAAiB,EAAE,SAAyB,EAAE,QAAoB;QAC1F,MAAM,GAAG,GAAG,oBAAoB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,CAAA;QAElE,4EAA4E;QAC5E,IAAI,IAAI,CAAC,yBAAyB,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC5C,oFAAoF;YACpF,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,yBAAyB,CAAC,OAAO,EAAE,CAAC,CAAA;YAC3E,0DAA0D;YAC1D,MAAM,OAAO,CAAC,GAAG,CACf,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAC1F,CAAA;QACH,CAAC;QAED,yBAAyB;QACzB,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,GAAG,EAAE,CAAA;QAC1F,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAErD,yCAAyC;QACzC,IAAI,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrE,MAAM,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,CAAA;YACtE,QAAQ,EAAE,CAAA;YACV,OAAM;QACR,CAAC;QAED,0DAA0D;QAC1D,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,GAAG,UAAU,CAClD,GAAG,EAAE,CACH,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAChC,GAAG,EACH,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,CACjE,EACH,IAAI,CAAC,OAAO,CACb,CAAA;QACH,CAAC;QAED,QAAQ,EAAE,CAAA;IACZ,CAAC;IAED,qDAAqD;IAC5C,KAAK,CAAC,MAAM,CAAC,QAAoB;QACxC,MAAM,IAAI,CAAC,eAAe,EAAE,CAAA;QAC5B,QAAQ,EAAE,CAAA;IACZ,CAAC;IAEO,KAAK,CAAC,eAAe;QAC3B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,EAAE,CAAC;YAClE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,sBAAsB,CAAC,GAAG,CAAC,CAAA;YACxD,MAAM,IAAI,CAAC,yBAAyB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;QACxD,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,yBAAyB,CAAC,KAAa,EAAE,SAAiB;QACtE,MAAM,GAAG,GAAG,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;QAElD,gBAAgB;QAChB,IAAI,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5C,YAAY,CAAC,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,CAAC,CAAA;YACrD,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,GAAG,SAAS,CAAA;QACrD,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,IAAI,EAAE,CAAA;QAE9D,4BAA4B;QAC5B,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAA;QAClD,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,GAAG,EAAE,CAAA;IAC9C,CAAC;CACF;AAED,MAAM,oBAAoB,GAAG,CAAC,KAAa,EAAE,SAAiB,EAAU,EAAE,CAAC,GAAG,KAAK,IAAI,SAAS,EAAE,CAAA;AAClG,MAAM,sBAAsB,GAAG,CAAC,GAAW,EAAwC,EAAE;IACnF,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IACzC,qBAAqB;IACrB,IAAI,CAAC,KAAK,IAAI,CAAC,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;IAC/E,oBAAoB;IAEpB,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,EAAE,CAAA;AAC7D,CAAC,CAAA"}
1
+ {"version":3,"file":"KafkaMessageBatchStream.js","sourceRoot":"","sources":["../../lib/utils/KafkaMessageBatchStream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AAoCpC;;;;;;;;;;;;;;;;GAgBG;AACH,8HAA8H;AAC9H,MAAM,OAAO,uBAAuE,SAAQ,MAAM;IAC/E,SAAS,CAAQ;IACjB,OAAO,CAAQ;IAExB,QAAQ,CAAY;IACpB,eAAe,CAA4B;IAC3C,eAAe,CAA8B;IAC7C,eAAe,CAAS;IAEhC,YAAY,OAAiC;QAC3C,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,qBAAqB,EAAE,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAAA;QACjF,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAA;QAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,mBAAmB,CAAA;QAC1C,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAA;QAClB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAA;IAC9B,CAAC;IAED;;;;OAIG;IACM,KAAK;QACZ,IAAI,CAAC,eAAe,GAAG,KAAK,CAAA;QAC5B,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,OAAM;QAEjC,MAAM,EAAE,GAAG,IAAI,CAAC,eAAe,CAAA;QAC/B,IAAI,CAAC,eAAe,GAAG,SAAS,CAAA;QAChC,EAAE,EAAE,CAAA,CAAC,2BAA2B;IAClC,CAAC;IAED;;;;OAIG;IACM,MAAM,CAAC,OAAiB,EAAE,SAAyB,EAAE,QAA0B;QACtF,IAAI,WAAW,GAAG,IAAI,CAAA;QAEtB,IAAI,CAAC;YACH,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAE3B,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC3C,mCAAmC;gBACnC,WAAW,GAAG,IAAI,CAAC,aAAa,EAAE,CAAA;YACpC,CAAC;iBAAM,CAAC;gBACN,iDAAiD;gBACjD,sDAAsD;gBACtD,IAAI,CAAC,eAAe,KAAK,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;YAC/E,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,oEAAoE;YACpE,IAAI,CAAC,WAAW;gBAAE,IAAI,CAAC,eAAe,GAAG,QAAQ,CAAA;;gBAC5C,QAAQ,EAAE,CAAA;QACjB,CAAC;IACH,CAAC;IAEQ,MAAM,CAAC,QAA0B;QACxC,gBAAgB;QAChB,YAAY,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;QAClC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAA;QAChC,+CAA+C;QAC/C,iEAAiE;QACjE,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAA;QAClB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACf,QAAQ,EAAE,CAAA;IACZ,CAAC;IAEO,aAAa;QACnB,YAAY,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;QAClC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAA;QAEhC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;YAC3E,OAAO,KAAK,CAAA;QACd,CAAC;QAED,uDAAuD;QACvD,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;QAElE,yDAAyD;QACzD,MAAM,wBAAwB,GAA+B,EAAE,CAAA;QAC/D,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;YACnC,MAAM,GAAG,GAAG,oBAAoB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,CAAA;YAClE,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC;gBAAE,wBAAwB,CAAC,GAAG,CAAC,GAAG,EAAE,CAAA;YACtE,wBAAwB,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC7C,CAAC;QAED,0DAA0D;QAC1D,yFAAyF;QACzF,wFAAwF;QACxF,kFAAkF;QAClF,IAAI,WAAW,GAAG,IAAI,CAAA;QACtB,KAAK,MAAM,cAAc,IAAI,MAAM,CAAC,MAAM,CAAC,wBAAwB,CAAC,EAAE,CAAC;YACrE,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;QACzC,CAAC;QAED,IAAI,CAAC,WAAW;YAAE,IAAI,CAAC,eAAe,GAAG,IAAI,CAAA;QAE7C,OAAO,WAAW,CAAA;IACpB,CAAC;CACF;AAED,MAAM,oBAAoB,GAAG,CAAC,KAAa,EAAE,SAAiB,EAAU,EAAE,CAAC,GAAG,KAAK,IAAI,SAAS,EAAE,CAAA"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@message-queue-toolkit/kafka",
3
- "version": "0.9.1",
3
+ "version": "0.10.1",
4
4
  "engines": {
5
5
  "node": ">= 22.14.0"
6
6
  },
@@ -53,7 +53,7 @@
53
53
  "dependencies": {
54
54
  "@lokalise/node-core": "^14.2.0",
55
55
  "@lokalise/universal-ts-utils": "^4.5.1",
56
- "@platformatic/kafka": "^1.24.0"
56
+ "@platformatic/kafka": "^1.29.0"
57
57
  },
58
58
  "peerDependencies": {
59
59
  "@message-queue-toolkit/core": ">=23.0.0",