@message-queue-toolkit/kafka 0.9.1-beta.4 → 0.9.1-beta.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/dist/utils/KafkaMessageBatchStream.d.ts +7 -8
  2. package/dist/utils/KafkaMessageBatchStream.js +34 -17
  3. package/dist/utils/KafkaMessageBatchStream.js.map +1 -1
  4. package/package.json +1 -1
package/dist/utils/KafkaMessageBatchStream.d.ts CHANGED
@@ -1,8 +1,5 @@
1
- import { Transform, type TransformCallback } from 'node:stream';
2
- type MessageWithTopicAndPartition = {
3
- topic: string;
4
- partition: number;
5
- };
1
+ import { Duplex } from 'node:stream';
2
+ type CallbackFunction = (error?: Error | null) => void;
6
3
  export type KafkaMessageBatchOptions = {
7
4
  batchSize: number;
8
5
  timeoutMilliseconds: number;
@@ -19,18 +16,20 @@ export type MessageBatch<TMessage> = {
19
16
  * When the downstream consumer is slow, the stream will automatically pause accepting new messages
20
17
  * until the consumer catches up, preventing memory leaks and OOM errors.
21
18
  */
22
- export declare class KafkaMessageBatchStream<TMessage extends MessageWithTopicAndPartition> extends Transform {
19
+ export declare class KafkaMessageBatchStream<TMessage> extends Duplex {
23
20
  private readonly batchSize;
24
21
  private readonly timeout;
25
22
  private readonly messages;
26
23
  private existingTimeout;
24
+ private pendingCallback;
27
25
  private isFlushing;
28
26
  constructor(options: {
29
27
  batchSize: number;
30
28
  timeoutMilliseconds: number;
31
29
  });
32
- _transform(message: TMessage, _encoding: BufferEncoding, callback: TransformCallback): void;
33
- _flush(callback: () => void): void;
30
+ _read(): void;
31
+ _write(message: TMessage, _encoding: BufferEncoding, callback: CallbackFunction): void;
32
+ _final(callback: CallbackFunction): void;
34
33
  private flushMessages;
35
34
  push(chunk: TMessage[] | null, encoding?: BufferEncoding): boolean;
36
35
  }
package/dist/utils/KafkaMessageBatchStream.js CHANGED
@@ -1,4 +1,4 @@
1
- import { Transform } from 'node:stream';
1
+ import { Duplex } from 'node:stream';
2
2
  /**
3
3
  * Collects messages in batches based on provided batchSize and flushes them when messages amount or timeout is reached.
4
4
  *
@@ -6,39 +6,55 @@ import { Transform } from 'node:stream';
6
6
  * When the downstream consumer is slow, the stream will automatically pause accepting new messages
7
7
  * until the consumer catches up, preventing memory leaks and OOM errors.
8
8
  */
9
- export class KafkaMessageBatchStream extends Transform {
9
+ export class KafkaMessageBatchStream extends Duplex {
10
10
  batchSize;
11
11
  timeout;
12
12
  messages;
13
13
  existingTimeout;
14
+ pendingCallback;
14
15
  isFlushing = false;
15
16
  constructor(options) {
16
17
  super({ objectMode: true });
17
18
  this.batchSize = options.batchSize;
18
19
  this.timeout = options.timeoutMilliseconds;
19
20
  this.messages = [];
21
+ // Listen to 'drain' event to resume after backpressure
22
+ this.on('drain', () => {
23
+ if (!this.pendingCallback)
24
+ return;
25
+ const cb = this.pendingCallback;
26
+ this.pendingCallback = undefined;
27
+ cb();
28
+ });
29
+ }
30
+ _read() {
31
+ // No-op, as we push data when we have a full batch or timeout
20
32
  }
21
- _transform(message, _encoding, callback) {
33
+ _write(message, _encoding, callback) {
34
+ // biome-ignore lint/correctness/noUnusedVariables: All good in this version
35
+ let canContinue = true;
22
36
  try {
23
37
  this.messages.push(message);
24
- // Check if the batch is complete by size
25
38
  if (this.messages.length >= this.batchSize) {
26
- this.flushMessages();
27
- return;
39
+ canContinue = this.flushMessages();
28
40
  }
29
- else if (!this.existingTimeout) {
30
- // Start timeout if not already started
31
- this.existingTimeout = setTimeout(() => this.flushMessages(), this.timeout);
41
+ else {
42
+ // If backpressure happens, we don't have a callback to hold
43
+ // The next _write will handle backpressure
44
+ // TODO: check if we can handle this.
45
+ this.existingTimeout ??= setTimeout(() => this.flushMessages(), this.timeout);
32
46
  }
33
47
  }
34
48
  finally {
35
- callback(null);
49
+ // TODO: to be enabled in next beta version
50
+ // if (!canContinue) this.pendingCallback = callback
51
+ // else callback()
52
+ callback();
36
53
  }
37
54
  }
38
- // Flush all remaining batches when stream is closing
39
- _flush(callback) {
55
+ _final(callback) {
40
56
  this.flushMessages();
41
- this.push(null); // end of stream
57
+ this.push(null); // End readable side
42
58
  callback();
43
59
  }
44
60
  flushMessages() {
@@ -47,13 +63,14 @@ export class KafkaMessageBatchStream extends Transform {
47
63
  this.existingTimeout = undefined;
48
64
  }
49
65
  if (this.isFlushing)
50
- return;
66
+ return true;
51
67
  this.isFlushing = true;
52
68
  const messages = this.messages.splice(0, this.messages.length);
53
- if (messages.length) {
54
- this.push(messages);
55
- }
69
+ let canContinue = true;
70
+ if (messages.length)
71
+ canContinue = this.push(messages);
56
72
  this.isFlushing = false;
73
+ return canContinue;
57
74
  }
58
75
  push(chunk, encoding) {
59
76
  return super.push(chunk, encoding);
package/dist/utils/KafkaMessageBatchStream.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"KafkaMessageBatchStream.js","sourceRoot":"","sources":["../../lib/utils/KafkaMessageBatchStream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAA0B,MAAM,aAAa,CAAA;AAY/D;;;;;;GAMG;AACH,MAAM,OAAO,uBAEX,SAAQ,SAAS;IACA,SAAS,CAAQ;IACjB,OAAO,CAAQ;IAEf,QAAQ,CAAY;IAC7B,eAAe,CAA4B;IAC3C,UAAU,GAAY,KAAK,CAAA;IAEnC,YAAY,OAA2D;QACrE,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAA;QAC3B,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAA;QAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,mBAAmB,CAAA;QAE1C,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAA;IACpB,CAAC;IAEQ,UAAU,CAAC,OAAiB,EAAE,SAAyB,EAAE,QAA2B;QAC3F,IAAI,CAAC;YACH,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAE3B,yCAAyC;YACzC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC3C,IAAI,CAAC,aAAa,EAAE,CAAA;gBACpB,OAAM;YACR,CAAC;iBAAM,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;gBACjC,uCAAuC;gBACvC,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;YAC7E,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,QAAQ,CAAC,IAAI,CAAC,CAAA;QAChB,CAAC;IACH,CAAC;IAED,qDAAqD;IAC5C,MAAM,CAAC,QAAoB;QAClC,IAAI,CAAC,aAAa,EAAE,CAAA;QACpB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA,CAAC,gBAAgB;QAChC,QAAQ,EAAE,CAAA;IACZ,CAAC;IAEO,aAAa;QACnB,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,YAAY,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;YAClC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAA;QAClC,CAAC;QACD,IAAI,IAAI,CAAC,UAAU;YAAE,OAAM;QAC3B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAA;QAEtB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;QAC9D,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;YACpB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACrB,CAAC;QACD,IAAI,CAAC,UAAU,GAAG,KAAK,CAAA;IACzB,CAAC;IAEQ,IAAI,CAAC,KAAwB,EAAE,QAAyB;QAC/D,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;IACpC,CAAC;CACF"}
1
+ {"version":3,"file":"KafkaMessageBatchStream.js","sourceRoot":"","sources":["../../lib/utils/KafkaMessageBatchStream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AAWpC;;;;;;GAMG;AACH,MAAM,OAAO,uBAAkC,SAAQ,MAAM;IAC1C,SAAS,CAAQ;IACjB,OAAO,CAAQ;IAEf,QAAQ,CAAY;IAC7B,eAAe,CAA4B;IAC3C,eAAe,CAA8B;IAC7C,UAAU,GAAY,KAAK,CAAA;IAEnC,YAAY,OAA2D;QACrE,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAA;QAC3B,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAA;QAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,mBAAmB,CAAA;QAE1C,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAA;QAElB,uDAAuD;QACvD,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACpB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,OAAM;YAEjC,MAAM,EAAE,GAAG,IAAI,CAAC,eAAe,CAAA;YAC/B,IAAI,CAAC,eAAe,GAAG,SAAS,CAAA;YAChC,EAAE,EAAE,CAAA;QACN,CAAC,CAAC,CAAA;IACJ,CAAC;IAEQ,KAAK;QACZ,8DAA8D;IAChE,CAAC;IAEQ,MAAM,CAAC,OAAiB,EAAE,SAAyB,EAAE,QAA0B;QACtF,4EAA4E;QAC5E,IAAI,WAAW,GAAG,IAAI,CAAA;QAEtB,IAAI,CAAC;YACH,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAE3B,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC3C,WAAW,GAAG,IAAI,CAAC,aAAa,EAAE,CAAA;YACpC,CAAC;iBAAM,CAAC;gBACN,4DAA4D;gBAC5D,2CAA2C;gBAC3C,qCAAqC;gBACrC,IAAI,CAAC,eAAe,KAAK,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;YAC/E,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,2CAA2C;YAC3C,oDAAoD;YACpD,kBAAkB;YAClB,QAAQ,EAAE,CAAA;QACZ,CAAC;IACH,CAAC;IAEQ,MAAM,CAAC,QAA0B;QACxC,IAAI,CAAC,aAAa,EAAE,CAAA;QACpB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA,CAAC,oBAAoB;QACpC,QAAQ,EAAE,CAAA;IACZ,CAAC;IAEO,aAAa;QACnB,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,YAAY,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;YAClC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAA;QAClC,CAAC;QACD,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO,IAAI,CAAA;QAChC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAA;QAEtB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;QAC9D,IAAI,WAAW,GAAG,IAAI,CAAA;QACtB,IAAI,QAAQ,CAAC,MAAM;YAAE,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACtD,IAAI,CAAC,UAAU,GAAG,KAAK,CAAA;QAEvB,OAAO,WAAW,CAAA;IACpB,CAAC;IAEQ,IAAI,CAAC,KAAwB,EAAE,QAAyB;QAC/D,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;IACpC,CAAC;CACF"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@message-queue-toolkit/kafka",
3
- "version": "0.9.1-beta.4",
3
+ "version": "0.9.1-beta.6",
4
4
  "engines": {
5
5
  "node": ">= 22.14.0"
6
6
  },