@message-queue-toolkit/kafka 0.9.1-beta.5 → 0.9.1-beta.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,8 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
type
|
|
3
|
-
topic: string;
|
|
4
|
-
partition: number;
|
|
5
|
-
};
|
|
1
|
+
import { Duplex } from 'node:stream';
|
|
2
|
+
type CallbackFunction = (error?: Error | null) => void;
|
|
6
3
|
export type KafkaMessageBatchOptions = {
|
|
7
4
|
batchSize: number;
|
|
8
5
|
timeoutMilliseconds: number;
|
|
@@ -19,18 +16,20 @@ export type MessageBatch<TMessage> = {
|
|
|
19
16
|
* When the downstream consumer is slow, the stream will automatically pause accepting new messages
|
|
20
17
|
* until the consumer catches up, preventing memory leaks and OOM errors.
|
|
21
18
|
*/
|
|
22
|
-
export declare class KafkaMessageBatchStream<TMessage
|
|
19
|
+
export declare class KafkaMessageBatchStream<TMessage> extends Duplex {
|
|
23
20
|
private readonly batchSize;
|
|
24
21
|
private readonly timeout;
|
|
25
22
|
private readonly messages;
|
|
26
23
|
private existingTimeout;
|
|
24
|
+
private pendingCallback;
|
|
27
25
|
private isFlushing;
|
|
28
26
|
constructor(options: {
|
|
29
27
|
batchSize: number;
|
|
30
28
|
timeoutMilliseconds: number;
|
|
31
29
|
});
|
|
32
|
-
|
|
33
|
-
|
|
30
|
+
_read(): void;
|
|
31
|
+
_write(message: TMessage, _encoding: BufferEncoding, callback: CallbackFunction): void;
|
|
32
|
+
_final(callback: CallbackFunction): void;
|
|
34
33
|
private flushMessages;
|
|
35
34
|
push(chunk: TMessage[] | null, encoding?: BufferEncoding): boolean;
|
|
36
35
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Duplex } from 'node:stream';
|
|
2
2
|
/**
|
|
3
3
|
* Collects messages in batches based on provided batchSize and flushes them when messages amount or timeout is reached.
|
|
4
4
|
*
|
|
@@ -6,37 +6,55 @@ import { Transform } from 'node:stream';
|
|
|
6
6
|
* When the downstream consumer is slow, the stream will automatically pause accepting new messages
|
|
7
7
|
* until the consumer catches up, preventing memory leaks and OOM errors.
|
|
8
8
|
*/
|
|
9
|
-
export class KafkaMessageBatchStream extends
|
|
9
|
+
export class KafkaMessageBatchStream extends Duplex {
|
|
10
10
|
batchSize;
|
|
11
11
|
timeout;
|
|
12
12
|
messages;
|
|
13
13
|
existingTimeout;
|
|
14
|
+
pendingCallback;
|
|
14
15
|
isFlushing = false;
|
|
15
16
|
constructor(options) {
|
|
16
17
|
super({ objectMode: true });
|
|
17
18
|
this.batchSize = options.batchSize;
|
|
18
19
|
this.timeout = options.timeoutMilliseconds;
|
|
19
20
|
this.messages = [];
|
|
21
|
+
// Listen to 'drain' event to resume after backpressure
|
|
22
|
+
this.on('drain', () => {
|
|
23
|
+
if (!this.pendingCallback)
|
|
24
|
+
return;
|
|
25
|
+
const cb = this.pendingCallback;
|
|
26
|
+
this.pendingCallback = undefined;
|
|
27
|
+
cb();
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
_read() {
|
|
31
|
+
// No-op, as we push data when we have a full batch or timeout
|
|
20
32
|
}
|
|
21
|
-
|
|
33
|
+
_write(message, _encoding, callback) {
|
|
34
|
+
// biome-ignore lint/correctness/noUnusedVariables: All good in this version
|
|
35
|
+
let canContinue = true;
|
|
22
36
|
try {
|
|
23
37
|
this.messages.push(message);
|
|
24
|
-
// Check if the batch is complete by size
|
|
25
38
|
if (this.messages.length >= this.batchSize) {
|
|
26
|
-
this.flushMessages();
|
|
27
|
-
return;
|
|
39
|
+
canContinue = this.flushMessages();
|
|
28
40
|
}
|
|
29
|
-
else
|
|
30
|
-
//
|
|
31
|
-
|
|
41
|
+
else {
|
|
42
|
+
// If backpressure happens, we don't have a callback to hold
|
|
43
|
+
// The next _write will handle backpressure
|
|
44
|
+
// TODO: check if we can handle this.
|
|
45
|
+
this.existingTimeout ??= setTimeout(() => this.flushMessages(), this.timeout);
|
|
32
46
|
}
|
|
33
47
|
}
|
|
34
48
|
finally {
|
|
35
|
-
|
|
49
|
+
// TODO: to be enabled in next beta version
|
|
50
|
+
// if (!canContinue) this.pendingCallback = callback
|
|
51
|
+
// else callback()
|
|
52
|
+
callback();
|
|
36
53
|
}
|
|
37
54
|
}
|
|
38
55
|
_final(callback) {
|
|
39
56
|
this.flushMessages();
|
|
57
|
+
this.push(null); // End readable side
|
|
40
58
|
callback();
|
|
41
59
|
}
|
|
42
60
|
flushMessages() {
|
|
@@ -45,13 +63,14 @@ export class KafkaMessageBatchStream extends Transform {
|
|
|
45
63
|
this.existingTimeout = undefined;
|
|
46
64
|
}
|
|
47
65
|
if (this.isFlushing)
|
|
48
|
-
return;
|
|
66
|
+
return true;
|
|
49
67
|
this.isFlushing = true;
|
|
50
68
|
const messages = this.messages.splice(0, this.messages.length);
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
69
|
+
let canContinue = true;
|
|
70
|
+
if (messages.length)
|
|
71
|
+
canContinue = this.push(messages);
|
|
54
72
|
this.isFlushing = false;
|
|
73
|
+
return canContinue;
|
|
55
74
|
}
|
|
56
75
|
push(chunk, encoding) {
|
|
57
76
|
return super.push(chunk, encoding);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"KafkaMessageBatchStream.js","sourceRoot":"","sources":["../../lib/utils/KafkaMessageBatchStream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"KafkaMessageBatchStream.js","sourceRoot":"","sources":["../../lib/utils/KafkaMessageBatchStream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AAWpC;;;;;;GAMG;AACH,MAAM,OAAO,uBAAkC,SAAQ,MAAM;IAC1C,SAAS,CAAQ;IACjB,OAAO,CAAQ;IAEf,QAAQ,CAAY;IAC7B,eAAe,CAA4B;IAC3C,eAAe,CAA8B;IAC7C,UAAU,GAAY,KAAK,CAAA;IAEnC,YAAY,OAA2D;QACrE,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAA;QAC3B,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAA;QAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,mBAAmB,CAAA;QAE1C,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAA;QAElB,uDAAuD;QACvD,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACpB,IAAI,CAAC,IAAI,CAAC,eAAe;gBAAE,OAAM;YAEjC,MAAM,EAAE,GAAG,IAAI,CAAC,eAAe,CAAA;YAC/B,IAAI,CAAC,eAAe,GAAG,SAAS,CAAA;YAChC,EAAE,EAAE,CAAA;QACN,CAAC,CAAC,CAAA;IACJ,CAAC;IAEQ,KAAK;QACZ,8DAA8D;IAChE,CAAC;IAEQ,MAAM,CAAC,OAAiB,EAAE,SAAyB,EAAE,QAA0B;QACtF,4EAA4E;QAC5E,IAAI,WAAW,GAAG,IAAI,CAAA;QAEtB,IAAI,CAAC;YACH,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAE3B,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC3C,WAAW,GAAG,IAAI,CAAC,aAAa,EAAE,CAAA;YACpC,CAAC;iBAAM,CAAC;gBACN,4DAA4D;gBAC5D,2CAA2C;gBAC3C,qCAAqC;gBACrC,IAAI,CAAC,eAAe,KAAK,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;YAC/E,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,2CAA2C;YAC3C,oDAAoD;YACpD,kBAAkB;YAClB,QAAQ,EAAE,CAAA;QACZ,CAAC;IACH,CAAC;IAEQ,MAAM,CAAC,QAA0B;QACxC,IAAI,CAAC,aAAa,EAAE,CAAA;QACpB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA,CAAC,oBAAoB;QACpC,QAAQ,EAAE,CAAA;IACZ,CAAC;IAEO,aAAa;QACnB,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,YAAY,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;YAClC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAA;QAClC,CAAC;QACD,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO,IAAI,CAAA;QAChC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAA;QAEtB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;QAC9D,IAAI,WAAW,GAAG,IAAI,CAAA;QACtB,IAAI,QAAQ,CAAC,MAAM;YAAE,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACtD,IAAI,CAAC,UAAU,GAAG,KAAK,CAAA;QAEvB,OAAO,WAAW,CAAA;IACpB,CAAC;IAEQ,IAAI,CAAC,KAAwB,EAAE,QAAyB;QAC/D,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;IACpC,CAAC;CACF"}
|