@message-queue-toolkit/kafka 0.9.1-beta.5 → 0.9.1-beta.7d
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,8 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
type
|
|
3
|
-
topic: string;
|
|
4
|
-
partition: number;
|
|
5
|
-
};
|
|
1
|
+
import { Duplex } from 'node:stream';
|
|
2
|
+
type CallbackFunction = (error?: Error | null) => void;
|
|
6
3
|
export type KafkaMessageBatchOptions = {
|
|
7
4
|
batchSize: number;
|
|
8
5
|
timeoutMilliseconds: number;
|
|
@@ -19,18 +16,20 @@ export type MessageBatch<TMessage> = {
|
|
|
19
16
|
* When the downstream consumer is slow, the stream will automatically pause accepting new messages
|
|
20
17
|
* until the consumer catches up, preventing memory leaks and OOM errors.
|
|
21
18
|
*/
|
|
22
|
-
export declare class KafkaMessageBatchStream<TMessage
|
|
19
|
+
export declare class KafkaMessageBatchStream<TMessage> extends Duplex {
|
|
23
20
|
private readonly batchSize;
|
|
24
21
|
private readonly timeout;
|
|
25
22
|
private readonly messages;
|
|
26
23
|
private existingTimeout;
|
|
24
|
+
private pendingCallback;
|
|
27
25
|
private isFlushing;
|
|
28
26
|
constructor(options: {
|
|
29
27
|
batchSize: number;
|
|
30
28
|
timeoutMilliseconds: number;
|
|
31
29
|
});
|
|
32
|
-
|
|
33
|
-
|
|
30
|
+
_read(): void;
|
|
31
|
+
_write(message: TMessage, _encoding: BufferEncoding, callback: CallbackFunction): void;
|
|
32
|
+
_final(callback: CallbackFunction): void;
|
|
34
33
|
private flushMessages;
|
|
35
34
|
push(chunk: TMessage[] | null, encoding?: BufferEncoding): boolean;
|
|
36
35
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Duplex } from 'node:stream';
|
|
2
2
|
/**
|
|
3
3
|
* Collects messages in batches based on provided batchSize and flushes them when messages amount or timeout is reached.
|
|
4
4
|
*
|
|
@@ -6,11 +6,12 @@ import { Transform } from 'node:stream';
|
|
|
6
6
|
* When the downstream consumer is slow, the stream will automatically pause accepting new messages
|
|
7
7
|
* until the consumer catches up, preventing memory leaks and OOM errors.
|
|
8
8
|
*/
|
|
9
|
-
export class KafkaMessageBatchStream extends
|
|
9
|
+
export class KafkaMessageBatchStream extends Duplex {
|
|
10
10
|
batchSize;
|
|
11
11
|
timeout;
|
|
12
12
|
messages;
|
|
13
13
|
existingTimeout;
|
|
14
|
+
pendingCallback;
|
|
14
15
|
isFlushing = false;
|
|
15
16
|
constructor(options) {
|
|
16
17
|
super({ objectMode: true });
|
|
@@ -18,40 +19,52 @@ export class KafkaMessageBatchStream extends Transform {
|
|
|
18
19
|
this.timeout = options.timeoutMilliseconds;
|
|
19
20
|
this.messages = [];
|
|
20
21
|
}
|
|
21
|
-
|
|
22
|
+
_read() {
|
|
23
|
+
// When _read is called, it means the downstream consumer is ready for more data
|
|
24
|
+
// This is when we should resume the writable side by calling the pending callback if it exists
|
|
25
|
+
if (!this.pendingCallback)
|
|
26
|
+
return;
|
|
27
|
+
const cb = this.pendingCallback;
|
|
28
|
+
this.pendingCallback = undefined;
|
|
29
|
+
cb();
|
|
30
|
+
}
|
|
31
|
+
_write(message, _encoding, callback) {
|
|
32
|
+
let canContinue = true;
|
|
22
33
|
try {
|
|
23
34
|
this.messages.push(message);
|
|
24
|
-
// Check if the batch is complete by size
|
|
25
35
|
if (this.messages.length >= this.batchSize) {
|
|
26
|
-
this.flushMessages();
|
|
27
|
-
return;
|
|
36
|
+
canContinue = this.flushMessages();
|
|
28
37
|
}
|
|
29
|
-
else
|
|
30
|
-
//
|
|
31
|
-
|
|
38
|
+
else {
|
|
39
|
+
// If backpressure happens, we don't have a callback to hold
|
|
40
|
+
// The next _write will handle backpressure
|
|
41
|
+
this.existingTimeout ??= setTimeout(() => this.flushMessages(), this.timeout);
|
|
32
42
|
}
|
|
33
43
|
}
|
|
34
44
|
finally {
|
|
35
|
-
|
|
45
|
+
if (!canContinue)
|
|
46
|
+
this.pendingCallback = callback;
|
|
47
|
+
else
|
|
48
|
+
callback();
|
|
36
49
|
}
|
|
37
50
|
}
|
|
38
51
|
_final(callback) {
|
|
39
52
|
this.flushMessages();
|
|
53
|
+
this.push(null); // End readable side
|
|
40
54
|
callback();
|
|
41
55
|
}
|
|
42
56
|
flushMessages() {
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
this.existingTimeout = undefined;
|
|
46
|
-
}
|
|
57
|
+
clearTimeout(this.existingTimeout);
|
|
58
|
+
this.existingTimeout = undefined;
|
|
47
59
|
if (this.isFlushing)
|
|
48
|
-
return;
|
|
60
|
+
return true;
|
|
49
61
|
this.isFlushing = true;
|
|
50
62
|
const messages = this.messages.splice(0, this.messages.length);
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
63
|
+
let canContinue = true;
|
|
64
|
+
if (messages.length)
|
|
65
|
+
canContinue = this.push(messages);
|
|
54
66
|
this.isFlushing = false;
|
|
67
|
+
return canContinue;
|
|
55
68
|
}
|
|
56
69
|
push(chunk, encoding) {
|
|
57
70
|
return super.push(chunk, encoding);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"KafkaMessageBatchStream.js","sourceRoot":"","sources":["../../lib/utils/KafkaMessageBatchStream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"KafkaMessageBatchStream.js","sourceRoot":"","sources":["../../lib/utils/KafkaMessageBatchStream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AAWpC;;;;;;GAMG;AACH,MAAM,OAAO,uBAAkC,SAAQ,MAAM;IAC1C,SAAS,CAAQ;IACjB,OAAO,CAAQ;IAEf,QAAQ,CAAY;IAC7B,eAAe,CAA4B;IAC3C,eAAe,CAA8B;IAC7C,UAAU,GAAY,KAAK,CAAA;IAEnC,YAAY,OAA2D;QACrE,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAA;QAC3B,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAA;QAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,mBAAmB,CAAA;QAE1C,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAA;IACpB,CAAC;IAEQ,KAAK;QACZ,gFAAgF;QAChF,+FAA+F;QAC/F,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,OAAM;QAEjC,MAAM,EAAE,GAAG,IAAI,CAAC,eAAe,CAAA;QAC/B,IAAI,CAAC,eAAe,GAAG,SAAS,CAAA;QAChC,EAAE,EAAE,CAAA;IACN,CAAC;IAEQ,MAAM,CAAC,OAAiB,EAAE,SAAyB,EAAE,QAA0B;QACtF,IAAI,WAAW,GAAG,IAAI,CAAA;QAEtB,IAAI,CAAC;YACH,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAE3B,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC3C,WAAW,GAAG,IAAI,CAAC,aAAa,EAAE,CAAA;YACpC,CAAC;iBAAM,CAAC;gBACN,4DAA4D;gBAC5D,2CAA2C;gBAC3C,IAAI,CAAC,eAAe,KAAK,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;YAC/E,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,WAAW;gBAAE,IAAI,CAAC,eAAe,GAAG,QAAQ,CAAA;;gBAC5C,QAAQ,EAAE,CAAA;QACjB,CAAC;IACH,CAAC;IAEQ,MAAM,CAAC,QAA0B;QACxC,IAAI,CAAC,aAAa,EAAE,CAAA;QACpB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA,CAAC,oBAAoB;QACpC,QAAQ,EAAE,CAAA;IACZ,CAAC;IAEO,aAAa;QACnB,YAAY,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;QAClC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAA;QAEhC,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO,IAAI,CAAA;QAChC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAA;QAEtB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;QAC9D,IAAI,WAAW,GAAG,IAAI,CAAA;QACtB,IAAI,QAAQ,CAAC,MAAM;YAAE,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACtD,IAAI,CAAC,UAAU,GAAG,KAAK,CAAA;QAEvB,OAAO,WAAW,CAAA;IACpB,CAAC;IAEQ,IAAI,CAAC,KAAwB,EAAE,QAAyB;QAC/D,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;IACpC,CAAC;CACF"}
|