@message-queue-toolkit/kafka 0.9.0 → 0.9.1-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/AbstractKafkaConsumer.d.ts +1 -0
- package/dist/AbstractKafkaConsumer.js +17 -5
- package/dist/AbstractKafkaConsumer.js.map +1 -1
- package/dist/utils/KafkaMessageBatchStream.d.ts +7 -4
- package/dist/utils/KafkaMessageBatchStream.js +13 -19
- package/dist/utils/KafkaMessageBatchStream.js.map +1 -1
- package/package.json +3 -3
|
@@ -33,6 +33,7 @@ export declare abstract class AbstractKafkaConsumer<TopicsConfig extends TopicCo
|
|
|
33
33
|
get isActive(): boolean;
|
|
34
34
|
init(): Promise<void>;
|
|
35
35
|
private handleSyncStream;
|
|
36
|
+
private handleSyncStreamBatch;
|
|
36
37
|
close(): Promise<void>;
|
|
37
38
|
private resolveHandler;
|
|
38
39
|
private consume;
|
|
@@ -95,14 +95,13 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
|
|
|
95
95
|
heartbeatInterval: consumeOptions.heartbeatInterval,
|
|
96
96
|
});
|
|
97
97
|
this.consumerStream = await this.consumer.consume({ ...consumeOptions, topics });
|
|
98
|
-
this.consumerStream.on('error', (error) => this.handlerError(error));
|
|
99
98
|
if (this.options.batchProcessingEnabled && this.options.batchProcessingOptions) {
|
|
100
|
-
this.messageBatchStream = new KafkaMessageBatchStream(
|
|
99
|
+
this.messageBatchStream = new KafkaMessageBatchStream({
|
|
100
|
+
batchSize: this.options.batchProcessingOptions.batchSize,
|
|
101
|
+
timeoutMilliseconds: this.options.batchProcessingOptions.timeoutMilliseconds,
|
|
102
|
+
});
|
|
101
103
|
this.consumerStream.pipe(this.messageBatchStream);
|
|
102
104
|
}
|
|
103
|
-
else {
|
|
104
|
-
this.handleSyncStream(this.consumerStream).catch((error) => this.handlerError(error));
|
|
105
|
-
}
|
|
106
105
|
}
|
|
107
106
|
catch (error) {
|
|
108
107
|
throw new InternalError({
|
|
@@ -111,12 +110,25 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
|
|
|
111
110
|
cause: error,
|
|
112
111
|
});
|
|
113
112
|
}
|
|
113
|
+
if (this.options.batchProcessingEnabled && this.messageBatchStream) {
|
|
114
|
+
this.handleSyncStreamBatch(this.messageBatchStream).catch((error) => this.handlerError(error));
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
this.handleSyncStream(this.consumerStream).catch((error) => this.handlerError(error));
|
|
118
|
+
}
|
|
119
|
+
this.consumerStream.on('error', (error) => this.handlerError(error));
|
|
120
|
+
this.messageBatchStream?.on('error', (error) => this.handlerError(error));
|
|
114
121
|
}
|
|
115
122
|
async handleSyncStream(stream) {
|
|
116
123
|
for await (const message of stream) {
|
|
117
124
|
await this.consume(message.topic, message);
|
|
118
125
|
}
|
|
119
126
|
}
|
|
127
|
+
async handleSyncStreamBatch(stream) {
|
|
128
|
+
for await (const messageBatch of stream) {
|
|
129
|
+
await this.consume(messageBatch.topic, messageBatch.messages);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
120
132
|
async close() {
|
|
121
133
|
if (!this.consumerStream && !this.messageBatchStream) {
|
|
122
134
|
// Leaving the group in case consumer joined but streams were not created
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AbstractKafkaConsumer.js","sourceRoot":"","sources":["../lib/AbstractKafkaConsumer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EACL,aAAa,EACb,qBAAqB,GAEtB,MAAM,qBAAqB,CAAA;AAK5B,OAAO,EAEL,QAAQ,EAGR,aAAa,EACb,aAAa,EACb,kBAAkB,GACnB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACL,oBAAoB,GAGrB,MAAM,2BAA2B,CAAA;AAYlC,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAA;AACpG,OAAO,EAEL,uBAAuB,GACxB,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAA;AAkCtE,MAAM,wBAAwB,GAAG,IAAI,GAAG,CAAC;IACvC,kBAAkB;IAClB,iBAAiB;IACjB,qBAAqB;CACtB,CAAC,CAAA;AAEF;;;GAGG;AACH,MAAM,qBAAqB,GAAG,CAAC,CAAA;AAE/B,MAAM,OAAgB,qBAIpB,SAAQ,oBAGT;IACkB,QAAQ,CAA0C;IAC3D,cAAc,CAAiD;IAC/D,kBAAkB,CAEzB;IAEgB,+BAA+B,CAAiC;IAChE,gBAAgB,CAAkB;IAEnD,YACE,YAAuC,EACvC,OAAqF,EACrF,gBAAkC;QAElC,KAAK,CAAC,YAAY,EAAE,OAAO,CAAC,CAAA;QAE5B,IAAI,CAAC,+BAA+B,GAAG,YAAY,CAAC,+BAA+B,CAAA;QACnF,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAA;QAExC,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC;YAC3B,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK;YACrB,GAAG,IAAI,CAAC,OAAO;YACf,UAAU,EAAE,KAAK,EAAE,4BAA4B;YAC/C,aAAa,EAAE;gBACb,GAAG,EAAE,kBAAkB;gBACvB,KAAK,EAAE,oBAAoB;gBAC3B,SAAS,EAAE,kBAAkB;gBAC7B,WAAW,EAAE,kBAAkB;aAChC;SACF,CAAC,CAAA;QAEF,MAAM,UAAU,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAA;QACnF,qBAAqB;QACrB,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,EAAE,CAC5C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,6BAA6B,CAAC,CAC7D,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,iBAAiB,EAAE,CAAC,CAAC,EAAE,EAAE,CACxC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,kDAAkD,CAAC,CAClF,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,sBAAsB,EAAE,CAAC,CAAC,EAAE,EAAE,CAC7C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,+BAA+B,CAAC,CAC/D,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,0BAA0B,EAAE,CAAC,CAAC,EAAE,EAAE,CACjD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,sBAAsB,CAAC,CACtD,CAAA;QACD,oBAAoB;IACtB,CAAC;IAED;;OAEG;IACH,IAAI,WAAW;QACb,uDAAuD;QACvD,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO,KAAK,CAAA;QAClE,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAA;YAClC,qBAAqB;QACvB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,yEAAyE;YACzE,OAAO,KAAK,CAAA;QACd,CAAC;QACD,oBAAoB;IACtB,CAAC;IAED;;;OAGG;IACH,IAAI,QAAQ;QACV,uDAAuD;QACvD,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO,KAAK,CAAA;QAClE,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAA;YAC/B,qBAAqB;QACvB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,yEAAyE;YACzE,OAAO,KAAK,CAAA;QACd,CAAC;QACD,oBAAoB;IACtB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;QACjD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QACjD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QAE9E,IAAI,CAAC;YACH,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC,OAAO,CAAA,CAAC,8CAA8C;YAEtG,0IAA0I;YAC1I,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC5B,cAAc,EAAE,cAAc,CAAC,cAAc;gBAC7C,gBAAgB,EAAE,cAAc,CAAC,gBAAgB;gBACjD,iBAAiB,EAAE,cAAc,CAAC,iBAAiB;aACpD,CAAC,CAAA;YAEF,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,GAAG,cAAc,EAAE,MAAM,EAAE,CAAC,CAAA;YAChF,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;YAEpE,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;gBAC/E,IAAI,CAAC,kBAAkB,GAAG,IAAI,uBAAuB,CAGnD,CAAC,KAAK,EAAE,EAAE,CACR,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EACtF,IAAI,CAAC,OAAO,CAAC,sBAAsB,CACpC,CAAA;gBACD,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAA;YACnD,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;YACvF,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,aAAa,CAAC;gBACtB,OAAO,EAAE,sBAAsB;gBAC/B,SAAS,EAAE,2BAA2B;gBACtC,KAAK,EAAE,KAAK;aACb,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,MAAsD;QAEtD,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;YACnC,MAAM,IAAI,CAAC,OAAO,CAChB,OAAO,CAAC,KAAK,EACb,OAAoE,CACrE,CAAA;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACrD,yEAAyE;YACzE,IAAI,IAAI,CAAC,QAAQ;gBAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAA;YAC7C,OAAM;QACR,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAA;YACjC,IAAI,CAAC,cAAc,GAAG,SAAS,CAAA;QACjC,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAA;YACrE,IAAI,CAAC,kBAAkB,GAAG,SAAS,CAAA;QACrC,CAAC;QAED,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;IAC7B,CAAC;IAEO,cAAc,CAAC,KAAoC;QACzD,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IACrC,CAAC;IAEO,KAAK,CAAC,OAAO,CACnB,KAAa,EACb,cAAoE;QAEpE,MAAM,+BAA+B,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAClD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,sBAAsB,CAAC,CAAA;QAEnF,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;QAEhD,iFAAiF;QACjF,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;QAEtD,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CACtC,aAAa,EACb,cAAc,EACd,+BAA+B,CAChC,CAAA;QAED,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,+BAA+B,CAAC,CAAA;YAC5F,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;QACpC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,kBAAkB,EAAE,aAAa,CAAC,MAAM,EAAE,EAClF,sCAAsC,CACvC,CAAA;QACH,CAAC;QAED,kFAAkF;QAClF,MAAM,YAAY,GAAG,aAAa,CAAC,CAAC,CAAE,CAAA;QACtC,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAA;QAE3D,MAAM,aAAa,GAAG,UAAU,EAAE,CAAA;QAClC,IAAI,CAAC,+BAA+B,EAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,EAAE,aAAa,CAAC,CAAA;QAE5F,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,uBAAuB,CACzD,KAAK;QACL,0EAA0E;QAC1E,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,EAClE,aAAa,CAAC,OAAO,EACrB,cAAc,CACf,CAAA;QAED,IAAI,CAAC,uBAAuB,CAAC,aAAa,EAAE,gBAAgB,EAAE,+BAA+B,CAAC,CAAA;QAE9F,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,aAAa,CAAC,CAAA;QAEzD,+EAA+E;QAC/E,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;IACpC,CAAC;IAEO,aAAa,CACnB,aAIC,EACD,cAAoE,EACpE,+BAAuC;QAEvC,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAA;QAEzF,MAAM,aAAa,GAAgE,EAAE,CAAA;QAErF,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,yEAAyE;YACzE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACnB,SAAQ;YACV,CAAC;YAED,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAEjE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;gBACzB,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,EAAE;oBACnC,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC;iBAC9C,CAAC,CAAA;gBACF,IAAI,CAAC,sBAAsB,CAAC;oBAC1B,OAAO,EAAE,OAAO;oBAChB,gBAAgB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE;oBACpE,+BAA+B;iBAChC,CAAC,CAAA;gBACF,SAAQ;YACV,CAAC;YAED,aAAa,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,IAAI,EAAE,CAAC,CAAA;QAC7D,CAAC;QAED,OAAO,aAAa,CAAA;IACtB,CAAC;IAEO,KAAK,CAAC,uBAAuB,CACnC,KAAa,EACb,cAA4C,EAC5C,OAA6E,EAC7E,cAA8B;QAE9B,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,gBAAgB,GAA4B;YAC9C,MAAM,EAAE,OAAO;YACf,WAAW,EAAE,cAAc;SAC5B,CAAA;QACD,GAAG,CAAC;YACF,qCAAqC;YACrC,IAAI,OAAO,GAAG,CAAC;gBAAE,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAA;YAE3D,gBAAgB,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,cAAc,CAAC,CAAA;YAC1F,IAAI,gBAAgB,CAAC,MAAM,KAAK,UAAU;gBAAE,MAAK;YAEjD,OAAO,EAAE,CAAA;QACX,CAAC,QAAQ,OAAO,GAAG,qBAAqB,EAAC;QAEzC,OAAO,gBAAgB,CAAA;IACzB,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,KAAa,EACb,cAA4C,EAC5C,OAA6E,EAC7E,cAA8B;QAE9B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;YAC7C,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,CAAC,OAAO,EAAE,CAAC;gBACpD,MAAM,IAAI,KAAK,CACb,6EAA6E,CAC9E,CAAA;YACH,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,OAAO,EAAE,CAAC;gBACpD,MAAM,IAAI,KAAK,CACb,iFAAiF,CAClF,CAAA;YACH,CAAC;YAED,MAAM,OAAO;YACX,mGAAmG;YACnG,cAEuC,EACvC,IAAI,CAAC,gBAAgB,EACrB,cAAc,CACf,CAAA;YACD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAA;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC;gBAChD,CAAC,CAAC,EAAE,SAAS,EAAE,cAAc,CAAC,MAAM,EAAE;gBACtC,CAAC,CAAC,EAAE,OAAO,EAAE,qBAAqB,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,CAAA;YAC5D,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE;gBACvB,KAAK;gBACL,GAAG,YAAY;aAChB,CAAC,CAAA;QACJ,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,CAAA;IACzD,CAAC;IAEO,uBAAuB,CAC7B,QAA0C,EAC1C,gBAAyC,EACzC,+BAAuC;QAEvC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,sBAAsB,CAAC;gBAC1B,OAAO;gBACP,gBAAgB;gBAChB,+BAA+B;aAChC,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,cAAoE;QACjF,IAAI,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;YAClC,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;YAEzD,wEAAwE;YACxE,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAA;QACvE,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,CAAA;QAC3C,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAkE;QAC5F,MAAM,UAAU,GAAG;YACjB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,SAAS,EAAE,OAAO,CAAC,SAAS;SAC7B,CAAA;QACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,0BAA0B,CAAC,CAAA;QAEzD,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,MAAM,EAAE,CAAA;YACtB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,gCAAgC,CAAC,CAAA;QACjE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAA;YACtD,IAAI,KAAK,YAAY,aAAa;gBAAE,OAAO,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAA;YAClF,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAEO,2BAA2B,CAAC,aAA4B;QAC9D,kFAAkF;QAClF,KAAK,MAAM,KAAK,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YACzC,IACE,KAAK,YAAY,aAAa;gBAC9B,KAAK,CAAC,OAAO;gBACb,wBAAwB,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAC3C,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,KAAK,CACf;oBACE,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,oBAAoB,EAAE,aAAa,CAAC,OAAO;oBAC3C,oBAAoB,EAAE,KAAK,CAAC,OAAO;oBACnC,KAAK,EAAE,aAAa;iBACrB,EACD,6BAA6B,KAAK,CAAC,OAAO,EAAE,CAC7C,CAAA;YACH,CAAC;iBAAM,CAAC;gBACN,yCAAyC;gBACzC,MAAM,aAAa,CAAA;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IAEO,oBAAoB,CAAC,KAAa;QACxC,MAAM,mBAAmB,GAAG,SAAS,IAAI,CAAC,WAAW,CAAC,IAAI,IAAI,KAAK,EAAE,CAAA;QACrE,OAAO,IAAI,CAAC,OAAO,CAAC,sBAAsB;YACxC,CAAC,CAAC,GAAG,mBAAmB,QAAQ;YAChC,CAAC,CAAC,mBAAmB,CAAA;IACzB,CAAC;IAEO,iBAAiB,CACvB,OAAkE;QAElE,IAAI,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC,CAAA;QACnE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,KAAK,GAAG,UAAU,EAAE,CAAA;QAE7D,OAAO;YACL,KAAK;YACL,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;gBACxB,cAAc,EAAE,KAAK;gBACrB,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;gBAC7B,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,UAAU,EAAE,OAAO,CAAC,GAAG;aACxB,CAAC;SACH,CAAA;IACH,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"AbstractKafkaConsumer.js","sourceRoot":"","sources":["../lib/AbstractKafkaConsumer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EACL,aAAa,EACb,qBAAqB,GAEtB,MAAM,qBAAqB,CAAA;AAK5B,OAAO,EAEL,QAAQ,EAGR,aAAa,EACb,aAAa,EACb,kBAAkB,GACnB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACL,oBAAoB,GAGrB,MAAM,2BAA2B,CAAA;AAYlC,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAA;AACpG,OAAO,EAEL,uBAAuB,GACxB,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAA;AAkCtE,MAAM,wBAAwB,GAAG,IAAI,GAAG,CAAC;IACvC,kBAAkB;IAClB,iBAAiB;IACjB,qBAAqB;CACtB,CAAC,CAAA;AAEF;;;GAGG;AACH,MAAM,qBAAqB,GAAG,CAAC,CAAA;AAE/B,MAAM,OAAgB,qBAIpB,SAAQ,oBAGT;IACkB,QAAQ,CAA0C;IAC3D,cAAc,CAAiD;IAC/D,kBAAkB,CAEzB;IAEgB,+BAA+B,CAAiC;IAChE,gBAAgB,CAAkB;IAEnD,YACE,YAAuC,EACvC,OAAqF,EACrF,gBAAkC;QAElC,KAAK,CAAC,YAAY,EAAE,OAAO,CAAC,CAAA;QAE5B,IAAI,CAAC,+BAA+B,GAAG,YAAY,CAAC,+BAA+B,CAAA;QACnF,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAA;QAExC,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC;YAC3B,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK;YACrB,GAAG,IAAI,CAAC,OAAO;YACf,UAAU,EAAE,KAAK,EAAE,4BAA4B;YAC/C,aAAa,EAAE;gBACb,GAAG,EAAE,kBAAkB;gBACvB,KAAK,EAAE,oBAAoB;gBAC3B,SAAS,EAAE,kBAAkB;gBAC7B,WAAW,EAAE,kBAAkB;aAChC;SACF,CAAC,CAAA;QAEF,MAAM,UAAU,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAA;QACnF,qBAAqB;QACrB,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,EAAE,CAC5C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,6BAA6B,CAAC,CAC7D,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,iBAAiB,EAAE,CAAC,CAAC,EAAE,EAAE,CACxC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,kDAAkD,CAAC,CAClF,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,sBAAsB,EAAE,CAAC,CAAC,EAAE,EAAE,CAC7C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,+BAA+B,CAAC,CAC/D,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,0BAA0B,EAAE,CAAC,CAAC,EAAE,EAAE,CACjD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,sBAAsB,CAAC,CACtD,CAAA;QACD,oBAAoB;IACtB,CAAC;IAED;;OAEG;IACH,IAAI,WAAW;QACb,uDAAuD;QACvD,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO,KAAK,CAAA;QAClE,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAA;YAClC,qBAAqB;QACvB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,yEAAyE;YACzE,OAAO,KAAK,CAAA;QACd,CAAC;QACD,oBAAoB;IACtB,CAAC;IAED;;;OAGG;IACH,IAAI,QAAQ;QACV,uDAAuD;QACvD,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO,KAAK,CAAA;QAClE,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAA;YAC/B,qBAAqB;QACvB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,yEAAyE;YACzE,OAAO,KAAK,CAAA;QACd,CAAC;QACD,oBAAoB;IACtB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;QACjD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QACjD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QAE9E,IAAI,CAAC;YACH,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC,OAAO,CAAA,CAAC,8CAA8C;YAEtG,0IAA0I;YAC1I,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC5B,cAAc,EAAE,cAAc,CAAC,cAAc;gBAC7C,gBAAgB,EAAE,cAAc,CAAC,gBAAgB;gBACjD,iBAAiB,EAAE,cAAc,CAAC,iBAAiB;aACpD,CAAC,CAAA;YAEF,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,GAAG,cAAc,EAAE,MAAM,EAAE,CAAC,CAAA;YAChF,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;gBAC/E,IAAI,CAAC,kBAAkB,GAAG,IAAI,uBAAuB,CAEnD;oBACA,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,SAAS;oBACxD,mBAAmB,EAAE,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,mBAAmB;iBAC7E,CAAC,CAAA;gBACF,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAA;YACnD,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,aAAa,CAAC;gBACtB,OAAO,EAAE,sBAAsB;gBAC/B,SAAS,EAAE,2BAA2B;gBACtC,KAAK,EAAE,KAAK;aACb,CAAC,CAAA;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACnE,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;QAChG,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;QACvF,CAAC;QAED,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;QACpE,IAAI,CAAC,kBAAkB,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;IAC3E,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,MAAsD;QAEtD,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;YACnC,MAAM,IAAI,CAAC,OAAO,CAChB,OAAO,CAAC,KAAK,EACb,OAAoE,CACrE,CAAA;QACH,CAAC;IACH,CAAC;IACO,KAAK,CAAC,qBAAqB,CACjC,MAA0F;QAE1F,IAAI,KAAK,EAAE,MAAM,YAAY,IAAI,MAAM,EAAE,CAAC;YACxC,MAAM,IAAI,CAAC,OAAO,CAChB,YAAY,CAAC,KAAK,EAClB,YAAY,CAAC,QAAqE,CACnF,CAAA;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACrD,yEAAyE;YACzE,IAAI,IAAI,CAAC,QAAQ;gBAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAA;YAC7C,OAAM;QACR,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAA;YACjC,IAAI,CAAC,cAAc,GAAG,SAAS,CAAA;QACjC,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAA;YACrE,IAAI,CAAC,kBAAkB,GAAG,SAAS,CAAA;QACrC,CAAC;QAED,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;IAC7B,CAAC;IAEO,cAAc,CAAC,KAAoC;QACzD,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IACrC,CAAC;IAEO,KAAK,CAAC,OAAO,CACnB,KAAa,EACb,cAAoE;QAEpE,MAAM,+BAA+B,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAClD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,sBAAsB,CAAC,CAAA;QAEnF,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;QAEhD,iFAAiF;QACjF,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;QAEtD,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CACtC,aAAa,EACb,cAAc,EACd,+BAA+B,CAChC,CAAA;QAED,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,+BAA+B,CAAC,CAAA;YAC5F,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;QACpC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,kBAAkB,EAAE,aAAa,CAAC,MAAM,EAAE,EAClF,sCAAsC,CACvC,CAAA;QACH,CAAC;QAED,kFAAkF;QAClF,MAAM,YAAY,GAAG,aAAa,CAAC,CAAC,CAAE,CAAA;QACtC,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAA;QAE3D,MAAM,aAAa,GAAG,UAAU,EAAE,CAAA;QAClC,IAAI,CAAC,+BAA+B,EAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,EAAE,aAAa,CAAC,CAAA;QAE5F,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,uBAAuB,CACzD,KAAK;QACL,0EAA0E;QAC1E,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,EAClE,aAAa,CAAC,OAAO,EACrB,cAAc,CACf,CAAA;QAED,IAAI,CAAC,uBAAuB,CAAC,aAAa,EAAE,gBAAgB,EAAE,+BAA+B,CAAC,CAAA;QAE9F,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,aAAa,CAAC,CAAA;QAEzD,+EAA+E;QAC/E,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;IACpC,CAAC;IAEO,aAAa,CACnB,aAIC,EACD,cAAoE,EACpE,+BAAuC;QAEvC,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAA;QAEzF,MAAM,aAAa,GAAgE,EAAE,CAAA;QAErF,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,yEAAyE;YACzE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACnB,SAAQ;YACV,CAAC;YAED,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAEjE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;gBACzB,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,EAAE;oBACnC,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC;iBAC9C,CAAC,CAAA;gBACF,IAAI,CAAC,sBAAsB,CAAC;oBAC1B,OAAO,EAAE,OAAO;oBAChB,gBAAgB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE;oBACpE,+BAA+B;iBAChC,CAAC,CAAA;gBACF,SAAQ;YACV,CAAC;YAED,aAAa,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,IAAI,EAAE,CAAC,CAAA;QAC7D,CAAC;QAED,OAAO,aAAa,CAAA;IACtB,CAAC;IAEO,KAAK,CAAC,uBAAuB,CACnC,KAAa,EACb,cAA4C,EAC5C,OAA6E,EAC7E,cAA8B;QAE9B,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,gBAAgB,GAA4B;YAC9C,MAAM,EAAE,OAAO;YACf,WAAW,EAAE,cAAc;SAC5B,CAAA;QACD,GAAG,CAAC;YACF,qCAAqC;YACrC,IAAI,OAAO,GAAG,CAAC;gBAAE,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAA;YAE3D,gBAAgB,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,cAAc,CAAC,CAAA;YAC1F,IAAI,gBAAgB,CAAC,MAAM,KAAK,UAAU;gBAAE,MAAK;YAEjD,OAAO,EAAE,CAAA;QACX,CAAC,QAAQ,OAAO,GAAG,qBAAqB,EAAC;QAEzC,OAAO,gBAAgB,CAAA;IACzB,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,KAAa,EACb,cAA4C,EAC5C,OAA6E,EAC7E,cAA8B;QAE9B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;YAC7C,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,CAAC,OAAO,EAAE,CAAC;gBACpD,MAAM,IAAI,KAAK,CACb,6EAA6E,CAC9E,CAAA;YACH,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,OAAO,EAAE,CAAC;gBACpD,MAAM,IAAI,KAAK,CACb,iFAAiF,CAClF,CAAA;YACH,CAAC;YAED,MAAM,OAAO;YACX,mGAAmG;YACnG,cAEuC,EACvC,IAAI,CAAC,gBAAgB,EACrB,cAAc,CACf,CAAA;YACD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAA;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC;gBAChD,CAAC,CAAC,EAAE,SAAS,EAAE,cAAc,CAAC,MAAM,EAAE;gBACtC,CAAC,CAAC,EAAE,OAAO,EAAE,qBAAqB,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,CAAA;YAC5D,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE;gBACvB,KAAK;gBACL,GAAG,YAAY;aAChB,CAAC,CAAA;QACJ,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,CAAA;IACzD,CAAC;IAEO,uBAAuB,CAC7B,QAA0C,EAC1C,gBAAyC,EACzC,+BAAuC;QAEvC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,sBAAsB,CAAC;gBAC1B,OAAO;gBACP,gBAAgB;gBAChB,+BAA+B;aAChC,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,cAAoE;QACjF,IAAI,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;YAClC,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;YAEzD,wEAAwE;YACxE,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAA;QACvE,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,CAAA;QAC3C,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAkE;QAC5F,MAAM,UAAU,GAAG;YACjB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,SAAS,EAAE,OAAO,CAAC,SAAS;SAC7B,CAAA;QACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,0BAA0B,CAAC,CAAA;QAEzD,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,MAAM,EAAE,CAAA;YACtB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,gCAAgC,CAAC,CAAA;QACjE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAA;YACtD,IAAI,KAAK,YAAY,aAAa;gBAAE,OAAO,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAA;YAClF,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAEO,2BAA2B,CAAC,aAA4B;QAC9D,kFAAkF;QAClF,KAAK,MAAM,KAAK,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YACzC,IACE,KAAK,YAAY,aAAa;gBAC9B,KAAK,CAAC,OAAO;gBACb,wBAAwB,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAC3C,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,KAAK,CACf;oBACE,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,oBAAoB,EAAE,aAAa,CAAC,OAAO;oBAC3C,oBAAoB,EAAE,KAAK,CAAC,OAAO;oBACnC,KAAK,EAAE,aAAa;iBACrB,EACD,6BAA6B,KAAK,CAAC,OAAO,EAAE,CAC7C,CAAA;YACH,CAAC;iBAAM,CAAC;gBACN,yCAAyC;gBACzC,MAAM,aAAa,CAAA;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IAEO,oBAAoB,CAAC,KAAa;QACxC,MAAM,mBAAmB,GAAG,SAAS,IAAI,CAAC,WAAW,CAAC,IAAI,IAAI,KAAK,EAAE,CAAA;QACrE,OAAO,IAAI,CAAC,OAAO,CAAC,sBAAsB;YACxC,CAAC,CAAC,GAAG,mBAAmB,QAAQ;YAChC,CAAC,CAAC,mBAAmB,CAAA;IACzB,CAAC;IAEO,iBAAiB,CACvB,OAAkE;QAElE,IAAI,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC,CAAA;QACnE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,KAAK,GAAG,UAAU,EAAE,CAAA;QAE7D,OAAO;YACL,KAAK;YACL,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;gBACxB,cAAc,EAAE,KAAK;gBACrB,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;gBAC7B,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,UAAU,EAAE,OAAO,CAAC,GAAG;aACxB,CAAC;SACH,CAAA;IACH,CAAC;CACF"}
|
|
@@ -21,19 +21,22 @@ export type OnMessageBatchCallback<TMessage> = (batch: MessageBatch<TMessage>) =
|
|
|
21
21
|
* until the consumer catches up, preventing memory leaks and OOM errors.
|
|
22
22
|
*/
|
|
23
23
|
export declare class KafkaMessageBatchStream<TMessage extends MessageWithTopicAndPartition> extends Transform {
|
|
24
|
-
private readonly onBatch;
|
|
25
24
|
private readonly batchSize;
|
|
26
25
|
private readonly timeout;
|
|
27
26
|
private readonly currentBatchPerTopicPartition;
|
|
28
27
|
private readonly batchTimeoutPerTopicPartition;
|
|
29
|
-
|
|
30
|
-
constructor(onBatch: OnMessageBatchCallback<TMessage>, options: {
|
|
28
|
+
constructor(options: {
|
|
31
29
|
batchSize: number;
|
|
32
30
|
timeoutMilliseconds: number;
|
|
33
31
|
});
|
|
34
|
-
_transform(message: TMessage, _encoding: BufferEncoding, callback: () => void):
|
|
32
|
+
_transform(message: TMessage, _encoding: BufferEncoding, callback: () => void): void;
|
|
35
33
|
_flush(callback: () => void): Promise<void>;
|
|
36
34
|
private flushAllBatches;
|
|
37
35
|
private flushCurrentBatchMessages;
|
|
36
|
+
push(chunk: {
|
|
37
|
+
topic: string;
|
|
38
|
+
partition: number;
|
|
39
|
+
messages: TMessage[];
|
|
40
|
+
}, encoding?: BufferEncoding): boolean;
|
|
38
41
|
}
|
|
39
42
|
export {};
|
|
@@ -7,42 +7,32 @@ import { Transform } from 'node:stream';
|
|
|
7
7
|
* until the consumer catches up, preventing memory leaks and OOM errors.
|
|
8
8
|
*/
|
|
9
9
|
export class KafkaMessageBatchStream extends Transform {
|
|
10
|
-
onBatch;
|
|
11
10
|
batchSize;
|
|
12
11
|
timeout;
|
|
13
12
|
currentBatchPerTopicPartition;
|
|
14
13
|
batchTimeoutPerTopicPartition;
|
|
15
|
-
|
|
16
|
-
constructor(onBatch, options) {
|
|
14
|
+
constructor(options) {
|
|
17
15
|
super({ objectMode: true });
|
|
18
|
-
this.onBatch = onBatch;
|
|
19
16
|
this.batchSize = options.batchSize;
|
|
20
17
|
this.timeout = options.timeoutMilliseconds;
|
|
21
18
|
this.currentBatchPerTopicPartition = {};
|
|
22
19
|
this.batchTimeoutPerTopicPartition = {};
|
|
23
20
|
}
|
|
24
|
-
|
|
21
|
+
_transform(message, _encoding, callback) {
|
|
25
22
|
const key = getTopicPartitionKey(message.topic, message.partition);
|
|
26
|
-
// Wait for all pending timeout flushes to complete to maintain backpressure
|
|
27
|
-
if (this.timeoutProcessingPromises.size > 0) {
|
|
28
|
-
// Capture a snapshot of current promises to avoid race conditions with new timeouts
|
|
29
|
-
const promiseEntries = Array.from(this.timeoutProcessingPromises.entries());
|
|
30
|
-
// Wait for all to complete and then clean up from the map
|
|
31
|
-
await Promise.all(promiseEntries.map(([k, p]) => p.finally(() => this.timeoutProcessingPromises.delete(k))));
|
|
32
|
-
}
|
|
33
23
|
// Accumulate the message
|
|
34
24
|
if (!this.currentBatchPerTopicPartition[key])
|
|
35
25
|
this.currentBatchPerTopicPartition[key] = [];
|
|
36
26
|
this.currentBatchPerTopicPartition[key].push(message);
|
|
37
27
|
// Check if the batch is complete by size
|
|
38
28
|
if (this.currentBatchPerTopicPartition[key].length >= this.batchSize) {
|
|
39
|
-
|
|
29
|
+
this.flushCurrentBatchMessages(message.topic, message.partition);
|
|
40
30
|
callback();
|
|
41
31
|
return;
|
|
42
32
|
}
|
|
43
33
|
// Start timeout for this partition if not already started
|
|
44
34
|
if (!this.batchTimeoutPerTopicPartition[key]) {
|
|
45
|
-
this.batchTimeoutPerTopicPartition[key] = setTimeout(() => this.
|
|
35
|
+
this.batchTimeoutPerTopicPartition[key] = setTimeout(() => this.flushCurrentBatchMessages(message.topic, message.partition), this.timeout);
|
|
46
36
|
}
|
|
47
37
|
callback();
|
|
48
38
|
}
|
|
@@ -51,13 +41,13 @@ export class KafkaMessageBatchStream extends Transform {
|
|
|
51
41
|
await this.flushAllBatches();
|
|
52
42
|
callback();
|
|
53
43
|
}
|
|
54
|
-
|
|
44
|
+
flushAllBatches() {
|
|
55
45
|
for (const key of Object.keys(this.currentBatchPerTopicPartition)) {
|
|
56
46
|
const { topic, partition } = splitTopicPartitionKey(key);
|
|
57
|
-
|
|
47
|
+
this.flushCurrentBatchMessages(topic, partition);
|
|
58
48
|
}
|
|
59
49
|
}
|
|
60
|
-
|
|
50
|
+
flushCurrentBatchMessages(topic, partition) {
|
|
61
51
|
const key = getTopicPartitionKey(topic, partition);
|
|
62
52
|
// Clear timeout
|
|
63
53
|
if (this.batchTimeoutPerTopicPartition[key]) {
|
|
@@ -65,10 +55,14 @@ export class KafkaMessageBatchStream extends Transform {
|
|
|
65
55
|
this.batchTimeoutPerTopicPartition[key] = undefined;
|
|
66
56
|
}
|
|
67
57
|
const messages = this.currentBatchPerTopicPartition[key] ?? [];
|
|
68
|
-
|
|
69
|
-
|
|
58
|
+
if (!messages.length)
|
|
59
|
+
return;
|
|
60
|
+
this.push({ topic, partition, messages: messages });
|
|
70
61
|
this.currentBatchPerTopicPartition[key] = [];
|
|
71
62
|
}
|
|
63
|
+
push(chunk, encoding) {
|
|
64
|
+
return super.push(chunk, encoding);
|
|
65
|
+
}
|
|
72
66
|
}
|
|
73
67
|
const getTopicPartitionKey = (topic, partition) => `${topic}:${partition}`;
|
|
74
68
|
const splitTopicPartitionKey = (key) => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"KafkaMessageBatchStream.js","sourceRoot":"","sources":["../../lib/utils/KafkaMessageBatchStream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAavC;;;;;;GAMG;AACH,MAAM,OAAO,uBAEX,SAAQ,SAAS;IACA,
|
|
1
|
+
{"version":3,"file":"KafkaMessageBatchStream.js","sourceRoot":"","sources":["../../lib/utils/KafkaMessageBatchStream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAavC;;;;;;GAMG;AACH,MAAM,OAAO,uBAEX,SAAQ,SAAS;IACA,SAAS,CAAQ;IACjB,OAAO,CAAQ;IAEf,6BAA6B,CAA4B;IACzD,6BAA6B,CAA4C;IAC1F,YAAY,OAA2D;QACrE,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAA;QAC3B,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAA;QAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,mBAAmB,CAAA;QAC1C,IAAI,CAAC,6BAA6B,GAAG,EAAE,CAAA;QACvC,IAAI,CAAC,6BAA6B,GAAG,EAAE,CAAA;IACzC,CAAC;IAEQ,UAAU,CAAC,OAAiB,EAAE,SAAyB,EAAE,QAAoB;QACpF,MAAM,GAAG,GAAG,oBAAoB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,CAAA;QAElE,yBAAyB;QACzB,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,GAAG,EAAE,CAAA;QAC1F,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAErD,yCAAyC;QACzC,IAAI,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrE,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,CAAA;YAChE,QAAQ,EAAE,CAAA;YACV,OAAM;QACR,CAAC;QAED,0DAA0D;QAC1D,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,GAAG,UAAU,CAClD,GAAG,EAAE,CAAC,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,EACtE,IAAI,CAAC,OAAO,CACb,CAAA;QACH,CAAC;QAED,QAAQ,EAAE,CAAA;IACZ,CAAC;IAED,qDAAqD;IAC5C,KAAK,CAAC,MAAM,CAAC,QAAoB;QACxC,MAAM,IAAI,CAAC,eAAe,EAAE,CAAA;QAC5B,QAAQ,EAAE,CAAA;IACZ,CAAC;IAEO,eAAe;QACrB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,EAAE,CAAC;YAClE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,sBAAsB,CAAC,GAAG,CAAC,CAAA;YACxD,IAAI,CAAC,yBAAyB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;QAClD,CAAC;IACH,CAAC;IAEO,yBAAyB,CAAC,KAAa,EAAE,SAAiB;QAChE,MAAM,GAAG,GAAG,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;QAElD,gBAAgB;QAChB,IAAI,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5C,YAAY,CAAC,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,CAAC,CAAA;YACrD,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,GAAG,SAAS,CAAA;QACrD,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,IAAI,EAAE,CAAA;QAC9D,IAAI,CAAC,QAAQ,CAAC,MAAM;YAAE,OAAM;QAE5B,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAA;QACnD,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,GAAG,EAAE,CAAA;IAC9C,CAAC;IAEQ,IAAI,CACX,KAAiE,EACjE,QAAyB;QAEzB,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;IACpC,CAAC;CACF;AAED,MAAM,oBAAoB,GAAG,CAAC,KAAa,EAAE,SAAiB,EAAU,EAAE,CAAC,GAAG,KAAK,IAAI,SAAS,EAAE,CAAA;AAClG,MAAM,sBAAsB,GAAG,CAAC,GAAW,EAAwC,EAAE;IACnF,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IACzC,qBAAqB;IACrB,IAAI,CAAC,KAAK,IAAI,CAAC,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;IAC/E,oBAAoB;IAEpB,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,EAAE,CAAA;AAC7D,CAAC,CAAA"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@message-queue-toolkit/kafka",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.1-beta.2",
|
|
4
4
|
"engines": {
|
|
5
5
|
"node": ">= 22.14.0"
|
|
6
6
|
},
|
|
@@ -67,12 +67,12 @@
|
|
|
67
67
|
"@message-queue-toolkit/core": ">=23.0.0",
|
|
68
68
|
"@message-queue-toolkit/schemas": ">=7.0.0",
|
|
69
69
|
"@types/node": "^25.0.2",
|
|
70
|
-
"@vitest/coverage-v8": "^
|
|
70
|
+
"@vitest/coverage-v8": "^4.0.15",
|
|
71
71
|
"awilix": "^12.0.1",
|
|
72
72
|
"awilix-manager": "^6.0.0",
|
|
73
73
|
"rimraf": "^6.0.1",
|
|
74
74
|
"typescript": "^5.9.2",
|
|
75
|
-
"vitest": "^
|
|
75
|
+
"vitest": "^4.0.15",
|
|
76
76
|
"zod": "^4.0.17"
|
|
77
77
|
}
|
|
78
78
|
}
|