@message-queue-toolkit/kafka 0.8.3 → 0.9.1-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/AbstractKafkaConsumer.d.ts +2 -2
- package/dist/AbstractKafkaConsumer.js +90 -21
- package/dist/AbstractKafkaConsumer.js.map +1 -1
- package/dist/utils/KafkaMessageBatchStream.d.ts +12 -14
- package/dist/utils/KafkaMessageBatchStream.js +50 -45
- package/dist/utils/KafkaMessageBatchStream.js.map +1 -1
- package/package.json +3 -3
|
@@ -3,7 +3,7 @@ import { type ConsumeOptions, type ConsumerOptions } from '@platformatic/kafka';
|
|
|
3
3
|
import { AbstractKafkaService, type BaseKafkaOptions } from './AbstractKafkaService.ts';
|
|
4
4
|
import type { KafkaHandlerRouting } from './handler-routing/KafkaHandlerRoutingBuilder.ts';
|
|
5
5
|
import type { KafkaConfig, KafkaDependencies, TopicConfig } from './types.ts';
|
|
6
|
-
import {
|
|
6
|
+
import type { KafkaMessageBatchOptions } from './utils/KafkaMessageBatchStream.ts';
|
|
7
7
|
export type KafkaConsumerDependencies = KafkaDependencies & Pick<QueueConsumerDependencies, 'transactionObservabilityManager'>;
|
|
8
8
|
export type KafkaBatchProcessingOptions<BatchProcessingEnabled> = BatchProcessingEnabled extends true ? {
|
|
9
9
|
batchProcessingEnabled: true;
|
|
@@ -32,8 +32,8 @@ export declare abstract class AbstractKafkaConsumer<TopicsConfig extends TopicCo
|
|
|
32
32
|
*/
|
|
33
33
|
get isActive(): boolean;
|
|
34
34
|
init(): Promise<void>;
|
|
35
|
+
private handleAsyncBatch;
|
|
35
36
|
private handleSyncStream;
|
|
36
|
-
private handleSyncStreamBatch;
|
|
37
37
|
close(): Promise<void>;
|
|
38
38
|
private resolveHandler;
|
|
39
39
|
private consume;
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { setInterval } from 'node:timers';
|
|
2
3
|
import { setTimeout } from 'node:timers/promises';
|
|
3
4
|
import { InternalError, stringValueSerializer, } from '@lokalise/node-core';
|
|
4
5
|
import { Consumer, ProtocolError, ResponseError, stringDeserializer, } from '@platformatic/kafka';
|
|
5
6
|
import { AbstractKafkaService, } from "./AbstractKafkaService.js";
|
|
6
7
|
import { ILLEGAL_GENERATION, REBALANCE_IN_PROGRESS, UNKNOWN_MEMBER_ID } from "./utils/errorCodes.js";
|
|
7
|
-
import { KafkaMessageBatchStream, } from "./utils/KafkaMessageBatchStream.js";
|
|
8
8
|
import { safeJsonDeserializer } from "./utils/safeJsonDeserializer.js";
|
|
9
9
|
const commitErrorCodesToIgnore = new Set([
|
|
10
10
|
ILLEGAL_GENERATION,
|
|
@@ -54,12 +54,13 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
|
|
|
54
54
|
return false;
|
|
55
55
|
try {
|
|
56
56
|
return this.consumer.isConnected();
|
|
57
|
+
/* v8 ignore start */
|
|
57
58
|
}
|
|
58
59
|
catch (_) {
|
|
59
60
|
// this should not happen, but if so it means the consumer is not healthy
|
|
60
|
-
/* v8 ignore next */
|
|
61
61
|
return false;
|
|
62
62
|
}
|
|
63
|
+
/* v8 ignore stop */
|
|
63
64
|
}
|
|
64
65
|
/**
|
|
65
66
|
* Returns `true` if the consumer is not closed, and it is currently an active member of a consumer group.
|
|
@@ -71,12 +72,13 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
|
|
|
71
72
|
return false;
|
|
72
73
|
try {
|
|
73
74
|
return this.consumer.isActive();
|
|
75
|
+
/* v8 ignore start */
|
|
74
76
|
}
|
|
75
77
|
catch (_) {
|
|
76
78
|
// this should not happen, but if so it means the consumer is not healthy
|
|
77
|
-
/* v8 ignore next */
|
|
78
79
|
return false;
|
|
79
80
|
}
|
|
81
|
+
/* v8 ignore stop */
|
|
80
82
|
}
|
|
81
83
|
async init() {
|
|
82
84
|
if (this.consumerStream)
|
|
@@ -93,12 +95,46 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
|
|
|
93
95
|
heartbeatInterval: consumeOptions.heartbeatInterval,
|
|
94
96
|
});
|
|
95
97
|
this.consumerStream = await this.consumer.consume({ ...consumeOptions, topics });
|
|
98
|
+
this.consumerStream.on('error', (error) => this.handlerError(error));
|
|
96
99
|
if (this.options.batchProcessingEnabled && this.options.batchProcessingOptions) {
|
|
97
|
-
this.
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
this.
|
|
100
|
+
this.consumerStream.on('end', () => this.logger.debug('stream ended'));
|
|
101
|
+
this.consumerStream.on('close', () => this.logger.debug('stream closed'));
|
|
102
|
+
this.consumerStream.on('pause', () => this.logger.debug('stream paused'));
|
|
103
|
+
this.consumerStream.on('resume', () => this.logger.debug('stream resumed'));
|
|
104
|
+
/*this.messageBatchStream = new KafkaMessageBatchStream<
|
|
105
|
+
DeserializedMessage<SupportedMessageValues<TopicsConfig>>
|
|
106
|
+
>(
|
|
107
|
+
async (batch) => {
|
|
108
|
+
const messagesMapped: Record<string, DeserializedMessage<SupportedMessageValues<TopicsConfig>>[]> = {};
|
|
109
|
+
|
|
110
|
+
for (const item of batch) {
|
|
111
|
+
if (!messagesMapped[item.topic])
|
|
112
|
+
messagesMapped[item.topic] = [];
|
|
113
|
+
|
|
114
|
+
messagesMapped[item.topic]!.push(item);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const promises: Promise<void>[] = [];
|
|
118
|
+
|
|
119
|
+
for (const topic in messagesMapped) {
|
|
120
|
+
promises.push(this.consume(topic, messagesMapped[topic]!));
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
try {
|
|
124
|
+
await Promise.all(promises)
|
|
125
|
+
} catch (error) {
|
|
126
|
+
console.error("failed to await all consumes", error);
|
|
127
|
+
//this.handlerError(error);
|
|
128
|
+
}
|
|
129
|
+
},
|
|
130
|
+
this.options.batchProcessingOptions,
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
this.consumerStream.pipe(this.messageBatchStream)*/
|
|
134
|
+
this.handleAsyncBatch(this.consumerStream);
|
|
135
|
+
}
|
|
136
|
+
else {
|
|
137
|
+
this.handleSyncStream(this.consumerStream).catch((error) => this.handlerError(error));
|
|
102
138
|
}
|
|
103
139
|
}
|
|
104
140
|
catch (error) {
|
|
@@ -108,24 +144,58 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
|
|
|
108
144
|
cause: error,
|
|
109
145
|
});
|
|
110
146
|
}
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
147
|
+
}
|
|
148
|
+
handleAsyncBatch(stream) {
|
|
149
|
+
let messages = [];
|
|
150
|
+
let processingMessages = false;
|
|
151
|
+
const processMessages = async () => {
|
|
152
|
+
if (processingMessages)
|
|
153
|
+
return;
|
|
154
|
+
processingMessages = true;
|
|
155
|
+
const messagesMapped = {};
|
|
156
|
+
for (const message of messages) {
|
|
157
|
+
const key = `${message.topic}.${message.partition}`;
|
|
158
|
+
if (!messagesMapped[key])
|
|
159
|
+
messagesMapped[key] = [];
|
|
160
|
+
messagesMapped[key]?.push(message);
|
|
161
|
+
}
|
|
162
|
+
const promises = [];
|
|
163
|
+
for (const key in messagesMapped) {
|
|
164
|
+
// @ts-expect-error
|
|
165
|
+
promises.push(this.consume(messagesMapped[key][0].topic, messagesMapped[key]));
|
|
166
|
+
}
|
|
167
|
+
messages = [];
|
|
168
|
+
try {
|
|
169
|
+
await Promise.all(promises);
|
|
170
|
+
}
|
|
171
|
+
finally {
|
|
172
|
+
processingMessages = false;
|
|
173
|
+
}
|
|
174
|
+
};
|
|
175
|
+
setInterval(() => processMessages(), 2000);
|
|
176
|
+
stream.on('data', (message) => {
|
|
177
|
+
messages.push(message);
|
|
178
|
+
if (messages.length < 2000)
|
|
179
|
+
return;
|
|
180
|
+
stream.pause();
|
|
181
|
+
// biome-ignore lint/suspicious/noAsyncPromiseExecutor: it is for testing purposes
|
|
182
|
+
const promise = new Promise(async (resolve, reject) => {
|
|
183
|
+
try {
|
|
184
|
+
await processMessages();
|
|
185
|
+
resolve(undefined);
|
|
186
|
+
}
|
|
187
|
+
catch (error) {
|
|
188
|
+
reject(error);
|
|
189
|
+
}
|
|
190
|
+
});
|
|
191
|
+
promise.finally(() => stream.resume());
|
|
192
|
+
});
|
|
118
193
|
}
|
|
119
194
|
async handleSyncStream(stream) {
|
|
120
195
|
for await (const message of stream) {
|
|
121
196
|
await this.consume(message.topic, message);
|
|
122
197
|
}
|
|
123
198
|
}
|
|
124
|
-
async handleSyncStreamBatch(stream) {
|
|
125
|
-
for await (const messageBatch of stream) {
|
|
126
|
-
await this.consume(messageBatch.topic, messageBatch.messages);
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
199
|
async close() {
|
|
130
200
|
if (!this.consumerStream && !this.messageBatchStream) {
|
|
131
201
|
// Leaving the group in case consumer joined but streams were not created
|
|
@@ -164,7 +234,6 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
|
|
|
164
234
|
// biome-ignore lint/style/noNonNullAssertion: we check validMessages length above
|
|
165
235
|
const firstMessage = validMessages[0];
|
|
166
236
|
const requestContext = this.getRequestContext(firstMessage);
|
|
167
|
-
/* v8 ignore next */
|
|
168
237
|
const transactionId = randomUUID();
|
|
169
238
|
this.transactionObservabilityManager?.start(this.buildTransactionName(topic), transactionId);
|
|
170
239
|
const processingResult = await this.tryToConsumeWithRetries(topic,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AbstractKafkaConsumer.js","sourceRoot":"","sources":["../lib/AbstractKafkaConsumer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EACL,aAAa,EACb,qBAAqB,GAEtB,MAAM,qBAAqB,CAAA;AAK5B,OAAO,EAEL,QAAQ,EAGR,aAAa,EACb,aAAa,EACb,kBAAkB,GACnB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACL,oBAAoB,GAGrB,MAAM,2BAA2B,CAAA;AAYlC,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAA;AACpG,OAAO,EAEL,uBAAuB,GACxB,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAA;AAkCtE,MAAM,wBAAwB,GAAG,IAAI,GAAG,CAAC;IACvC,kBAAkB;IAClB,iBAAiB;IACjB,qBAAqB;CACtB,CAAC,CAAA;AAEF;;;GAGG;AACH,MAAM,qBAAqB,GAAG,CAAC,CAAA;AAE/B,MAAM,OAAgB,qBAIpB,SAAQ,oBAGT;IACkB,QAAQ,CAA0C;IAC3D,cAAc,CAAiD;IAC/D,kBAAkB,CAEzB;IAEgB,+BAA+B,CAAiC;IAChE,gBAAgB,CAAkB;IAEnD,YACE,YAAuC,EACvC,OAAqF,EACrF,gBAAkC;QAElC,KAAK,CAAC,YAAY,EAAE,OAAO,CAAC,CAAA;QAE5B,IAAI,CAAC,+BAA+B,GAAG,YAAY,CAAC,+BAA+B,CAAA;QACnF,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAA;QAExC,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC;YAC3B,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK;YACrB,GAAG,IAAI,CAAC,OAAO;YACf,UAAU,EAAE,KAAK,EAAE,4BAA4B;YAC/C,aAAa,EAAE;gBACb,GAAG,EAAE,kBAAkB;gBACvB,KAAK,EAAE,oBAAoB;gBAC3B,SAAS,EAAE,kBAAkB;gBAC7B,WAAW,EAAE,kBAAkB;aAChC;SACF,CAAC,CAAA;QAEF,MAAM,UAAU,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAA;QACnF,qBAAqB;QACrB,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,EAAE,CAC5C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,6BAA6B,CAAC,CAC7D,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,iBAAiB,EAAE,CAAC,CAAC,EAAE,EAAE,CACxC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,kDAAkD,CAAC,CAClF,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,sBAAsB,EAAE,CAAC,CAAC,EAAE,EAAE,CAC7C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,+BAA+B,CAAC,CAC/D,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,0BAA0B,EAAE,CAAC,CAAC,EAAE,EAAE,CACjD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,sBAAsB,CAAC,CACtD,CAAA;QACD,oBAAoB;IACtB,CAAC;IAED;;OAEG;IACH,IAAI,WAAW;QACb,uDAAuD;QACvD,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO,KAAK,CAAA;QAClE,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAA;QACpC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,yEAAyE;YACzE,oBAAoB;YACpB,OAAO,KAAK,CAAA;QACd,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,IAAI,QAAQ;QACV,uDAAuD;QACvD,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO,KAAK,CAAA;QAClE,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAA;QACjC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,yEAAyE;YACzE,oBAAoB;YACpB,OAAO,KAAK,CAAA;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;QACjD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QACjD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QAE9E,IAAI,CAAC;YACH,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC,OAAO,CAAA,CAAC,8CAA8C;YAEtG,0IAA0I;YAC1I,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC5B,cAAc,EAAE,cAAc,CAAC,cAAc;gBAC7C,gBAAgB,EAAE,cAAc,CAAC,gBAAgB;gBACjD,iBAAiB,EAAE,cAAc,CAAC,iBAAiB;aACpD,CAAC,CAAA;YAEF,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,GAAG,cAAc,EAAE,MAAM,EAAE,CAAC,CAAA;YAChF,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;gBAC/E,IAAI,CAAC,kBAAkB,GAAG,IAAI,uBAAuB,CAEnD;oBACA,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,SAAS;oBACxD,mBAAmB,EAAE,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,mBAAmB;iBAC7E,CAAC,CAAA;gBACF,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAA;YACnD,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,aAAa,CAAC;gBACtB,OAAO,EAAE,sBAAsB;gBAC/B,SAAS,EAAE,2BAA2B;gBACtC,KAAK,EAAE,KAAK;aACb,CAAC,CAAA;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACnE,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;QAChG,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;QACvF,CAAC;QAED,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;IACtE,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,MAAsD;QAEtD,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;YACnC,MAAM,IAAI,CAAC,OAAO,CAChB,OAAO,CAAC,KAAK,EACb,OAAoE,CACrE,CAAA;QACH,CAAC;IACH,CAAC;IACO,KAAK,CAAC,qBAAqB,CACjC,MAA0F;QAE1F,IAAI,KAAK,EAAE,MAAM,YAAY,IAAI,MAAM,EAAE,CAAC;YACxC,MAAM,IAAI,CAAC,OAAO,CAChB,YAAY,CAAC,KAAK,EAClB,YAAY,CAAC,QAAqE,CACnF,CAAA;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACrD,yEAAyE;YACzE,IAAI,IAAI,CAAC,QAAQ;gBAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAA;YAC7C,OAAM;QACR,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAA;YACjC,IAAI,CAAC,cAAc,GAAG,SAAS,CAAA;QACjC,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAA;YACrE,IAAI,CAAC,kBAAkB,GAAG,SAAS,CAAA;QACrC,CAAC;QAED,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;IAC7B,CAAC;IAEO,cAAc,CAAC,KAAoC;QACzD,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IACrC,CAAC;IAEO,KAAK,CAAC,OAAO,CACnB,KAAa,EACb,cAAoE;QAEpE,MAAM,+BAA+B,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAClD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,sBAAsB,CAAC,CAAA;QAEnF,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;QAEhD,iFAAiF;QACjF,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;QAEtD,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CACtC,aAAa,EACb,cAAc,EACd,+BAA+B,CAChC,CAAA;QAED,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,+BAA+B,CAAC,CAAA;YAC5F,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;QACpC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,kBAAkB,EAAE,aAAa,CAAC,MAAM,EAAE,EAClF,sCAAsC,CACvC,CAAA;QACH,CAAC;QAED,kFAAkF;QAClF,MAAM,YAAY,GAAG,aAAa,CAAC,CAAC,CAAE,CAAA;QACtC,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAA;QAE3D,oBAAoB;QACpB,MAAM,aAAa,GAAG,UAAU,EAAE,CAAA;QAClC,IAAI,CAAC,+BAA+B,EAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,EAAE,aAAa,CAAC,CAAA;QAE5F,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,uBAAuB,CACzD,KAAK;QACL,0EAA0E;QAC1E,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,EAClE,aAAa,CAAC,OAAO,EACrB,cAAc,CACf,CAAA;QAED,IAAI,CAAC,uBAAuB,CAAC,aAAa,EAAE,gBAAgB,EAAE,+BAA+B,CAAC,CAAA;QAE9F,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,aAAa,CAAC,CAAA;QAEzD,+EAA+E;QAC/E,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;IACpC,CAAC;IAEO,aAAa,CACnB,aAIC,EACD,cAAoE,EACpE,+BAAuC;QAEvC,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAA;QAEzF,MAAM,aAAa,GAAgE,EAAE,CAAA;QAErF,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,yEAAyE;YACzE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACnB,SAAQ;YACV,CAAC;YAED,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAEjE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;gBACzB,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,EAAE;oBACnC,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC;iBAC9C,CAAC,CAAA;gBACF,IAAI,CAAC,sBAAsB,CAAC;oBAC1B,OAAO,EAAE,OAAO;oBAChB,gBAAgB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE;oBACpE,+BAA+B;iBAChC,CAAC,CAAA;gBACF,SAAQ;YACV,CAAC;YAED,aAAa,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,IAAI,EAAE,CAAC,CAAA;QAC7D,CAAC;QAED,OAAO,aAAa,CAAA;IACtB,CAAC;IAEO,KAAK,CAAC,uBAAuB,CACnC,KAAa,EACb,cAA4C,EAC5C,OAA6E,EAC7E,cAA8B;QAE9B,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,gBAAgB,GAA4B;YAC9C,MAAM,EAAE,OAAO;YACf,WAAW,EAAE,cAAc;SAC5B,CAAA;QACD,GAAG,CAAC;YACF,qCAAqC;YACrC,IAAI,OAAO,GAAG,CAAC;gBAAE,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAA;YAE3D,gBAAgB,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,cAAc,CAAC,CAAA;YAC1F,IAAI,gBAAgB,CAAC,MAAM,KAAK,UAAU;gBAAE,MAAK;YAEjD,OAAO,EAAE,CAAA;QACX,CAAC,QAAQ,OAAO,GAAG,qBAAqB,EAAC;QAEzC,OAAO,gBAAgB,CAAA;IACzB,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,KAAa,EACb,cAA4C,EAC5C,OAA6E,EAC7E,cAA8B;QAE9B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;YAC7C,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,CAAC,OAAO,EAAE,CAAC;gBACpD,MAAM,IAAI,KAAK,CACb,6EAA6E,CAC9E,CAAA;YACH,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,OAAO,EAAE,CAAC;gBACpD,MAAM,IAAI,KAAK,CACb,iFAAiF,CAClF,CAAA;YACH,CAAC;YAED,MAAM,OAAO;YACX,mGAAmG;YACnG,cAEuC,EACvC,IAAI,CAAC,gBAAgB,EACrB,cAAc,CACf,CAAA;YACD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAA;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC;gBAChD,CAAC,CAAC,EAAE,SAAS,EAAE,cAAc,CAAC,MAAM,EAAE;gBACtC,CAAC,CAAC,EAAE,OAAO,EAAE,qBAAqB,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,CAAA;YAC5D,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE;gBACvB,KAAK;gBACL,GAAG,YAAY;aAChB,CAAC,CAAA;QACJ,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,CAAA;IACzD,CAAC;IAEO,uBAAuB,CAC7B,QAA0C,EAC1C,gBAAyC,EACzC,+BAAuC;QAEvC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,sBAAsB,CAAC;gBAC1B,OAAO;gBACP,gBAAgB;gBAChB,+BAA+B;aAChC,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,cAAoE;QACjF,IAAI,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;YAClC,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;YAEzD,wEAAwE;YACxE,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAA;QACvE,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,CAAA;QAC3C,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAkE;QAC5F,MAAM,UAAU,GAAG;YACjB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,SAAS,EAAE,OAAO,CAAC,SAAS;SAC7B,CAAA;QACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,0BAA0B,CAAC,CAAA;QAEzD,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,MAAM,EAAE,CAAA;YACtB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,gCAAgC,CAAC,CAAA;QACjE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAA;YACtD,IAAI,KAAK,YAAY,aAAa;gBAAE,OAAO,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAA;YAClF,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAEO,2BAA2B,CAAC,aAA4B;QAC9D,kFAAkF;QAClF,KAAK,MAAM,KAAK,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YACzC,IACE,KAAK,YAAY,aAAa;gBAC9B,KAAK,CAAC,OAAO;gBACb,wBAAwB,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAC3C,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,KAAK,CACf;oBACE,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,oBAAoB,EAAE,aAAa,CAAC,OAAO;oBAC3C,oBAAoB,EAAE,KAAK,CAAC,OAAO;oBACnC,KAAK,EAAE,aAAa;iBACrB,EACD,6BAA6B,KAAK,CAAC,OAAO,EAAE,CAC7C,CAAA;YACH,CAAC;iBAAM,CAAC;gBACN,yCAAyC;gBACzC,MAAM,aAAa,CAAA;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IAEO,oBAAoB,CAAC,KAAa;QACxC,MAAM,mBAAmB,GAAG,SAAS,IAAI,CAAC,WAAW,CAAC,IAAI,IAAI,KAAK,EAAE,CAAA;QACrE,OAAO,IAAI,CAAC,OAAO,CAAC,sBAAsB;YACxC,CAAC,CAAC,GAAG,mBAAmB,QAAQ;YAChC,CAAC,CAAC,mBAAmB,CAAA;IACzB,CAAC;IAEO,iBAAiB,CACvB,OAAkE;QAElE,IAAI,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC,CAAA;QACnE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,KAAK,GAAG,UAAU,EAAE,CAAA;QAE7D,OAAO;YACL,KAAK;YACL,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;gBACxB,cAAc,EAAE,KAAK;gBACrB,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;gBAC7B,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,UAAU,EAAE,OAAO,CAAC,GAAG;aACxB,CAAC;SACH,CAAA;IACH,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"AbstractKafkaConsumer.js","sourceRoot":"","sources":["../lib/AbstractKafkaConsumer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EACL,aAAa,EACb,qBAAqB,GAEtB,MAAM,qBAAqB,CAAA;AAK5B,OAAO,EAEL,QAAQ,EAIR,aAAa,EACb,aAAa,EACb,kBAAkB,GACnB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACL,oBAAoB,GAGrB,MAAM,2BAA2B,CAAA;AAYlC,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAA;AAKpG,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAA;AAkCtE,MAAM,wBAAwB,GAAG,IAAI,GAAG,CAAC;IACvC,kBAAkB;IAClB,iBAAiB;IACjB,qBAAqB;CACtB,CAAC,CAAA;AAEF;;;GAGG;AACH,MAAM,qBAAqB,GAAG,CAAC,CAAA;AAE/B,MAAM,OAAgB,qBAIpB,SAAQ,oBAGT;IACkB,QAAQ,CAA0C;IAC3D,cAAc,CAAiD;IAC/D,kBAAkB,CAEzB;IAEgB,+BAA+B,CAAiC;IAChE,gBAAgB,CAAkB;IAEnD,YACE,YAAuC,EACvC,OAAqF,EACrF,gBAAkC;QAElC,KAAK,CAAC,YAAY,EAAE,OAAO,CAAC,CAAA;QAE5B,IAAI,CAAC,+BAA+B,GAAG,YAAY,CAAC,+BAA+B,CAAA;QACnF,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAA;QAExC,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC;YAC3B,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK;YACrB,GAAG,IAAI,CAAC,OAAO;YACf,UAAU,EAAE,KAAK,EAAE,4BAA4B;YAC/C,aAAa,EAAE;gBACb,GAAG,EAAE,kBAAkB;gBACvB,KAAK,EAAE,oBAAoB;gBAC3B,SAAS,EAAE,kBAAkB;gBAC7B,WAAW,EAAE,kBAAkB;aAChC;SACF,CAAC,CAAA;QAEF,MAAM,UAAU,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAA;QACnF,qBAAqB;QACrB,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,EAAE,CAC5C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,6BAA6B,CAAC,CAC7D,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,iBAAiB,EAAE,CAAC,CAAC,EAAE,EAAE,CACxC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,kDAAkD,CAAC,CAClF,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,sBAAsB,EAAE,CAAC,CAAC,EAAE,EAAE,CAC7C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,+BAA+B,CAAC,CAC/D,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,0BAA0B,EAAE,CAAC,CAAC,EAAE,EAAE,CACjD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,sBAAsB,CAAC,CACtD,CAAA;QACD,oBAAoB;IACtB,CAAC;IAED;;OAEG;IACH,IAAI,WAAW;QACb,uDAAuD;QACvD,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO,KAAK,CAAA;QAClE,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAA;YAClC,qBAAqB;QACvB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,yEAAyE;YACzE,OAAO,KAAK,CAAA;QACd,CAAC;QACD,oBAAoB;IACtB,CAAC;IAED;;;OAGG;IACH,IAAI,QAAQ;QACV,uDAAuD;QACvD,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO,KAAK,CAAA;QAClE,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAA;YAC/B,qBAAqB;QACvB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,yEAAyE;YACzE,OAAO,KAAK,CAAA;QACd,CAAC;QACD,oBAAoB;IACtB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;QACjD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QACjD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QAE9E,IAAI,CAAC;YACH,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC,OAAO,CAAA,CAAC,8CAA8C;YAEtG,0IAA0I;YAC1I,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC5B,cAAc,EAAE,cAAc,CAAC,cAAc;gBAC7C,gBAAgB,EAAE,cAAc,CAAC,gBAAgB;gBACjD,iBAAiB,EAAE,cAAc,CAAC,iBAAiB;aACpD,CAAC,CAAA;YAEF,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,GAAG,cAAc,EAAE,MAAM,EAAE,CAAC,CAAA;YAChF,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;YAEpE,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;gBAC/E,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAA;gBACtE,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAA;gBACzE,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAA;gBACzE,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAA;gBAE3E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mEA6BmD;gBAEnD,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;YAC5C,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;YACvF,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,aAAa,CAAC;gBACtB,OAAO,EAAE,sBAAsB;gBAC/B,SAAS,EAAE,2BAA2B;gBACtC,KAAK,EAAE,KAAK;aACb,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAEO,gBAAgB,CAAC,MAAsD;QAC7E,IAAI,QAAQ,GAA8C,EAAE,CAAA;QAC5D,IAAI,kBAAkB,GAAY,KAAK,CAAA;QAEvC,MAAM,eAAe,GAAG,KAAK,IAAI,EAAE;YACjC,IAAI,kBAAkB;gBAAE,OAAM;YAE9B,kBAAkB,GAAG,IAAI,CAAA;YAEzB,MAAM,cAAc,GAA8D,EAAE,CAAA;YAEpF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,MAAM,GAAG,GAAG,GAAG,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,SAAS,EAAE,CAAA;gBAEnD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC;oBAAE,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,CAAA;gBAElD,cAAc,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;YACpC,CAAC;YAED,MAAM,QAAQ,GAAuB,EAAE,CAAA;YAEvC,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;gBACjC,mBAAmB;gBACnB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YAChF,CAAC;YAED,QAAQ,GAAG,EAAE,CAAA;YAEb,IAAI,CAAC;gBACH,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;YAC7B,CAAC;oBAAS,CAAC;gBACT,kBAAkB,GAAG,KAAK,CAAA;YAC5B,CAAC;QACH,CAAC,CAAA;QAED,WAAW,CAAC,GAAG,EAAE,CAAC,eAAe,EAAE,EAAE,IAAI,CAAC,CAAA;QAE1C,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE;YAC5B,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAEtB,IAAI,QAAQ,CAAC,MAAM,GAAG,IAAI;gBAAE,OAAM;YAElC,MAAM,CAAC,KAAK,EAAE,CAAA;YAEd,kFAAkF;YAClF,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;gBACpD,IAAI,CAAC;oBACH,MAAM,eAAe,EAAE,CAAA;oBACvB,OAAO,CAAC,SAAS,CAAC,CAAA;gBACpB,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,CAAC,KAAK,CAAC,CAAA;gBACf,CAAC;YACH,CAAC,CAAC,CAAA;YAEF,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;QACxC,CAAC,CAAC,CAAA;IACJ,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,MAAsD;QAEtD,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;YACnC,MAAM,IAAI,CAAC,OAAO,CAChB,OAAO,CAAC,KAAK,EACb,OAAoE,CACrE,CAAA;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACrD,yEAAyE;YACzE,IAAI,IAAI,CAAC,QAAQ;gBAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAA;YAC7C,OAAM;QACR,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAA;YACjC,IAAI,CAAC,cAAc,GAAG,SAAS,CAAA;QACjC,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAA;YACrE,IAAI,CAAC,kBAAkB,GAAG,SAAS,CAAA;QACrC,CAAC;QAED,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;IAC7B,CAAC;IAEO,cAAc,CAAC,KAAoC;QACzD,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IACrC,CAAC;IAEO,KAAK,CAAC,OAAO,CACnB,KAAa,EACb,cAAoE;QAEpE,MAAM,+BAA+B,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAClD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,sBAAsB,CAAC,CAAA;QAEnF,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;QAEhD,iFAAiF;QACjF,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;QAEtD,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CACtC,aAAa,EACb,cAAc,EACd,+BAA+B,CAChC,CAAA;QAED,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,+BAA+B,CAAC,CAAA;YAC5F,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;QACpC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,kBAAkB,EAAE,aAAa,CAAC,MAAM,EAAE,EAClF,sCAAsC,CACvC,CAAA;QACH,CAAC;QAED,kFAAkF;QAClF,MAAM,YAAY,GAAG,aAAa,CAAC,CAAC,CAAE,CAAA;QACtC,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAA;QAE3D,MAAM,aAAa,GAAG,UAAU,EAAE,CAAA;QAClC,IAAI,CAAC,+BAA+B,EAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,EAAE,aAAa,CAAC,CAAA;QAE5F,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,uBAAuB,CACzD,KAAK;QACL,0EAA0E;QAC1E,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,EAClE,aAAa,CAAC,OAAO,EACrB,cAAc,CACf,CAAA;QAED,IAAI,CAAC,uBAAuB,CAAC,aAAa,EAAE,gBAAgB,EAAE,+BAA+B,CAAC,CAAA;QAE9F,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,aAAa,CAAC,CAAA;QAEzD,+EAA+E;QAC/E,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;IACpC,CAAC;IAEO,aAAa,CACnB,aAIC,EACD,cAAoE,EACpE,+BAAuC;QAEvC,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAA;QAEzF,MAAM,aAAa,GAAgE,EAAE,CAAA;QAErF,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,yEAAyE;YACzE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACnB,SAAQ;YACV,CAAC;YAED,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAEjE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;gBACzB,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,EAAE;oBACnC,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC;iBAC9C,CAAC,CAAA;gBACF,IAAI,CAAC,sBAAsB,CAAC;oBAC1B,OAAO,EAAE,OAAO;oBAChB,gBAAgB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE;oBACpE,+BAA+B;iBAChC,CAAC,CAAA;gBACF,SAAQ;YACV,CAAC;YAED,aAAa,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,IAAI,EAAE,CAAC,CAAA;QAC7D,CAAC;QAED,OAAO,aAAa,CAAA;IACtB,CAAC;IAEO,KAAK,CAAC,uBAAuB,CACnC,KAAa,EACb,cAA4C,EAC5C,OAA6E,EAC7E,cAA8B;QAE9B,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,gBAAgB,GAA4B;YAC9C,MAAM,EAAE,OAAO;YACf,WAAW,EAAE,cAAc;SAC5B,CAAA;QACD,GAAG,CAAC;YACF,qCAAqC;YACrC,IAAI,OAAO,GAAG,CAAC;gBAAE,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAA;YAE3D,gBAAgB,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,cAAc,CAAC,CAAA;YAC1F,IAAI,gBAAgB,CAAC,MAAM,KAAK,UAAU;gBAAE,MAAK;YAEjD,OAAO,EAAE,CAAA;QACX,CAAC,QAAQ,OAAO,GAAG,qBAAqB,EAAC;QAEzC,OAAO,gBAAgB,CAAA;IACzB,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,KAAa,EACb,cAA4C,EAC5C,OAA6E,EAC7E,cAA8B;QAE9B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;YAC7C,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,CAAC,OAAO,EAAE,CAAC;gBACpD,MAAM,IAAI,KAAK,CACb,6EAA6E,CAC9E,CAAA;YACH,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,OAAO,EAAE,CAAC;gBACpD,MAAM,IAAI,KAAK,CACb,iFAAiF,CAClF,CAAA;YACH,CAAC;YAED,MAAM,OAAO;YACX,mGAAmG;YACnG,cAEuC,EACvC,IAAI,CAAC,gBAAgB,EACrB,cAAc,CACf,CAAA;YACD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAA;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC;gBAChD,CAAC,CAAC,EAAE,SAAS,EAAE,cAAc,CAAC,MAAM,EAAE;gBACtC,CAAC,CAAC,EAAE,OAAO,EAAE,qBAAqB,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,CAAA;YAC5D,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE;gBACvB,KAAK;gBACL,GAAG,YAAY;aAChB,CAAC,CAAA;QACJ,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,CAAA;IACzD,CAAC;IAEO,uBAAuB,CAC7B,QAA0C,EAC1C,gBAAyC,EACzC,+BAAuC;QAEvC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,sBAAsB,CAAC;gBAC1B,OAAO;gBACP,gBAAgB;gBAChB,+BAA+B;aAChC,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,cAAoE;QACjF,IAAI,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;YAClC,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;YAEzD,wEAAwE;YACxE,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAA;QACvE,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,CAAA;QAC3C,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAkE;QAC5F,MAAM,UAAU,GAAG;YACjB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,SAAS,EAAE,OAAO,CAAC,SAAS;SAC7B,CAAA;QACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,0BAA0B,CAAC,CAAA;QAEzD,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,MAAM,EAAE,CAAA;YACtB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,gCAAgC,CAAC,CAAA;QACjE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAA;YACtD,IAAI,KAAK,YAAY,aAAa;gBAAE,OAAO,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAA;YAClF,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAEO,2BAA2B,CAAC,aAA4B;QAC9D,kFAAkF;QAClF,KAAK,MAAM,KAAK,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YACzC,IACE,KAAK,YAAY,aAAa;gBAC9B,KAAK,CAAC,OAAO;gBACb,wBAAwB,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAC3C,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,KAAK,CACf;oBACE,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,oBAAoB,EAAE,aAAa,CAAC,OAAO;oBAC3C,oBAAoB,EAAE,KAAK,CAAC,OAAO;oBACnC,KAAK,EAAE,aAAa;iBACrB,EACD,6BAA6B,KAAK,CAAC,OAAO,EAAE,CAC7C,CAAA;YACH,CAAC;iBAAM,CAAC;gBACN,yCAAyC;gBACzC,MAAM,aAAa,CAAA;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IAEO,oBAAoB,CAAC,KAAa;QACxC,MAAM,mBAAmB,GAAG,SAAS,IAAI,CAAC,WAAW,CAAC,IAAI,IAAI,KAAK,EAAE,CAAA;QACrE,OAAO,IAAI,CAAC,OAAO,CAAC,sBAAsB;YACxC,CAAC,CAAC,GAAG,mBAAmB,QAAQ;YAChC,CAAC,CAAC,mBAAmB,CAAA;IACzB,CAAC;IAEO,iBAAiB,CACvB,OAAkE;QAElE,IAAI,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC,CAAA;QACnE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,KAAK,GAAG,UAAU,EAAE,CAAA;QAE7D,OAAO;YACL,KAAK;YACL,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;gBACxB,cAAc,EAAE,KAAK;gBACrB,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;gBAC7B,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,UAAU,EAAE,OAAO,CAAC,GAAG;aACxB,CAAC;SACH,CAAA;IACH,CAAC;CACF"}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
type CallbackFunction = (error?: Error | null) => void;
|
|
1
|
+
import { Transform } from 'node:stream';
|
|
3
2
|
type MessageWithTopicAndPartition = {
|
|
4
3
|
topic: string;
|
|
5
4
|
partition: number;
|
|
@@ -13,29 +12,28 @@ export type MessageBatch<TMessage> = {
|
|
|
13
12
|
partition: number;
|
|
14
13
|
messages: TMessage[];
|
|
15
14
|
};
|
|
16
|
-
export
|
|
17
|
-
on(event: string | symbol, listener: (...args: any[]) => void): this;
|
|
18
|
-
on(event: 'data', listener: (chunk: MessageBatch<TMessage>) => void): this;
|
|
19
|
-
push(chunk: MessageBatch<TMessage> | null): boolean;
|
|
20
|
-
}
|
|
15
|
+
export type OnMessageBatchCallback<TMessage> = (batch: MessageBatch<TMessage>) => Promise<void>;
|
|
21
16
|
/**
|
|
22
17
|
* Collects messages in batches based on provided batchSize and flushes them when messages amount or timeout is reached.
|
|
18
|
+
*
|
|
19
|
+
* This implementation uses Transform stream which properly handles backpressure by design.
|
|
20
|
+
* When the downstream consumer is slow, the stream will automatically pause accepting new messages
|
|
21
|
+
* until the consumer catches up, preventing memory leaks and OOM errors.
|
|
23
22
|
*/
|
|
24
|
-
export declare class KafkaMessageBatchStream<TMessage extends MessageWithTopicAndPartition> extends
|
|
23
|
+
export declare class KafkaMessageBatchStream<TMessage extends MessageWithTopicAndPartition> extends Transform {
|
|
24
|
+
private readonly onBatch;
|
|
25
25
|
private readonly batchSize;
|
|
26
26
|
private readonly timeout;
|
|
27
27
|
private readonly currentBatchPerTopicPartition;
|
|
28
28
|
private readonly batchTimeoutPerTopicPartition;
|
|
29
|
-
|
|
29
|
+
private readonly timeoutProcessingPromises;
|
|
30
|
+
constructor(onBatch: OnMessageBatchCallback<TMessage>, options: {
|
|
30
31
|
batchSize: number;
|
|
31
32
|
timeoutMilliseconds: number;
|
|
32
33
|
});
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
_final(callback: CallbackFunction): void;
|
|
34
|
+
_transform(message: TMessage, _encoding: BufferEncoding, callback: () => void): Promise<void>;
|
|
35
|
+
_flush(callback: () => void): Promise<void>;
|
|
36
36
|
private flushAllBatches;
|
|
37
37
|
private flushCurrentBatchMessages;
|
|
38
|
-
private getTopicPartitionKey;
|
|
39
|
-
private splitTopicPartitionKey;
|
|
40
38
|
}
|
|
41
39
|
export {};
|
|
@@ -1,77 +1,82 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Transform } from 'node:stream';
|
|
2
2
|
/**
|
|
3
3
|
* Collects messages in batches based on provided batchSize and flushes them when messages amount or timeout is reached.
|
|
4
|
+
*
|
|
5
|
+
* This implementation uses Transform stream which properly handles backpressure by design.
|
|
6
|
+
* When the downstream consumer is slow, the stream will automatically pause accepting new messages
|
|
7
|
+
* until the consumer catches up, preventing memory leaks and OOM errors.
|
|
4
8
|
*/
|
|
5
|
-
|
|
6
|
-
|
|
9
|
+
export class KafkaMessageBatchStream extends Transform {
|
|
10
|
+
onBatch;
|
|
7
11
|
batchSize;
|
|
8
12
|
timeout;
|
|
9
13
|
currentBatchPerTopicPartition;
|
|
10
14
|
batchTimeoutPerTopicPartition;
|
|
11
|
-
|
|
15
|
+
timeoutProcessingPromises = new Map();
|
|
16
|
+
constructor(onBatch, options) {
|
|
12
17
|
super({ objectMode: true });
|
|
18
|
+
this.onBatch = onBatch;
|
|
13
19
|
this.batchSize = options.batchSize;
|
|
14
20
|
this.timeout = options.timeoutMilliseconds;
|
|
15
21
|
this.currentBatchPerTopicPartition = {};
|
|
16
22
|
this.batchTimeoutPerTopicPartition = {};
|
|
17
23
|
}
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
else {
|
|
27
|
-
// biome-ignore lint/style/noNonNullAssertion: non-existing entry is handled above
|
|
28
|
-
this.currentBatchPerTopicPartition[key].push(message);
|
|
24
|
+
async _transform(message, _encoding, callback) {
|
|
25
|
+
const key = getTopicPartitionKey(message.topic, message.partition);
|
|
26
|
+
// Wait for all pending timeout flushes to complete to maintain backpressure
|
|
27
|
+
if (this.timeoutProcessingPromises.size > 0) {
|
|
28
|
+
// Capture a snapshot of current promises to avoid race conditions with new timeouts
|
|
29
|
+
const promiseEntries = Array.from(this.timeoutProcessingPromises.entries());
|
|
30
|
+
// Wait for all to complete and then clean up from the map
|
|
31
|
+
await Promise.all(promiseEntries.map(([k, p]) => p.finally(() => this.timeoutProcessingPromises.delete(k))));
|
|
29
32
|
}
|
|
30
|
-
//
|
|
33
|
+
// Accumulate the message
|
|
34
|
+
if (!this.currentBatchPerTopicPartition[key])
|
|
35
|
+
this.currentBatchPerTopicPartition[key] = [];
|
|
36
|
+
this.currentBatchPerTopicPartition[key].push(message);
|
|
37
|
+
// Check if the batch is complete by size
|
|
31
38
|
if (this.currentBatchPerTopicPartition[key].length >= this.batchSize) {
|
|
32
|
-
this.flushCurrentBatchMessages(message.topic, message.partition);
|
|
33
|
-
|
|
39
|
+
await this.flushCurrentBatchMessages(message.topic, message.partition);
|
|
40
|
+
callback();
|
|
41
|
+
return;
|
|
34
42
|
}
|
|
43
|
+
// Start timeout for this partition if not already started
|
|
35
44
|
if (!this.batchTimeoutPerTopicPartition[key]) {
|
|
36
|
-
this.batchTimeoutPerTopicPartition[key] = setTimeout(() =>
|
|
37
|
-
this.flushCurrentBatchMessages(message.topic, message.partition);
|
|
38
|
-
}, this.timeout);
|
|
45
|
+
this.batchTimeoutPerTopicPartition[key] = setTimeout(() => this.timeoutProcessingPromises.set(key, this.flushCurrentBatchMessages(message.topic, message.partition)), this.timeout);
|
|
39
46
|
}
|
|
40
|
-
callback(
|
|
47
|
+
callback();
|
|
41
48
|
}
|
|
42
|
-
//
|
|
43
|
-
|
|
44
|
-
this.flushAllBatches();
|
|
45
|
-
this.push(null); // End readable side
|
|
49
|
+
// Flush all remaining batches when stream is closing
|
|
50
|
+
async _flush(callback) {
|
|
51
|
+
await this.flushAllBatches();
|
|
46
52
|
callback();
|
|
47
53
|
}
|
|
48
|
-
flushAllBatches() {
|
|
54
|
+
async flushAllBatches() {
|
|
49
55
|
for (const key of Object.keys(this.currentBatchPerTopicPartition)) {
|
|
50
|
-
const { topic, partition } =
|
|
51
|
-
this.flushCurrentBatchMessages(topic, partition);
|
|
56
|
+
const { topic, partition } = splitTopicPartitionKey(key);
|
|
57
|
+
await this.flushCurrentBatchMessages(topic, partition);
|
|
52
58
|
}
|
|
53
59
|
}
|
|
54
|
-
flushCurrentBatchMessages(topic, partition) {
|
|
55
|
-
const key =
|
|
60
|
+
async flushCurrentBatchMessages(topic, partition) {
|
|
61
|
+
const key = getTopicPartitionKey(topic, partition);
|
|
62
|
+
// Clear timeout
|
|
56
63
|
if (this.batchTimeoutPerTopicPartition[key]) {
|
|
57
64
|
clearTimeout(this.batchTimeoutPerTopicPartition[key]);
|
|
58
65
|
this.batchTimeoutPerTopicPartition[key] = undefined;
|
|
59
66
|
}
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
}
|
|
63
|
-
this.push({ topic, partition, messages: this.currentBatchPerTopicPartition[key] });
|
|
67
|
+
const messages = this.currentBatchPerTopicPartition[key] ?? [];
|
|
68
|
+
// Push the batch downstream
|
|
69
|
+
await this.onBatch({ topic, partition, messages });
|
|
64
70
|
this.currentBatchPerTopicPartition[key] = [];
|
|
65
71
|
}
|
|
66
|
-
getTopicPartitionKey(topic, partition) {
|
|
67
|
-
return `${topic}:${partition}`;
|
|
68
|
-
}
|
|
69
|
-
splitTopicPartitionKey(key) {
|
|
70
|
-
const [topic, partition] = key.split(':');
|
|
71
|
-
if (!topic || !partition) {
|
|
72
|
-
throw new Error('Invalid topic-partition key format');
|
|
73
|
-
}
|
|
74
|
-
return { topic, partition: Number.parseInt(partition, 10) };
|
|
75
|
-
}
|
|
76
72
|
}
|
|
73
|
+
const getTopicPartitionKey = (topic, partition) => `${topic}:${partition}`;
|
|
74
|
+
const splitTopicPartitionKey = (key) => {
|
|
75
|
+
const [topic, partition] = key.split(':');
|
|
76
|
+
/* v8 ignore start */
|
|
77
|
+
if (!topic || !partition)
|
|
78
|
+
throw new Error('Invalid topic-partition key format');
|
|
79
|
+
/* v8 ignore stop */
|
|
80
|
+
return { topic, partition: Number.parseInt(partition, 10) };
|
|
81
|
+
};
|
|
77
82
|
//# sourceMappingURL=KafkaMessageBatchStream.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"KafkaMessageBatchStream.js","sourceRoot":"","sources":["../../lib/utils/KafkaMessageBatchStream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"KafkaMessageBatchStream.js","sourceRoot":"","sources":["../../lib/utils/KafkaMessageBatchStream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAavC;;;;;;GAMG;AACH,MAAM,OAAO,uBAEX,SAAQ,SAAS;IACA,OAAO,CAAkC;IACzC,SAAS,CAAQ;IACjB,OAAO,CAAQ;IAEf,6BAA6B,CAA4B;IACzD,6BAA6B,CAA4C;IAEzE,yBAAyB,GAA+B,IAAI,GAAG,EAAE,CAAA;IAElF,YACE,OAAyC,EACzC,OAA2D;QAE3D,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAA;QAC3B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAA;QACtB,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAA;QAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,mBAAmB,CAAA;QAC1C,IAAI,CAAC,6BAA6B,GAAG,EAAE,CAAA;QACvC,IAAI,CAAC,6BAA6B,GAAG,EAAE,CAAA;IACzC,CAAC;IAEQ,KAAK,CAAC,UAAU,CAAC,OAAiB,EAAE,SAAyB,EAAE,QAAoB;QAC1F,MAAM,GAAG,GAAG,oBAAoB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,CAAA;QAElE,4EAA4E;QAC5E,IAAI,IAAI,CAAC,yBAAyB,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC5C,oFAAoF;YACpF,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,yBAAyB,CAAC,OAAO,EAAE,CAAC,CAAA;YAC3E,0DAA0D;YAC1D,MAAM,OAAO,CAAC,GAAG,CACf,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAC1F,CAAA;QACH,CAAC;QAED,yBAAyB;QACzB,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,GAAG,EAAE,CAAA;QAC1F,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAErD,yCAAyC;QACzC,IAAI,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrE,MAAM,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,CAAA;YACtE,QAAQ,EAAE,CAAA;YACV,OAAM;QACR,CAAC;QAED,0DAA0D;QAC1D,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,GAAG,UAAU,CAClD,GAAG,EAAE,CACH,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAChC,GAAG,EACH,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,CACjE,EACH,IAAI,CAAC,OAAO,CACb,CAAA;QACH,CAAC;QAED,QAAQ,EAAE,CAAA;IACZ,CAAC;IAED,qDAAqD;IAC5C,KAAK,CAAC,MAAM,CAAC,QAAoB;QACxC,MAAM,IAAI,CAAC,eAAe,EAAE,CAAA;QAC5B,QAAQ,EAAE,CAAA;IACZ,CAAC;IAEO,KAAK,CAAC,eAAe;QAC3B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,EAAE,CAAC;YAClE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,sBAAsB,CAAC,GAAG,CAAC,CAAA;YACxD,MAAM,IAAI,CAAC,yBAAyB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;QACxD,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,yBAAyB,CAAC,KAAa,EAAE,SAAiB;QACtE,MAAM,GAAG,GAAG,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;QAElD,gBAAgB;QAChB,IAAI,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5C,YAAY,CAAC,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,CAAC,CAAA;YACrD,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,GAAG,SAAS,CAAA;QACrD,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,IAAI,EAAE,CAAA;QAE9D,4BAA4B;QAC5B,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAA;QAClD,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,GAAG,EAAE,CAAA;IAC9C,CAAC;CACF;AAED,MAAM,oBAAoB,GAAG,CAAC,KAAa,EAAE,SAAiB,EAAU,EAAE,CAAC,GAAG,KAAK,IAAI,SAAS,EAAE,CAAA;AAClG,MAAM,sBAAsB,GAAG,CAAC,GAAW,EAAwC,EAAE;IACnF,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IACzC,qBAAqB;IACrB,IAAI,CAAC,KAAK,IAAI,CAAC,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;IAC/E,oBAAoB;IAEpB,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,EAAE,CAAA;AAC7D,CAAC,CAAA"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@message-queue-toolkit/kafka",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.9.1-beta.1",
|
|
4
4
|
"engines": {
|
|
5
5
|
"node": ">= 22.14.0"
|
|
6
6
|
},
|
|
@@ -67,12 +67,12 @@
|
|
|
67
67
|
"@message-queue-toolkit/core": ">=23.0.0",
|
|
68
68
|
"@message-queue-toolkit/schemas": ">=7.0.0",
|
|
69
69
|
"@types/node": "^25.0.2",
|
|
70
|
-
"@vitest/coverage-v8": "^
|
|
70
|
+
"@vitest/coverage-v8": "^4.0.15",
|
|
71
71
|
"awilix": "^12.0.1",
|
|
72
72
|
"awilix-manager": "^6.0.0",
|
|
73
73
|
"rimraf": "^6.0.1",
|
|
74
74
|
"typescript": "^5.9.2",
|
|
75
|
-
"vitest": "^
|
|
75
|
+
"vitest": "^4.0.15",
|
|
76
76
|
"zod": "^4.0.17"
|
|
77
77
|
}
|
|
78
78
|
}
|