@message-queue-toolkit/kafka 0.6.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -4
- package/dist/AbstractKafkaConsumer.d.ts +19 -6
- package/dist/AbstractKafkaConsumer.js +153 -65
- package/dist/AbstractKafkaConsumer.js.map +1 -1
- package/dist/AbstractKafkaPublisher.js +3 -4
- package/dist/AbstractKafkaPublisher.js.map +1 -1
- package/dist/AbstractKafkaService.d.ts +3 -6
- package/dist/AbstractKafkaService.js +3 -12
- package/dist/AbstractKafkaService.js.map +1 -1
- package/dist/handler-routing/KafkaHandlerConfig.d.ts +10 -0
- package/dist/{handler-container → handler-routing}/KafkaHandlerConfig.js +2 -0
- package/dist/handler-routing/KafkaHandlerConfig.js.map +1 -0
- package/dist/handler-routing/KafkaHandlerRoutingBuilder.d.ts +10 -0
- package/dist/{handler-container → handler-routing}/KafkaHandlerRoutingBuilder.js +1 -2
- package/dist/handler-routing/KafkaHandlerRoutingBuilder.js.map +1 -0
- package/dist/handler-routing/index.js.map +1 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/types.d.ts +5 -4
- package/dist/utils/KafkaMessageBatchStream.d.ts +41 -0
- package/dist/utils/KafkaMessageBatchStream.js +77 -0
- package/dist/utils/KafkaMessageBatchStream.js.map +1 -0
- package/package.json +10 -6
- package/dist/handler-container/KafkaHandlerConfig.d.ts +0 -9
- package/dist/handler-container/KafkaHandlerConfig.js.map +0 -1
- package/dist/handler-container/KafkaHandlerContainer.d.ts +0 -11
- package/dist/handler-container/KafkaHandlerContainer.js +0 -45
- package/dist/handler-container/KafkaHandlerContainer.js.map +0 -1
- package/dist/handler-container/KafkaHandlerRoutingBuilder.d.ts +0 -8
- package/dist/handler-container/KafkaHandlerRoutingBuilder.js.map +0 -1
- package/dist/handler-container/index.js.map +0 -1
- /package/dist/{handler-container → handler-routing}/index.d.ts +0 -0
- /package/dist/{handler-container → handler-routing}/index.js +0 -0
package/README.md
CHANGED
|
@@ -1,7 +1,40 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Kafka
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
This library provides utilities for implementing Kafka consumers and publishers.
|
|
4
|
+
While following the same patterns as other message broker implementations,
|
|
5
|
+
Kafka's unique characteristics require some specific adaptations in the publisher and consumer definitions.
|
|
4
6
|
|
|
7
|
+
> **_NOTE:_** Check [README.md](../../README.md) for transport-agnostic library documentation.
|
|
5
8
|
|
|
6
|
-
##
|
|
7
|
-
|
|
9
|
+
## Publishers
|
|
10
|
+
|
|
11
|
+
Use `AbstractKafkaPublisher` as a base class for publisher implementation.
|
|
12
|
+
|
|
13
|
+
See [test publisher](test/publisher/PermissionPublisher.ts) for an example of implementation.
|
|
14
|
+
|
|
15
|
+
## Consumers
|
|
16
|
+
|
|
17
|
+
Use `AbstractKafkaConsumer` as a base class for consumer implementation.
|
|
18
|
+
|
|
19
|
+
See [test consumer](test/consumer/PermissionConsumer.ts) for an example of implementation.
|
|
20
|
+
|
|
21
|
+
## Batch Processing
|
|
22
|
+
|
|
23
|
+
Kafka supports batch processing for improved throughput. To enable it, set `batchProcessingEnabled` to `true` and configure `batchProcessingOptions`.
|
|
24
|
+
|
|
25
|
+
When batch processing is enabled, message handlers receive an array of messages instead of a single message.
|
|
26
|
+
|
|
27
|
+
### Configuration Options
|
|
28
|
+
|
|
29
|
+
- `batchSize` - Maximum number of messages per batch
|
|
30
|
+
- `timeoutMilliseconds` - Maximum time to wait for a batch to fill before processing
|
|
31
|
+
|
|
32
|
+
### How It Works
|
|
33
|
+
|
|
34
|
+
Messages are buffered per topic-partition combination. Batches are processed when either:
|
|
35
|
+
- The buffer reaches the configured `batchSize`
|
|
36
|
+
- The `timeoutMilliseconds` timeout is reached since the first message was added
|
|
37
|
+
|
|
38
|
+
After successful batch processing, the offset of the last message in the batch is committed.
|
|
39
|
+
|
|
40
|
+
See [test batch consumer](test/consumer/PermissionBatchConsumer.ts) for an example of implementation.
|
|
@@ -1,19 +1,27 @@
|
|
|
1
1
|
import type { QueueConsumerDependencies } from '@message-queue-toolkit/core';
|
|
2
2
|
import { type ConsumeOptions, type ConsumerOptions } from '@platformatic/kafka';
|
|
3
3
|
import { AbstractKafkaService, type BaseKafkaOptions } from './AbstractKafkaService.ts';
|
|
4
|
-
import type { KafkaHandlerRouting } from './handler-
|
|
4
|
+
import type { KafkaHandlerRouting } from './handler-routing/KafkaHandlerRoutingBuilder.ts';
|
|
5
5
|
import type { KafkaConfig, KafkaDependencies, TopicConfig } from './types.ts';
|
|
6
|
+
import { type KafkaMessageBatchOptions } from './utils/KafkaMessageBatchStream.js';
|
|
6
7
|
export type KafkaConsumerDependencies = KafkaDependencies & Pick<QueueConsumerDependencies, 'transactionObservabilityManager'>;
|
|
7
|
-
export type
|
|
8
|
-
|
|
8
|
+
export type KafkaBatchProcessingOptions<BatchProcessingEnabled> = BatchProcessingEnabled extends true ? {
|
|
9
|
+
batchProcessingEnabled: true;
|
|
10
|
+
batchProcessingOptions: KafkaMessageBatchOptions;
|
|
11
|
+
} : {
|
|
12
|
+
batchProcessingEnabled: false;
|
|
13
|
+
batchProcessingOptions?: never;
|
|
9
14
|
};
|
|
10
|
-
export
|
|
15
|
+
export type KafkaConsumerOptions<TopicsConfig extends TopicConfig[], ExecutionContext, BatchProcessingEnabled extends boolean> = BaseKafkaOptions & Omit<ConsumerOptions<string, object, string, string>, 'deserializers' | 'autocommit' | keyof KafkaConfig> & Omit<ConsumeOptions<string, object, string, string>, 'topics'> & KafkaBatchProcessingOptions<BatchProcessingEnabled> & {
|
|
16
|
+
handlers: KafkaHandlerRouting<TopicsConfig, ExecutionContext, BatchProcessingEnabled>;
|
|
17
|
+
};
|
|
18
|
+
export declare abstract class AbstractKafkaConsumer<TopicsConfig extends TopicConfig[], ExecutionContext, BatchProcessingEnabled extends boolean = false> extends AbstractKafkaService<TopicsConfig, KafkaConsumerOptions<TopicsConfig, ExecutionContext, BatchProcessingEnabled>> {
|
|
11
19
|
private readonly consumer;
|
|
12
20
|
private consumerStream?;
|
|
21
|
+
private messageBatchStream?;
|
|
13
22
|
private readonly transactionObservabilityManager;
|
|
14
|
-
private readonly handlerContainer;
|
|
15
23
|
private readonly executionContext;
|
|
16
|
-
constructor(dependencies: KafkaConsumerDependencies, options: KafkaConsumerOptions<TopicsConfig, ExecutionContext>, executionContext: ExecutionContext);
|
|
24
|
+
constructor(dependencies: KafkaConsumerDependencies, options: KafkaConsumerOptions<TopicsConfig, ExecutionContext, BatchProcessingEnabled>, executionContext: ExecutionContext);
|
|
17
25
|
/**
|
|
18
26
|
* Returns true if all client's connections are currently connected and the client is connected to at least one broker.
|
|
19
27
|
*/
|
|
@@ -25,8 +33,13 @@ export declare abstract class AbstractKafkaConsumer<TopicsConfig extends TopicCo
|
|
|
25
33
|
get isActive(): boolean;
|
|
26
34
|
init(): Promise<void>;
|
|
27
35
|
close(): Promise<void>;
|
|
36
|
+
private resolveHandler;
|
|
28
37
|
private consume;
|
|
38
|
+
private parseMessages;
|
|
39
|
+
private tryToConsumeWithRetries;
|
|
29
40
|
private tryToConsume;
|
|
41
|
+
private handleMessagesProcessed;
|
|
42
|
+
private commit;
|
|
30
43
|
private commitMessage;
|
|
31
44
|
private handleResponseErrorOnCommit;
|
|
32
45
|
private buildTransactionName;
|
|
@@ -2,9 +2,9 @@ import { randomUUID } from 'node:crypto';
|
|
|
2
2
|
import { setTimeout } from 'node:timers/promises';
|
|
3
3
|
import { InternalError, stringValueSerializer, } from '@lokalise/node-core';
|
|
4
4
|
import { Consumer, ProtocolError, ResponseError, stringDeserializer, } from '@platformatic/kafka';
|
|
5
|
-
import { AbstractKafkaService } from "./AbstractKafkaService.js";
|
|
6
|
-
import { KafkaHandlerContainer } from "./handler-container/KafkaHandlerContainer.js";
|
|
5
|
+
import { AbstractKafkaService, } from "./AbstractKafkaService.js";
|
|
7
6
|
import { ILLEGAL_GENERATION, REBALANCE_IN_PROGRESS, UNKNOWN_MEMBER_ID } from "./utils/errorCodes.js";
|
|
7
|
+
import { KafkaMessageBatchStream, } from './utils/KafkaMessageBatchStream.js';
|
|
8
8
|
import { safeJsonDeserializer } from "./utils/safeJsonDeserializer.js";
|
|
9
9
|
const commitErrorCodesToIgnore = new Set([
|
|
10
10
|
ILLEGAL_GENERATION,
|
|
@@ -19,13 +19,12 @@ const MAX_IN_MEMORY_RETRIES = 3;
|
|
|
19
19
|
export class AbstractKafkaConsumer extends AbstractKafkaService {
|
|
20
20
|
consumer;
|
|
21
21
|
consumerStream;
|
|
22
|
+
messageBatchStream;
|
|
22
23
|
transactionObservabilityManager;
|
|
23
|
-
handlerContainer;
|
|
24
24
|
executionContext;
|
|
25
25
|
constructor(dependencies, options, executionContext) {
|
|
26
26
|
super(dependencies, options);
|
|
27
27
|
this.transactionObservabilityManager = dependencies.transactionObservabilityManager;
|
|
28
|
-
this.handlerContainer = new KafkaHandlerContainer(options.handlers, options.messageTypeField);
|
|
29
28
|
this.executionContext = executionContext;
|
|
30
29
|
this.consumer = new Consumer({
|
|
31
30
|
...this.options.kafka,
|
|
@@ -50,23 +49,43 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
|
|
|
50
49
|
* Returns true if all client's connections are currently connected and the client is connected to at least one broker.
|
|
51
50
|
*/
|
|
52
51
|
get isConnected() {
|
|
53
|
-
|
|
52
|
+
// Streams are created only when init method was called
|
|
53
|
+
if (!this.consumerStream && !this.messageBatchStream)
|
|
54
|
+
return false;
|
|
55
|
+
try {
|
|
56
|
+
return this.consumer.isConnected();
|
|
57
|
+
}
|
|
58
|
+
catch (_) {
|
|
59
|
+
// this should not happen, but if so it means the consumer is not healthy
|
|
60
|
+
/* v8 ignore next */
|
|
61
|
+
return false;
|
|
62
|
+
}
|
|
54
63
|
}
|
|
55
64
|
/**
|
|
56
65
|
* Returns `true` if the consumer is not closed, and it is currently an active member of a consumer group.
|
|
57
66
|
* This method will return `false` during consumer group rebalancing.
|
|
58
67
|
*/
|
|
59
68
|
get isActive() {
|
|
60
|
-
|
|
69
|
+
// Streams are created only when init method was called
|
|
70
|
+
if (!this.consumerStream && !this.messageBatchStream)
|
|
71
|
+
return false;
|
|
72
|
+
try {
|
|
73
|
+
return this.consumer.isActive();
|
|
74
|
+
}
|
|
75
|
+
catch (_) {
|
|
76
|
+
// this should not happen, but if so it means the consumer is not healthy
|
|
77
|
+
/* v8 ignore next */
|
|
78
|
+
return false;
|
|
79
|
+
}
|
|
61
80
|
}
|
|
62
81
|
async init() {
|
|
63
82
|
if (this.consumerStream)
|
|
64
83
|
return Promise.resolve();
|
|
65
|
-
const topics = this.
|
|
84
|
+
const topics = Object.keys(this.options.handlers);
|
|
66
85
|
if (topics.length === 0)
|
|
67
86
|
throw new Error('At least one topic must be defined');
|
|
68
87
|
try {
|
|
69
|
-
const { handlers, ...consumeOptions } = this.options; // Handlers cannot be passed to consume method
|
|
88
|
+
const { handlers: _, ...consumeOptions } = this.options; // Handlers cannot be passed to consume method
|
|
70
89
|
// https://github.com/platformatic/kafka/blob/main/docs/consumer.md#my-consumer-is-not-receiving-any-message-when-the-application-restarts
|
|
71
90
|
await this.consumer.joinGroup({
|
|
72
91
|
sessionTimeout: consumeOptions.sessionTimeout,
|
|
@@ -74,6 +93,13 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
|
|
|
74
93
|
heartbeatInterval: consumeOptions.heartbeatInterval,
|
|
75
94
|
});
|
|
76
95
|
this.consumerStream = await this.consumer.consume({ ...consumeOptions, topics });
|
|
96
|
+
if (this.options.batchProcessingEnabled && this.options.batchProcessingOptions) {
|
|
97
|
+
this.messageBatchStream = new KafkaMessageBatchStream({
|
|
98
|
+
batchSize: this.options.batchProcessingOptions.batchSize,
|
|
99
|
+
timeoutMilliseconds: this.options.batchProcessingOptions.timeoutMilliseconds,
|
|
100
|
+
});
|
|
101
|
+
this.consumerStream.pipe(this.messageBatchStream);
|
|
102
|
+
}
|
|
77
103
|
}
|
|
78
104
|
catch (error) {
|
|
79
105
|
throw new InternalError({
|
|
@@ -82,83 +108,146 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
|
|
|
82
108
|
cause: error,
|
|
83
109
|
});
|
|
84
110
|
}
|
|
85
|
-
this.
|
|
111
|
+
if (this.options.batchProcessingEnabled && this.messageBatchStream) {
|
|
112
|
+
this.messageBatchStream.on('data', async (messageBatch) => this.consume(messageBatch.topic, messageBatch.messages));
|
|
113
|
+
this.messageBatchStream.on('error', (error) => this.handlerError(error));
|
|
114
|
+
}
|
|
115
|
+
else {
|
|
116
|
+
this.consumerStream.on('data', (message) => this.consume(message.topic, message));
|
|
117
|
+
}
|
|
86
118
|
this.consumerStream.on('error', (error) => this.handlerError(error));
|
|
87
119
|
}
|
|
88
120
|
async close() {
|
|
89
|
-
if (!this.consumerStream)
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
121
|
+
if (!this.consumerStream && !this.messageBatchStream) {
|
|
122
|
+
// Leaving the group in case consumer joined but streams were not created
|
|
123
|
+
if (this.isActive)
|
|
124
|
+
this.consumer.leaveGroup();
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
if (this.consumerStream) {
|
|
128
|
+
await this.consumerStream.close();
|
|
129
|
+
this.consumerStream = undefined;
|
|
130
|
+
}
|
|
131
|
+
if (this.messageBatchStream) {
|
|
132
|
+
await new Promise((resolve) => this.messageBatchStream?.end(resolve));
|
|
133
|
+
this.messageBatchStream = undefined;
|
|
134
|
+
}
|
|
93
135
|
await this.consumer.close();
|
|
94
136
|
}
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
137
|
+
resolveHandler(topic) {
|
|
138
|
+
return this.options.handlers[topic];
|
|
139
|
+
}
|
|
140
|
+
async consume(topic, messageOrBatch) {
|
|
99
141
|
const messageProcessingStartTimestamp = Date.now();
|
|
100
|
-
const
|
|
142
|
+
const handlerConfig = this.resolveHandler(topic);
|
|
101
143
|
// if there is no handler for the message, we ignore it (simulating subscription)
|
|
102
|
-
if (!
|
|
103
|
-
return this.
|
|
144
|
+
if (!handlerConfig)
|
|
145
|
+
return this.commit(messageOrBatch);
|
|
146
|
+
const validMessages = this.parseMessages(handlerConfig, messageOrBatch, messageProcessingStartTimestamp);
|
|
147
|
+
if (!validMessages.length) {
|
|
148
|
+
return this.commit(messageOrBatch);
|
|
149
|
+
}
|
|
150
|
+
// biome-ignore lint/style/noNonNullAssertion: we check validMessages length above
|
|
151
|
+
const firstMessage = validMessages[0];
|
|
152
|
+
const requestContext = this.getRequestContext(firstMessage);
|
|
104
153
|
/* v8 ignore next */
|
|
105
|
-
const transactionId =
|
|
106
|
-
this.transactionObservabilityManager?.start(this.buildTransactionName(
|
|
107
|
-
const
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
154
|
+
const transactionId = randomUUID();
|
|
155
|
+
this.transactionObservabilityManager?.start(this.buildTransactionName(topic), transactionId);
|
|
156
|
+
const processingResult = await this.tryToConsumeWithRetries(topic,
|
|
157
|
+
// If batch processing is disabled, we have only single message to process
|
|
158
|
+
this.options.batchProcessingEnabled ? validMessages : firstMessage, handlerConfig.handler, requestContext);
|
|
159
|
+
this.handleMessagesProcessed(validMessages, processingResult, messageProcessingStartTimestamp);
|
|
160
|
+
this.transactionObservabilityManager?.stop(transactionId);
|
|
161
|
+
// We commit all messages, even if some of them were filtered out on validation
|
|
162
|
+
return this.commit(messageOrBatch);
|
|
163
|
+
}
|
|
164
|
+
parseMessages(handlerConfig, messageOrBatch, messageProcessingStartTimestamp) {
|
|
165
|
+
const messagesToCheck = Array.isArray(messageOrBatch) ? messageOrBatch : [messageOrBatch];
|
|
166
|
+
const validMessages = [];
|
|
167
|
+
for (const message of messagesToCheck) {
|
|
168
|
+
// message.value can be undefined if the message is not JSON-serializable
|
|
169
|
+
if (!message.value) {
|
|
170
|
+
continue;
|
|
171
|
+
}
|
|
172
|
+
const parseResult = handlerConfig.schema.safeParse(message.value);
|
|
173
|
+
if (!parseResult.success) {
|
|
174
|
+
this.handlerError(parseResult.error, {
|
|
175
|
+
topic: message.topic,
|
|
176
|
+
message: stringValueSerializer(message.value),
|
|
177
|
+
});
|
|
178
|
+
this.handleMessageProcessed({
|
|
179
|
+
message: message,
|
|
180
|
+
processingResult: { status: 'error', errorReason: 'invalidMessage' },
|
|
181
|
+
messageProcessingStartTimestamp,
|
|
182
|
+
});
|
|
183
|
+
continue;
|
|
184
|
+
}
|
|
185
|
+
validMessages.push({ ...message, value: parseResult.data });
|
|
119
186
|
}
|
|
120
|
-
|
|
121
|
-
|
|
187
|
+
return validMessages;
|
|
188
|
+
}
|
|
189
|
+
async tryToConsumeWithRetries(topic, messageOrBatch, handler, requestContext) {
|
|
122
190
|
let retries = 0;
|
|
123
|
-
let
|
|
191
|
+
let processingResult = {
|
|
192
|
+
status: 'error',
|
|
193
|
+
errorReason: 'handlerError',
|
|
194
|
+
};
|
|
124
195
|
do {
|
|
125
196
|
// exponential backoff -> 2^(retry-1)
|
|
126
197
|
if (retries > 0)
|
|
127
198
|
await setTimeout(Math.pow(2, retries - 1));
|
|
128
|
-
|
|
129
|
-
if (consumed)
|
|
199
|
+
processingResult = await this.tryToConsume(topic, messageOrBatch, handler, requestContext);
|
|
200
|
+
if (processingResult.status === 'consumed')
|
|
130
201
|
break;
|
|
131
202
|
retries++;
|
|
132
203
|
} while (retries < MAX_IN_MEMORY_RETRIES);
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
204
|
+
return processingResult;
|
|
205
|
+
}
|
|
206
|
+
async tryToConsume(topic, messageOrBatch, handler, requestContext) {
|
|
207
|
+
try {
|
|
208
|
+
const isBatch = Array.isArray(messageOrBatch);
|
|
209
|
+
if (this.options.batchProcessingEnabled && !isBatch) {
|
|
210
|
+
throw new Error('Batch processing is enabled, but a single message was passed to the handler');
|
|
211
|
+
}
|
|
212
|
+
if (!this.options.batchProcessingEnabled && isBatch) {
|
|
213
|
+
throw new Error('Batch processing is disabled, but a batch of messages was passed to the handler');
|
|
214
|
+
}
|
|
215
|
+
await handler(
|
|
216
|
+
// We need casting to match message type with handler type - it is safe as we verify the type above
|
|
217
|
+
messageOrBatch, this.executionContext, requestContext);
|
|
218
|
+
return { status: 'consumed' };
|
|
219
|
+
}
|
|
220
|
+
catch (error) {
|
|
221
|
+
const errorContext = Array.isArray(messageOrBatch)
|
|
222
|
+
? { batchSize: messageOrBatch.length }
|
|
223
|
+
: { message: stringValueSerializer(messageOrBatch.value) };
|
|
224
|
+
this.handlerError(error, {
|
|
225
|
+
topic,
|
|
226
|
+
...errorContext,
|
|
138
227
|
});
|
|
139
228
|
}
|
|
140
|
-
|
|
229
|
+
return { status: 'error', errorReason: 'handlerError' };
|
|
230
|
+
}
|
|
231
|
+
handleMessagesProcessed(messages, processingResult, messageProcessingStartTimestamp) {
|
|
232
|
+
for (const message of messages) {
|
|
141
233
|
this.handleMessageProcessed({
|
|
142
|
-
message
|
|
143
|
-
processingResult
|
|
234
|
+
message,
|
|
235
|
+
processingResult,
|
|
144
236
|
messageProcessingStartTimestamp,
|
|
145
237
|
});
|
|
146
238
|
}
|
|
147
|
-
this.transactionObservabilityManager?.stop(transactionId);
|
|
148
|
-
return this.commitMessage(validatedMessage);
|
|
149
239
|
}
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
240
|
+
commit(messageOrBatch) {
|
|
241
|
+
if (Array.isArray(messageOrBatch)) {
|
|
242
|
+
if (messageOrBatch.length === 0) {
|
|
243
|
+
return Promise.resolve();
|
|
244
|
+
}
|
|
245
|
+
// biome-ignore lint/style/noNonNullAssertion: we check the length above
|
|
246
|
+
return this.commitMessage(messageOrBatch[messageOrBatch.length - 1]);
|
|
154
247
|
}
|
|
155
|
-
|
|
156
|
-
this.
|
|
157
|
-
topic: message.topic,
|
|
158
|
-
message: stringValueSerializer(message.value),
|
|
159
|
-
});
|
|
248
|
+
else {
|
|
249
|
+
return this.commitMessage(messageOrBatch);
|
|
160
250
|
}
|
|
161
|
-
return false;
|
|
162
251
|
}
|
|
163
252
|
async commitMessage(message) {
|
|
164
253
|
try {
|
|
@@ -191,12 +280,11 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
|
|
|
191
280
|
}
|
|
192
281
|
}
|
|
193
282
|
}
|
|
194
|
-
buildTransactionName(
|
|
195
|
-
const
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
return name;
|
|
283
|
+
buildTransactionName(topic) {
|
|
284
|
+
const baseTransactionName = `kafka:${this.constructor.name}:${topic}`;
|
|
285
|
+
return this.options.batchProcessingEnabled
|
|
286
|
+
? `${baseTransactionName}:batch`
|
|
287
|
+
: baseTransactionName;
|
|
200
288
|
}
|
|
201
289
|
getRequestContext(message) {
|
|
202
290
|
let reqId = message.headers.get(this.resolveHeaderRequestIdField());
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AbstractKafkaConsumer.js","sourceRoot":"","sources":["../lib/AbstractKafkaConsumer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EACL,aAAa,EAEb,qBAAqB,GACtB,MAAM,qBAAqB,CAAA;AAE5B,OAAO,EAEL,QAAQ,EAIR,aAAa,EACb,aAAa,EACb,kBAAkB,GACnB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,oBAAoB,EAAyB,MAAM,2BAA2B,CAAA;AACvF,OAAO,EAAE,qBAAqB,EAAE,MAAM,8CAA8C,CAAA;AAUpF,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAA;AACpG,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAA;AAiBtE,MAAM,wBAAwB,GAAG,IAAI,GAAG,CAAC;IACvC,kBAAkB;IAClB,iBAAiB;IACjB,qBAAqB;CACtB,CAAC,CAAA;AAEF;;;GAGG;AACH,MAAM,qBAAqB,GAAG,CAAC,CAAA;AAE/B,MAAM,OAAgB,qBAGpB,SAAQ,oBAAwF;IAC/E,QAAQ,CAA0C;IAC3D,cAAc,CAAiD;IAEtD,+BAA+B,CAAiC;IAChE,gBAAgB,CAAuD;IACvE,gBAAgB,CAAkB;IAEnD,YACE,YAAuC,EACvC,OAA6D,EAC7D,gBAAkC;QAElC,KAAK,CAAC,YAAY,EAAE,OAAO,CAAC,CAAA;QAE5B,IAAI,CAAC,+BAA+B,GAAG,YAAY,CAAC,+BAA+B,CAAA;QACnF,IAAI,CAAC,gBAAgB,GAAG,IAAI,qBAAqB,CAC/C,OAAO,CAAC,QAAQ,EAChB,OAAO,CAAC,gBAAgB,CACzB,CAAA;QACD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAA;QAExC,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC;YAC3B,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK;YACrB,GAAG,IAAI,CAAC,OAAO;YACf,UAAU,EAAE,KAAK,EAAE,4BAA4B;YAC/C,aAAa,EAAE;gBACb,GAAG,EAAE,kBAAkB;gBACvB,KAAK,EAAE,oBAAoB;gBAC3B,SAAS,EAAE,kBAAkB;gBAC7B,WAAW,EAAE,kBAAkB;aAChC;SACF,CAAC,CAAA;QAEF,MAAM,UAAU,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAA;QACnF,qBAAqB;QACrB,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,EAAE,CAC5C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,6BAA6B,CAAC,CAC7D,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,iBAAiB,EAAE,CAAC,CAAC,EAAE,EAAE,CACxC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,kDAAkD,CAAC,CAClF,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,sBAAsB,EAAE,CAAC,CAAC,EAAE,EAAE,CAC7C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,+BAA+B,CAAC,CAC/D,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,0BAA0B,EAAE,CAAC,CAAC,EAAE,EAAE,CACjD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,sBAAsB,CAAC,CACtD,CAAA;QACD,oBAAoB;IACtB,CAAC;IAED;;OAEG;IACH,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAA;IACpC,CAAC;IAED;;;OAGG;IACH,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAA;IACjC,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAA;QAC3C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QAE9E,IAAI,CAAC;YACH,MAAM,EAAE,QAAQ,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC,OAAO,CAAA,CAAC,8CAA8C;YAEnG,0IAA0I;YAC1I,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC5B,cAAc,EAAE,cAAc,CAAC,cAAc;gBAC7C,gBAAgB,EAAE,cAAc,CAAC,gBAAgB;gBACjD,iBAAiB,EAAE,cAAc,CAAC,iBAAiB;aACpD,CAAC,CAAA;YACF,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,GAAG,cAAc,EAAE,MAAM,EAAE,CAAC,CAAA;QAClF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,aAAa,CAAC;gBACtB,OAAO,EAAE,sBAAsB;gBAC/B,SAAS,EAAE,2BAA2B;gBACtC,KAAK,EAAE,KAAK;aACb,CAAC,CAAA;QACJ,CAAC;QAED,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,CACzC,IAAI,CAAC,OAAO,CACV,OAAgF,CACjF,CACF,CAAA;QACD,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;IACtE,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,cAAc;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;QAElD,MAAM,IAAI,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;QAC7D,IAAI,CAAC,cAAc,GAAG,SAAS,CAAA;QAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;IAC7B,CAAC;IAEO,KAAK,CAAC,OAAO,CACnB,OAA8E;QAE9E,yEAAyE;QACzE,IAAI,CAAC,OAAO,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAA;QAEtD,MAAM,+BAA+B,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAElD,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;QAClF,iFAAiF;QACjF,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAA;QAEhD,oBAAoB;QACpB,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,UAAU,EAAE,CAAA;QAC1E,IAAI,CAAC,+BAA+B,EAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,EAAE,aAAa,CAAC,CAAA;QAE9F,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAC3D,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;YACzB,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,EAAE;gBACnC,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC;aAC9C,CAAC,CAAA;YACF,IAAI,CAAC,sBAAsB,CAAC;gBAC1B,OAAO,EAAE,OAAO;gBAChB,gBAAgB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE;gBACpE,+BAA+B;aAChC,CAAC,CAAA;YAEF,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAA;QACpC,CAAC;QAED,MAAM,gBAAgB,GAAG,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,IAAI,EAAE,CAAA;QAEhE,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAA;QAEtD,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,QAAQ,GAAG,KAAK,CAAA;QACpB,GAAG,CAAC;YACF,qCAAqC;YACrC,IAAI,OAAO,GAAG,CAAC;gBAAE,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAA;YAE3D,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,OAAO,CAAC,OAAO,EAAE,cAAc,CAAC,CAAA;YACrF,IAAI,QAAQ;gBAAE,MAAK;YAEnB,OAAO,EAAE,CAAA;QACX,CAAC,QAAQ,OAAO,GAAG,qBAAqB,EAAC;QAEzC,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,CAAC,sBAAsB,CAAC;gBAC1B,OAAO,EAAE,gBAAgB;gBACzB,gBAAgB,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE;gBACxC,+BAA+B;aAChC,CAAC,CAAA;QACJ,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,sBAAsB,CAAC;gBAC1B,OAAO,EAAE,gBAAgB;gBACzB,gBAAgB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE;gBAClE,+BAA+B;aAChC,CAAC,CAAA;QACJ,CAAC;QAED,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,aAAa,CAAC,CAAA;QAEzD,OAAO,IAAI,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAA;IAC7C,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,OAAsD,EACtD,OAAqD,EACrD,cAA8B;QAE9B,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,gBAAgB,EAAE,cAAc,CAAC,CAAA;YAC7D,OAAO,IAAI,CAAA;QACb,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE;gBACvB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC;aAC9C,CAAC,CAAA;QACJ,CAAC;QAED,OAAO,KAAK,CAAA;IACd,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAgD;QAC1E,IAAI,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,EAC9E,0BAA0B,CAC3B,CAAA;YACD,MAAM,OAAO,CAAC,MAAM,EAAE,CAAA;QACxB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,aAAa;gBAAE,OAAO,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAA;YAClF,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAEO,2BAA2B,CAAC,aAA4B;QAC9D,kFAAkF;QAClF,KAAK,MAAM,KAAK,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YACzC,IACE,KAAK,YAAY,aAAa;gBAC9B,KAAK,CAAC,OAAO;gBACb,wBAAwB,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAC3C,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,KAAK,CACf;oBACE,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,oBAAoB,EAAE,aAAa,CAAC,OAAO;oBAC3C,oBAAoB,EAAE,KAAK,CAAC,OAAO;oBACnC,KAAK,EAAE,aAAa;iBACrB,EACD,6BAA6B,KAAK,CAAC,OAAO,EAAE,CAC7C,CAAA;YACH,CAAC;iBAAM,CAAC;gBACN,yCAAyC;gBACzC,MAAM,aAAa,CAAA;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IAEO,oBAAoB,CAC1B,OAA8E;QAE9E,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAE1D,IAAI,IAAI,GAAG,SAAS,IAAI,CAAC,WAAW,CAAC,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAA;QAC5D,IAAI,WAAW,EAAE,IAAI,EAAE,CAAC,MAAM;YAAE,IAAI,IAAI,IAAI,WAAW,CAAC,IAAI,EAAE,EAAE,CAAA;QAEhE,OAAO,IAAI,CAAA;IACb,CAAC;IAEO,iBAAiB,CAAC,OAAgD;QACxE,IAAI,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC,CAAA;QACnE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,KAAK,GAAG,UAAU,EAAE,CAAA;QAE7D,OAAO;YACL,KAAK;YACL,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;gBACxB,cAAc,EAAE,KAAK;gBACrB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,UAAU,EAAE,OAAO,CAAC,GAAG;aACxB,CAAC;SACH,CAAA;IACH,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"AbstractKafkaConsumer.js","sourceRoot":"","sources":["../lib/AbstractKafkaConsumer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EACL,aAAa,EACb,qBAAqB,GAEtB,MAAM,qBAAqB,CAAA;AAK5B,OAAO,EAEL,QAAQ,EAGR,aAAa,EACb,aAAa,EACb,kBAAkB,GACnB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACL,oBAAoB,GAGrB,MAAM,2BAA2B,CAAA;AAYlC,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAA;AACpG,OAAO,EAEL,uBAAuB,GACxB,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAA;AAkCtE,MAAM,wBAAwB,GAAG,IAAI,GAAG,CAAC;IACvC,kBAAkB;IAClB,iBAAiB;IACjB,qBAAqB;CACtB,CAAC,CAAA;AAEF;;;GAGG;AACH,MAAM,qBAAqB,GAAG,CAAC,CAAA;AAE/B,MAAM,OAAgB,qBAIpB,SAAQ,oBAGT;IACkB,QAAQ,CAA0C;IAC3D,cAAc,CAAiD;IAC/D,kBAAkB,CAEzB;IAEgB,+BAA+B,CAAiC;IAChE,gBAAgB,CAAkB;IAEnD,YACE,YAAuC,EACvC,OAAqF,EACrF,gBAAkC;QAElC,KAAK,CAAC,YAAY,EAAE,OAAO,CAAC,CAAA;QAE5B,IAAI,CAAC,+BAA+B,GAAG,YAAY,CAAC,+BAA+B,CAAA;QACnF,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAA;QAExC,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC;YAC3B,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK;YACrB,GAAG,IAAI,CAAC,OAAO;YACf,UAAU,EAAE,KAAK,EAAE,4BAA4B;YAC/C,aAAa,EAAE;gBACb,GAAG,EAAE,kBAAkB;gBACvB,KAAK,EAAE,oBAAoB;gBAC3B,SAAS,EAAE,kBAAkB;gBAC7B,WAAW,EAAE,kBAAkB;aAChC;SACF,CAAC,CAAA;QAEF,MAAM,UAAU,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAA;QACnF,qBAAqB;QACrB,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,EAAE,CAC5C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,6BAA6B,CAAC,CAC7D,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,iBAAiB,EAAE,CAAC,CAAC,EAAE,EAAE,CACxC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,kDAAkD,CAAC,CAClF,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,sBAAsB,EAAE,CAAC,CAAC,EAAE,EAAE,CAC7C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,+BAA+B,CAAC,CAC/D,CAAA;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,0BAA0B,EAAE,CAAC,CAAC,EAAE,EAAE,CACjD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,sBAAsB,CAAC,CACtD,CAAA;QACD,oBAAoB;IACtB,CAAC;IAED;;OAEG;IACH,IAAI,WAAW;QACb,uDAAuD;QACvD,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO,KAAK,CAAA;QAClE,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAA;QACpC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,yEAAyE;YACzE,oBAAoB;YACpB,OAAO,KAAK,CAAA;QACd,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,IAAI,QAAQ;QACV,uDAAuD;QACvD,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB;YAAE,OAAO,KAAK,CAAA;QAClE,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAA;QACjC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,yEAAyE;YACzE,oBAAoB;YACpB,OAAO,KAAK,CAAA;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;QACjD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QACjD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QAE9E,IAAI,CAAC;YACH,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC,OAAO,CAAA,CAAC,8CAA8C;YAEtG,0IAA0I;YAC1I,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC5B,cAAc,EAAE,cAAc,CAAC,cAAc;gBAC7C,gBAAgB,EAAE,cAAc,CAAC,gBAAgB;gBACjD,iBAAiB,EAAE,cAAc,CAAC,iBAAiB;aACpD,CAAC,CAAA;YAEF,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,GAAG,cAAc,EAAE,MAAM,EAAE,CAAC,CAAA;YAChF,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;gBAC/E,IAAI,CAAC,kBAAkB,GAAG,IAAI,uBAAuB,CAEnD;oBACA,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,SAAS;oBACxD,mBAAmB,EAAE,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,mBAAmB;iBAC7E,CAAC,CAAA;gBACF,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAA;YACnD,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,aAAa,CAAC;gBACtB,OAAO,EAAE,sBAAsB;gBAC/B,SAAS,EAAE,2BAA2B;gBACtC,KAAK,EAAE,KAAK;aACb,CAAC,CAAA;QACJ,CAAC;QAED,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACnE,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,CACxD,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,EAAE,YAAY,CAAC,QAAQ,CAAC,CACxD,CAAA;YACD,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;QAC1E,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,CACzC,IAAI,CAAC,OAAO,CACV,OAAO,CAAC,KAAK,EACb,OAAoE,CACrE,CACF,CAAA;QACH,CAAC;QAED,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;IACtE,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACrD,yEAAyE;YACzE,IAAI,IAAI,CAAC,QAAQ;gBAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAA;YAC7C,OAAM;QACR,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAA;YACjC,IAAI,CAAC,cAAc,GAAG,SAAS,CAAA;QACjC,CAAC;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC5B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAA;YACrE,IAAI,CAAC,kBAAkB,GAAG,SAAS,CAAA;QACrC,CAAC;QAED,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;IAC7B,CAAC;IAEO,cAAc,CAAC,KAAoC;QACzD,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IACrC,CAAC;IAEO,KAAK,CAAC,OAAO,CACnB,KAAa,EACb,cAAoE;QAEpE,MAAM,+BAA+B,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAElD,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA;QAEhD,iFAAiF;QACjF,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;QAEtD,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CACtC,aAAa,EACb,cAAc,EACd,+BAA+B,CAChC,CAAA;QAED,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;QACpC,CAAC;QAED,kFAAkF;QAClF,MAAM,YAAY,GAAG,aAAa,CAAC,CAAC,CAAE,CAAA;QAEtC,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAA;QAE3D,oBAAoB;QACpB,MAAM,aAAa,GAAG,UAAU,EAAE,CAAA;QAClC,IAAI,CAAC,+BAA+B,EAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,EAAE,aAAa,CAAC,CAAA;QAE5F,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,uBAAuB,CACzD,KAAK;QACL,0EAA0E;QAC1E,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,EAClE,aAAa,CAAC,OAAO,EACrB,cAAc,CACf,CAAA;QAED,IAAI,CAAC,uBAAuB,CAAC,aAAa,EAAE,gBAAgB,EAAE,+BAA+B,CAAC,CAAA;QAE9F,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,aAAa,CAAC,CAAA;QAEzD,+EAA+E;QAC/E,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAA;IACpC,CAAC;IAEO,aAAa,CACnB,aAIC,EACD,cAAoE,EACpE,+BAAuC;QAEvC,MAAM,eAAe,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAA;QAEzF,MAAM,aAAa,GAAgE,EAAE,CAAA;QAErF,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,yEAAyE;YACzE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACnB,SAAQ;YACV,CAAC;YAED,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAEjE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;gBACzB,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,EAAE;oBACnC,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC;iBAC9C,CAAC,CAAA;gBACF,IAAI,CAAC,sBAAsB,CAAC;oBAC1B,OAAO,EAAE,OAAO;oBAChB,gBAAgB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE;oBACpE,+BAA+B;iBAChC,CAAC,CAAA;gBACF,SAAQ;YACV,CAAC;YAED,aAAa,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,IAAI,EAAE,CAAC,CAAA;QAC7D,CAAC;QAED,OAAO,aAAa,CAAA;IACtB,CAAC;IAEO,KAAK,CAAC,uBAAuB,CACnC,KAAa,EACb,cAA4C,EAC5C,OAA6E,EAC7E,cAA8B;QAE9B,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,gBAAgB,GAA4B;YAC9C,MAAM,EAAE,OAAO;YACf,WAAW,EAAE,cAAc;SAC5B,CAAA;QACD,GAAG,CAAC;YACF,qCAAqC;YACrC,IAAI,OAAO,GAAG,CAAC;gBAAE,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAA;YAE3D,gBAAgB,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,cAAc,CAAC,CAAA;YAC1F,IAAI,gBAAgB,CAAC,MAAM,KAAK,UAAU;gBAAE,MAAK;YAEjD,OAAO,EAAE,CAAA;QACX,CAAC,QAAQ,OAAO,GAAG,qBAAqB,EAAC;QAEzC,OAAO,gBAAgB,CAAA;IACzB,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,KAAa,EACb,cAA4C,EAC5C,OAA6E,EAC7E,cAA8B;QAE9B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;YAC7C,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,CAAC,OAAO,EAAE,CAAC;gBACpD,MAAM,IAAI,KAAK,CACb,6EAA6E,CAC9E,CAAA;YACH,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,sBAAsB,IAAI,OAAO,EAAE,CAAC;gBACpD,MAAM,IAAI,KAAK,CACb,iFAAiF,CAClF,CAAA;YACH,CAAC;YAED,MAAM,OAAO;YACX,mGAAmG;YACnG,cAEuC,EACvC,IAAI,CAAC,gBAAgB,EACrB,cAAc,CACf,CAAA;YACD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAA;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC;gBAChD,CAAC,CAAC,EAAE,SAAS,EAAE,cAAc,CAAC,MAAM,EAAE;gBACtC,CAAC,CAAC,EAAE,OAAO,EAAE,qBAAqB,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,CAAA;YAC5D,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE;gBACvB,KAAK;gBACL,GAAG,YAAY;aAChB,CAAC,CAAA;QACJ,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,CAAA;IACzD,CAAC;IAEO,uBAAuB,CAC7B,QAA0C,EAC1C,gBAAyC,EACzC,+BAAuC;QAEvC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC,sBAAsB,CAAC;gBAC1B,OAAO;gBACP,gBAAgB;gBAChB,+BAA+B;aAChC,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,cAAoE;QACjF,IAAI,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;YAClC,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAChC,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;YAC1B,CAAC;YACD,wEAAwE;YACxE,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAA;QACvE,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,CAAA;QAC3C,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAkE;QAC5F,IAAI,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,EAC9E,0BAA0B,CAC3B,CAAA;YACD,MAAM,OAAO,CAAC,MAAM,EAAE,CAAA;QACxB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,aAAa;gBAAE,OAAO,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAA;YAClF,MAAM,KAAK,CAAA;QACb,CAAC;IACH,CAAC;IAEO,2BAA2B,CAAC,aAA4B;QAC9D,kFAAkF;QAClF,KAAK,MAAM,KAAK,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YACzC,IACE,KAAK,YAAY,aAAa;gBAC9B,KAAK,CAAC,OAAO;gBACb,wBAAwB,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAC3C,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,KAAK,CACf;oBACE,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,oBAAoB,EAAE,aAAa,CAAC,OAAO;oBAC3C,oBAAoB,EAAE,KAAK,CAAC,OAAO;oBACnC,KAAK,EAAE,aAAa;iBACrB,EACD,6BAA6B,KAAK,CAAC,OAAO,EAAE,CAC7C,CAAA;YACH,CAAC;iBAAM,CAAC;gBACN,yCAAyC;gBACzC,MAAM,aAAa,CAAA;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IAEO,oBAAoB,CAAC,KAAa;QACxC,MAAM,mBAAmB,GAAG,SAAS,IAAI,CAAC,WAAW,CAAC,IAAI,IAAI,KAAK,EAAE,CAAA;QACrE,OAAO,IAAI,CAAC,OAAO,CAAC,sBAAsB;YACxC,CAAC,CAAC,GAAG,mBAAmB,QAAQ;YAChC,CAAC,CAAC,mBAAmB,CAAA;IACzB,CAAC;IAEO,iBAAiB,CACvB,OAAkE;QAElE,IAAI,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC,CAAA;QACnE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,KAAK,GAAG,UAAU,EAAE,CAAA;QAE7D,OAAO;YACL,KAAK;YACL,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;gBACxB,cAAc,EAAE,KAAK;gBACrB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,UAAU,EAAE,OAAO,CAAC,GAAG;aACxB,CAAC;SACH,CAAA;IACH,CAAC;CACF"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { InternalError, stringValueSerializer } from '@lokalise/node-core';
|
|
2
2
|
import { MessageSchemaContainer } from '@message-queue-toolkit/core';
|
|
3
|
-
import {
|
|
3
|
+
import { jsonSerializer, Producer, stringSerializer, } from '@platformatic/kafka';
|
|
4
4
|
import { AbstractKafkaService } from "./AbstractKafkaService.js";
|
|
5
5
|
export class AbstractKafkaPublisher extends AbstractKafkaService {
|
|
6
6
|
topicsConfig;
|
|
@@ -14,10 +14,9 @@ export class AbstractKafkaPublisher extends AbstractKafkaService {
|
|
|
14
14
|
if (this.topicsConfig.length === 0)
|
|
15
15
|
throw new Error('At least one topic must be defined');
|
|
16
16
|
this.schemaContainers = {};
|
|
17
|
-
for (const { topic,
|
|
17
|
+
for (const { topic, schema } of this.topicsConfig) {
|
|
18
18
|
this.schemaContainers[topic] = new MessageSchemaContainer({
|
|
19
|
-
messageSchemas:
|
|
20
|
-
messageTypeField: this.options.messageTypeField,
|
|
19
|
+
messageSchemas: [schema],
|
|
21
20
|
messageDefinitions: [],
|
|
22
21
|
});
|
|
23
22
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AbstractKafkaPublisher.js","sourceRoot":"","sources":["../lib/AbstractKafkaPublisher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAA;AAC1E,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAA;AACpE,OAAO,
|
|
1
|
+
{"version":3,"file":"AbstractKafkaPublisher.js","sourceRoot":"","sources":["../lib/AbstractKafkaPublisher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAA;AAC1E,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAA;AACpE,OAAO,EACL,cAAc,EAGd,QAAQ,EACR,gBAAgB,GACjB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,oBAAoB,EAAyB,MAAM,2BAA2B,CAAA;AAoBvF,MAAM,OAAgB,sBAEpB,SAAQ,oBAAuE;IAC9D,YAAY,CAAc;IAC1B,gBAAgB,CAAgD;IAEhE,QAAQ,CAA0C;IAC3D,WAAW,CAAS;IAE5B,YAAY,YAA+B,EAAE,OAA4C;QACvF,KAAK,CAAC,YAAY,EAAE,OAAO,CAAC,CAAA;QAC5B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAA;QAExB,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAA;QACxC,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QAEzF,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAA;QAC1B,KAAK,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YAClD,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,GAAG,IAAI,sBAAsB,CAAC;gBACxD,cAAc,EAAE,CAAC,MAAM,CAAC;gBACxB,kBAAkB,EAAE,EAAE;aACvB,CAAC,CAAA;QACJ,CAAC;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC;YAC3B,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK;YACrB,GAAG,IAAI,CAAC,OAAO;YACf,WAAW,EAAE;gBACX,GAAG,EAAE,gBAAgB;gBACrB,KAAK,EAAE,cAAc;gBACrB,SAAS,EAAE,gBAAgB;gBAC3B,WAAW,EAAE,gBAAgB;aAC9B;SACF,CAAC,CAAA;IACJ,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,WAAW;YAAE,OAAM;QAE5B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAA;YAC9B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;QACzB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,IAAI,aAAa,CAAC;gBACtB,OAAO,EAAE,sBAAsB;gBAC/B,SAAS,EAAE,2BAA2B;gBACtC,KAAK,EAAE,CAAC;aACT,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,WAAW;YAAE,OAAM;QAE7B,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;QAC3B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAA;IAC1B,CAAC;IAED,KAAK,CAAC,OAAO,CACX,KAAY,EACZ,OAAiE,EACjE,cAA+B,EAC/B,OAA6B;QAE7B,MAAM,+BAA+B,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAElD,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE,aAAa,CAAC,OAAiB,CAAC,CAAA;QACnF,IAAI,CAAC,YAAY;YAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,KAAK,EAAE,CAAC,CAAA;QACnF,IAAI,YAAY,CAAC,KAAK;YAAE,MAAM,YAAY,CAAC,KAAK,CAAA;QAEhD,MAAM,IAAI,CAAC,IAAI,EAAE,CAAA,CAAC,YAAY;QAE9B,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAGtD,CAAA;YAED,MAAM,OAAO,GAAG;gBACd,GAAG,OAAO,EAAE,OAAO;gBACnB,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC,EAAE,cAAc,EAAE,KAAK,IAAI,EAAE;aAClE,CAAA;YAED,MAAM,YAAY,GAAG,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,CAAA;YACzE,MAAM,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC;gBACxB,QAAQ,EAAE,CAAC,YAAY,CAAC;aACzB,CAAC,CAAA;YAEF,IAAI,CAAC,sBAAsB,CAAC;gBAC1B,OAAO,EAAE,YAAY;gBACrB,gBAAgB,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE;gBACzC,+BAA+B;aAChC,CAAC,CAAA;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG;gBACnB,KAAK;gBACL,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;gBAChC,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC;aACxC,CAAA;YACD,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,YAAY,CAAC,CAAA;YACtC,MAAM,IAAI,aAAa,CAAC;gBACtB,OAAO,EAAE,oCAAqC,KAAe,CAAC,OAAO,EAAE;gBACvE,SAAS,EAAE,qBAAqB;gBAChC,KAAK,EAAE,KAAK;gBACZ,OAAO,EAAE,YAAY;aACtB,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import { type CommonLogger, type ErrorReporter } from '@lokalise/node-core';
|
|
2
2
|
import type { MakeRequired, MayOmit } from '@lokalise/universal-ts-utils/node';
|
|
3
3
|
import { type HandlerSpy, type HandlerSpyParams, type MessageMetricsManager, type MessageProcessingResult, type PublicHandlerSpy } from '@message-queue-toolkit/core';
|
|
4
|
-
import type { BaseOptions
|
|
5
|
-
import type { KafkaConfig, KafkaDependencies, SupportedMessageValues, TopicConfig } from './types.ts';
|
|
4
|
+
import type { BaseOptions } from '@platformatic/kafka';
|
|
5
|
+
import type { DeserializedMessage, KafkaConfig, KafkaDependencies, SupportedMessageValues, TopicConfig } from './types.ts';
|
|
6
6
|
export type BaseKafkaOptions = {
|
|
7
7
|
kafka: KafkaConfig;
|
|
8
|
-
messageTypeField?: string;
|
|
9
8
|
messageIdField?: string;
|
|
10
9
|
/**
|
|
11
10
|
* The field in the message headers that contains the request ID.
|
|
@@ -16,7 +15,7 @@ export type BaseKafkaOptions = {
|
|
|
16
15
|
handlerSpy?: HandlerSpy<object> | HandlerSpyParams | boolean;
|
|
17
16
|
logMessages?: boolean;
|
|
18
17
|
} & Omit<BaseOptions, keyof KafkaConfig>;
|
|
19
|
-
type ProcessedMessage<TopicsConfig extends TopicConfig[]> = MayOmit<Pick<
|
|
18
|
+
export type ProcessedMessage<TopicsConfig extends TopicConfig[]> = MayOmit<Pick<DeserializedMessage<SupportedMessageValues<TopicsConfig>>, 'topic' | 'value' | 'timestamp'>, 'timestamp'>;
|
|
20
19
|
export declare abstract class AbstractKafkaService<TopicsConfig extends TopicConfig[], KafkaOptions extends BaseKafkaOptions> {
|
|
21
20
|
protected readonly errorReporter: ErrorReporter;
|
|
22
21
|
protected readonly logger: CommonLogger;
|
|
@@ -27,7 +26,6 @@ export declare abstract class AbstractKafkaService<TopicsConfig extends TopicCon
|
|
|
27
26
|
abstract init(): Promise<void>;
|
|
28
27
|
abstract close(): Promise<void>;
|
|
29
28
|
get handlerSpy(): PublicHandlerSpy<SupportedMessageValues<TopicsConfig>>;
|
|
30
|
-
protected resolveMessageType(message: SupportedMessageValues<TopicsConfig>): string | undefined;
|
|
31
29
|
protected resolveMessageId(message: SupportedMessageValues<TopicsConfig>): string | undefined;
|
|
32
30
|
protected resolveHeaderRequestIdField(): string;
|
|
33
31
|
protected handleMessageProcessed(params: {
|
|
@@ -37,4 +35,3 @@ export declare abstract class AbstractKafkaService<TopicsConfig extends TopicCon
|
|
|
37
35
|
}): void;
|
|
38
36
|
protected handlerError(error: unknown, context?: Record<string, unknown>): void;
|
|
39
37
|
}
|
|
40
|
-
export {};
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { resolveGlobalErrorLogObject, stringValueSerializer, } from '@lokalise/node-core';
|
|
1
|
+
import { isError, resolveGlobalErrorLogObject, stringValueSerializer, } from '@lokalise/node-core';
|
|
3
2
|
import { resolveHandlerSpy, } from '@message-queue-toolkit/core';
|
|
4
3
|
export class AbstractKafkaService {
|
|
5
4
|
errorReporter;
|
|
@@ -19,12 +18,6 @@ export class AbstractKafkaService {
|
|
|
19
18
|
return this._handlerSpy;
|
|
20
19
|
throw new Error('HandlerSpy was not instantiated, please pass `handlerSpy` parameter during creation.');
|
|
21
20
|
}
|
|
22
|
-
resolveMessageType(message) {
|
|
23
|
-
if (!this.options.messageTypeField)
|
|
24
|
-
return undefined;
|
|
25
|
-
// @ts-expect-error
|
|
26
|
-
return message[this.options.messageTypeField];
|
|
27
|
-
}
|
|
28
21
|
resolveMessageId(message) {
|
|
29
22
|
// @ts-expect-error
|
|
30
23
|
return message[this.options.messageIdField];
|
|
@@ -35,7 +28,6 @@ export class AbstractKafkaService {
|
|
|
35
28
|
handleMessageProcessed(params) {
|
|
36
29
|
const { message, processingResult } = params;
|
|
37
30
|
const messageId = this.resolveMessageId(message.value);
|
|
38
|
-
const messageType = this.resolveMessageType(message.value);
|
|
39
31
|
this._handlerSpy?.addProcessedMessage({ message: message.value, processingResult }, messageId);
|
|
40
32
|
if (this.options.logMessages) {
|
|
41
33
|
this.logger.debug({
|
|
@@ -43,7 +35,6 @@ export class AbstractKafkaService {
|
|
|
43
35
|
topic: message.topic,
|
|
44
36
|
processingResult,
|
|
45
37
|
messageId,
|
|
46
|
-
messageType,
|
|
47
38
|
}, `Finished processing message ${messageId}`);
|
|
48
39
|
}
|
|
49
40
|
if (this.messageMetricsManager) {
|
|
@@ -52,7 +43,7 @@ export class AbstractKafkaService {
|
|
|
52
43
|
processingResult,
|
|
53
44
|
queueName: message.topic,
|
|
54
45
|
messageId: messageId ?? 'unknown',
|
|
55
|
-
messageType:
|
|
46
|
+
messageType: 'unknown',
|
|
56
47
|
messageTimestamp: message.timestamp ? Number(message.timestamp) : undefined,
|
|
57
48
|
messageProcessingStartTimestamp: params.messageProcessingStartTimestamp,
|
|
58
49
|
messageProcessingEndTimestamp: Date.now(),
|
|
@@ -61,7 +52,7 @@ export class AbstractKafkaService {
|
|
|
61
52
|
}
|
|
62
53
|
handlerError(error, context = {}) {
|
|
63
54
|
this.logger.error({ ...resolveGlobalErrorLogObject(error), ...context });
|
|
64
|
-
if (
|
|
55
|
+
if (isError(error))
|
|
65
56
|
this.errorReporter.report({ error, context });
|
|
66
57
|
}
|
|
67
58
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AbstractKafkaService.js","sourceRoot":"","sources":["../lib/AbstractKafkaService.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"AbstractKafkaService.js","sourceRoot":"","sources":["../lib/AbstractKafkaService.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,OAAO,EACP,2BAA2B,EAC3B,qBAAqB,GACtB,MAAM,qBAAqB,CAAA;AAE5B,OAAO,EAML,iBAAiB,GAClB,MAAM,6BAA6B,CAAA;AA4BpC,MAAM,OAAgB,oBAAoB;IAIrB,aAAa,CAAe;IAC5B,MAAM,CAAc;IACpB,qBAAqB,CAEvC;IAEkB,OAAO,CAA8C;IACrD,WAAW,CAAmD;IAEjF,YAAY,YAA+B,EAAE,OAAqB;QAChE,IAAI,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAA;QACjC,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC,aAAa,CAAA;QAC/C,IAAI,CAAC,qBAAqB,GAAG,YAAY,CAAC,qBAAqB,CAAA;QAC/D,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,IAAI,EAAE,CAAA;QAE7E,IAAI,CAAC,WAAW,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAA;IAC/C,CAAC;IAKD,IAAI,UAAU;QACZ,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC,WAAW,CAAA;QAE7C,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;IACH,CAAC;IAES,gBAAgB,CAAC,OAA6C;QACtE,mBAAmB;QACnB,OAAO,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAuB,CAAA;IACnE,CAAC;IAES,2BAA2B;QACnC,OAAO,IAAI,CAAC,OAAO,CAAC,oBAAoB,IAAI,cAAc,CAAA;IAC5D,CAAC;IAES,sBAAsB,CAAC,MAIhC;QACC,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,GAAG,MAAM,CAAA;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAEtD,IAAI,CAAC,WAAW,EAAE,mBAAmB,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,EAAE,gBAAgB,EAAE,EAAE,SAAS,CAAC,CAAA;QAE9F,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,CACf;gBACE,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC;gBAC7C,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,gBAAgB;gBAChB,SAAS;aACV,EACD,+BAA+B,SAAS,EAAE,CAC3C,CAAA;QACH,CAAC;QAED,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC/B,IAAI,CAAC,qBAAqB,CAAC,wBAAwB,CAAC;gBAClD,OAAO,EAAE,OAAO,CAAC,KAAK;gBACtB,gBAAgB;gBAChB,SAAS,EAAE,OAAO,CAAC,KAAK;gBACxB,SAAS,EAAE,SAAS,IAAI,SAAS;gBACjC,WAAW,EAAE,SAAS;gBACtB,gBAAgB,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;gBAC3E,+BAA+B,EAAE,MAAM,CAAC,+BAA+B;gBACvE,6BAA6B,EAAE,IAAI,CAAC,GAAG,EAAE;aAC1C,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAES,YAAY,CAAC,KAAc,EAAE,UAAmC,EAAE;QAC1E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,2BAA2B,CAAC,KAAK,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC,CAAA;QACxE,IAAI,OAAO,CAAC,KAAK,CAAC;YAAE,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAA;IACnE,CAAC;CACF"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { ZodSchema } from 'zod/v4';
|
|
2
|
+
import type { DeserializedMessage, RequestContext } from '../types.js';
|
|
3
|
+
export type KafkaHandler<MessageValue extends object, ExecutionContext, BatchProcessingEnabled extends boolean = false> = (message: BatchProcessingEnabled extends false ? DeserializedMessage<MessageValue> : DeserializedMessage<MessageValue>[], context: ExecutionContext, requestContext: RequestContext) => Promise<void> | void;
|
|
4
|
+
export declare class KafkaHandlerConfig<MessageValue extends object, ExecutionContext, BatchProcessingEnabled extends boolean = false> {
|
|
5
|
+
readonly schema: ZodSchema<MessageValue, any>;
|
|
6
|
+
readonly handler: KafkaHandler<MessageValue, ExecutionContext, BatchProcessingEnabled>;
|
|
7
|
+
constructor(schema: ZodSchema<MessageValue, any>, handler: KafkaHandler<MessageValue, ExecutionContext, BatchProcessingEnabled>);
|
|
8
|
+
}
|
|
9
|
+
export declare class KafkaBatchHandlerConfig<MessageValue extends object, ExecutionContext> extends KafkaHandlerConfig<MessageValue, ExecutionContext, true> {
|
|
10
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"KafkaHandlerConfig.js","sourceRoot":"","sources":["../../lib/handler-routing/KafkaHandlerConfig.ts"],"names":[],"mappings":"AAeA,MAAM,OAAO,kBAAkB;IAK7B,2EAA2E;IAC3D,MAAM,CAA8B;IACpC,OAAO,CAAsE;IAE7F;IACE,2EAA2E;IAC3E,MAAoC,EACpC,OAA6E;QAE7E,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;QACpB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAA;IACxB,CAAC;CACF;AAED,MAAM,OAAO,uBAGX,SAAQ,kBAAwD;CAAG"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { SupportedMessageValuesForTopic, SupportedTopics, TopicConfig } from '../types.ts';
|
|
2
|
+
import type { KafkaHandlerConfig } from './KafkaHandlerConfig.ts';
|
|
3
|
+
export type KafkaHandlerRouting<TopicsConfig extends TopicConfig[], ExecutionContext, BatchProcessingEnabled extends boolean> = Partial<{
|
|
4
|
+
[Topic in SupportedTopics<TopicsConfig>]: KafkaHandlerConfig<SupportedMessageValuesForTopic<TopicsConfig, Topic>, ExecutionContext, BatchProcessingEnabled>;
|
|
5
|
+
}>;
|
|
6
|
+
export declare class KafkaHandlerRoutingBuilder<const TopicsConfig extends TopicConfig[], ExecutionContext, BatchProcessingEnabled extends boolean> {
|
|
7
|
+
private readonly configs;
|
|
8
|
+
addConfig<Topic extends SupportedTopics<TopicsConfig>, MessageValue extends SupportedMessageValuesForTopic<TopicsConfig, Topic>>(topic: Topic, config: KafkaHandlerConfig<MessageValue, ExecutionContext, BatchProcessingEnabled>): this;
|
|
9
|
+
build(): KafkaHandlerRouting<TopicsConfig, ExecutionContext, BatchProcessingEnabled>;
|
|
10
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"KafkaHandlerRoutingBuilder.js","sourceRoot":"","sources":["../../lib/handler-routing/KafkaHandlerRoutingBuilder.ts"],"names":[],"mappings":"AAoBA,MAAM,OAAO,0BAA0B;IAKpB,OAAO,GAIpB,EAAE,CAAA;IAEN,SAAS,CAIP,KAAY,EACZ,MAAkF;QAElF,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,MAIrB,CAAA;QAED,OAAO,IAAI,CAAA;IACb,CAAC;IAED,KAAK;QACH,OAAO,IAAI,CAAC,OAAO,CAAA;IACrB,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../lib/handler-routing/index.ts"],"names":[],"mappings":"AAAA,cAAc,yBAAyB,CAAA;AACvC,OAAO,EAAE,0BAA0B,EAAE,MAAM,iCAAiC,CAAA"}
|
package/dist/index.d.ts
CHANGED
package/dist/index.js
CHANGED
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../lib/index.ts"],"names":[],"mappings":"AAAA,cAAc,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../lib/index.ts"],"names":[],"mappings":"AAAA,cAAc,4BAA4B,CAAA;AAC1C,cAAc,6BAA6B,CAAA;AAC3C,cAAc,4BAA4B,CAAA;AAC1C,cAAc,YAAY,CAAA"}
|
package/dist/types.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { CommonLogger } from '@lokalise/node-core';
|
|
2
2
|
import type { QueueDependencies } from '@message-queue-toolkit/core';
|
|
3
|
-
import type { ConnectionOptions } from '@platformatic/kafka';
|
|
3
|
+
import type { ConnectionOptions, Message } from '@platformatic/kafka';
|
|
4
4
|
import type { ZodSchema, z } from 'zod/v4';
|
|
5
5
|
export interface RequestContext {
|
|
6
6
|
logger: CommonLogger;
|
|
@@ -13,15 +13,16 @@ export type KafkaConfig = {
|
|
|
13
13
|
} & ConnectionOptions;
|
|
14
14
|
export type TopicConfig<Topic extends string = string> = {
|
|
15
15
|
topic: Topic;
|
|
16
|
-
|
|
16
|
+
schema: ZodSchema<object>;
|
|
17
17
|
};
|
|
18
18
|
export type SupportedTopics<TopicsConfig extends TopicConfig[]> = TopicsConfig[number]['topic'];
|
|
19
19
|
type MessageSchemasForTopic<TopicsConfig extends TopicConfig[], Topic extends SupportedTopics<TopicsConfig>> = Extract<TopicsConfig[number], {
|
|
20
20
|
topic: Topic;
|
|
21
|
-
}>['
|
|
21
|
+
}>['schema'];
|
|
22
22
|
export type SupportedMessageValuesInputForTopic<TopicsConfig extends TopicConfig[], Topic extends SupportedTopics<TopicsConfig>> = z.input<MessageSchemasForTopic<TopicsConfig, Topic>>;
|
|
23
23
|
export type SupportedMessageValuesForTopic<TopicsConfig extends TopicConfig[], Topic extends SupportedTopics<TopicsConfig>> = z.output<MessageSchemasForTopic<TopicsConfig, Topic>>;
|
|
24
|
-
type MessageSchemas<TopicsConfig extends TopicConfig[]> = TopicsConfig[number]['
|
|
24
|
+
type MessageSchemas<TopicsConfig extends TopicConfig[]> = TopicsConfig[number]['schema'];
|
|
25
25
|
export type SupportedMessageValuesInput<TopicsConfig extends TopicConfig[]> = z.input<MessageSchemas<TopicsConfig>>;
|
|
26
26
|
export type SupportedMessageValues<TopicsConfig extends TopicConfig[]> = z.output<MessageSchemas<TopicsConfig>>;
|
|
27
|
+
export type DeserializedMessage<MessageValue extends object> = Message<string, MessageValue, string, string>;
|
|
27
28
|
export {};
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { Duplex } from 'node:stream';
|
|
2
|
+
type CallbackFunction = (error?: Error | null) => void;
|
|
3
|
+
type MessageWithTopicAndPartition = {
|
|
4
|
+
topic: string;
|
|
5
|
+
partition: number;
|
|
6
|
+
};
|
|
7
|
+
export type KafkaMessageBatchOptions = {
|
|
8
|
+
batchSize: number;
|
|
9
|
+
timeoutMilliseconds: number;
|
|
10
|
+
};
|
|
11
|
+
export type MessageBatch<TMessage> = {
|
|
12
|
+
topic: string;
|
|
13
|
+
partition: number;
|
|
14
|
+
messages: TMessage[];
|
|
15
|
+
};
|
|
16
|
+
export interface KafkaMessageBatchStream<TMessage extends MessageWithTopicAndPartition> extends Duplex {
|
|
17
|
+
on(event: string | symbol, listener: (...args: any[]) => void): this;
|
|
18
|
+
on(event: 'data', listener: (chunk: MessageBatch<TMessage>) => void): this;
|
|
19
|
+
push(chunk: MessageBatch<TMessage> | null): boolean;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Collects messages in batches based on provided batchSize and flushes them when messages amount or timeout is reached.
|
|
23
|
+
*/
|
|
24
|
+
export declare class KafkaMessageBatchStream<TMessage extends MessageWithTopicAndPartition> extends Duplex {
|
|
25
|
+
private readonly batchSize;
|
|
26
|
+
private readonly timeout;
|
|
27
|
+
private readonly currentBatchPerTopicPartition;
|
|
28
|
+
private readonly batchTimeoutPerTopicPartition;
|
|
29
|
+
constructor(options: {
|
|
30
|
+
batchSize: number;
|
|
31
|
+
timeoutMilliseconds: number;
|
|
32
|
+
});
|
|
33
|
+
_read(): void;
|
|
34
|
+
_write(message: TMessage, _encoding: BufferEncoding, callback: CallbackFunction): void;
|
|
35
|
+
_final(callback: CallbackFunction): void;
|
|
36
|
+
private flushAllBatches;
|
|
37
|
+
private flushCurrentBatchMessages;
|
|
38
|
+
private getTopicPartitionKey;
|
|
39
|
+
private splitTopicPartitionKey;
|
|
40
|
+
}
|
|
41
|
+
export {};
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { Duplex } from 'node:stream';
|
|
2
|
+
/**
|
|
3
|
+
* Collects messages in batches based on provided batchSize and flushes them when messages amount or timeout is reached.
|
|
4
|
+
*/
|
|
5
|
+
// biome-ignore lint/suspicious/noUnsafeDeclarationMerging: merging interface with class to add strong typing for 'data' event
|
|
6
|
+
export class KafkaMessageBatchStream extends Duplex {
|
|
7
|
+
batchSize;
|
|
8
|
+
timeout;
|
|
9
|
+
currentBatchPerTopicPartition;
|
|
10
|
+
batchTimeoutPerTopicPartition;
|
|
11
|
+
constructor(options) {
|
|
12
|
+
super({ objectMode: true });
|
|
13
|
+
this.batchSize = options.batchSize;
|
|
14
|
+
this.timeout = options.timeoutMilliseconds;
|
|
15
|
+
this.currentBatchPerTopicPartition = {};
|
|
16
|
+
this.batchTimeoutPerTopicPartition = {};
|
|
17
|
+
}
|
|
18
|
+
_read() {
|
|
19
|
+
// No-op, as we push data when we have a full batch or timeout
|
|
20
|
+
}
|
|
21
|
+
_write(message, _encoding, callback) {
|
|
22
|
+
const key = this.getTopicPartitionKey(message.topic, message.partition);
|
|
23
|
+
if (!this.currentBatchPerTopicPartition[key]) {
|
|
24
|
+
this.currentBatchPerTopicPartition[key] = [message];
|
|
25
|
+
}
|
|
26
|
+
else {
|
|
27
|
+
// biome-ignore lint/style/noNonNullAssertion: non-existing entry is handled above
|
|
28
|
+
this.currentBatchPerTopicPartition[key].push(message);
|
|
29
|
+
}
|
|
30
|
+
// biome-ignore lint/style/noNonNullAssertion: we ensure above that the array is defined
|
|
31
|
+
if (this.currentBatchPerTopicPartition[key].length >= this.batchSize) {
|
|
32
|
+
this.flushCurrentBatchMessages(message.topic, message.partition);
|
|
33
|
+
return callback(null);
|
|
34
|
+
}
|
|
35
|
+
if (!this.batchTimeoutPerTopicPartition[key]) {
|
|
36
|
+
this.batchTimeoutPerTopicPartition[key] = setTimeout(() => {
|
|
37
|
+
this.flushCurrentBatchMessages(message.topic, message.partition);
|
|
38
|
+
}, this.timeout);
|
|
39
|
+
}
|
|
40
|
+
callback(null);
|
|
41
|
+
}
|
|
42
|
+
// Write side is closed, flush the remaining messages
|
|
43
|
+
_final(callback) {
|
|
44
|
+
this.flushAllBatches();
|
|
45
|
+
this.push(null); // End readable side
|
|
46
|
+
callback();
|
|
47
|
+
}
|
|
48
|
+
flushAllBatches() {
|
|
49
|
+
for (const key of Object.keys(this.currentBatchPerTopicPartition)) {
|
|
50
|
+
const { topic, partition } = this.splitTopicPartitionKey(key);
|
|
51
|
+
this.flushCurrentBatchMessages(topic, partition);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
flushCurrentBatchMessages(topic, partition) {
|
|
55
|
+
const key = this.getTopicPartitionKey(topic, partition);
|
|
56
|
+
if (this.batchTimeoutPerTopicPartition[key]) {
|
|
57
|
+
clearTimeout(this.batchTimeoutPerTopicPartition[key]);
|
|
58
|
+
this.batchTimeoutPerTopicPartition[key] = undefined;
|
|
59
|
+
}
|
|
60
|
+
if (!this.currentBatchPerTopicPartition[key]?.length) {
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
this.push({ topic, partition, messages: this.currentBatchPerTopicPartition[key] });
|
|
64
|
+
this.currentBatchPerTopicPartition[key] = [];
|
|
65
|
+
}
|
|
66
|
+
getTopicPartitionKey(topic, partition) {
|
|
67
|
+
return `${topic}:${partition}`;
|
|
68
|
+
}
|
|
69
|
+
splitTopicPartitionKey(key) {
|
|
70
|
+
const [topic, partition] = key.split(':');
|
|
71
|
+
if (!topic || !partition) {
|
|
72
|
+
throw new Error('Invalid topic-partition key format');
|
|
73
|
+
}
|
|
74
|
+
return { topic, partition: Number.parseInt(partition, 10) };
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
//# sourceMappingURL=KafkaMessageBatchStream.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"KafkaMessageBatchStream.js","sourceRoot":"","sources":["../../lib/utils/KafkaMessageBatchStream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AAuBpC;;GAEG;AACH,8HAA8H;AAC9H,MAAM,OAAO,uBAAuE,SAAQ,MAAM;IAC/E,SAAS,CAAQ;IACjB,OAAO,CAAQ;IAEf,6BAA6B,CAA4B;IACzD,6BAA6B,CAA4C;IAE1F,YAAY,OAA2D;QACrE,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAA;QAC3B,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAA;QAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,mBAAmB,CAAA;QAC1C,IAAI,CAAC,6BAA6B,GAAG,EAAE,CAAA;QACvC,IAAI,CAAC,6BAA6B,GAAG,EAAE,CAAA;IACzC,CAAC;IAEQ,KAAK;QACZ,8DAA8D;IAChE,CAAC;IAEQ,MAAM,CAAC,OAAiB,EAAE,SAAyB,EAAE,QAA0B;QACtF,MAAM,GAAG,GAAG,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,CAAA;QAEvE,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QACrD,CAAC;aAAM,CAAC;YACN,kFAAkF;YAClF,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QACxD,CAAC;QAED,wFAAwF;QACxF,IAAI,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAE,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACtE,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,CAAA;YAChE,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAA;QACvB,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,GAAG,EAAE;gBACxD,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,CAAA;YAClE,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;QAClB,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC,CAAA;IAChB,CAAC;IAED,qDAAqD;IAC5C,MAAM,CAAC,QAA0B;QACxC,IAAI,CAAC,eAAe,EAAE,CAAA;QACtB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA,CAAC,oBAAoB;QACpC,QAAQ,EAAE,CAAA;IACZ,CAAC;IAEO,eAAe;QACrB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,EAAE,CAAC;YAClE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAA;YAC7D,IAAI,CAAC,yBAAyB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;QAClD,CAAC;IACH,CAAC;IAEO,yBAAyB,CAAC,KAAa,EAAE,SAAiB;QAChE,MAAM,GAAG,GAAG,IAAI,CAAC,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;QAEvD,IAAI,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5C,YAAY,CAAC,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,CAAC,CAAA;YACrD,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,GAAG,SAAS,CAAA;QACrD,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;YACrD,OAAM;QACR,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAClF,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,GAAG,EAAE,CAAA;IAC9C,CAAC;IAEO,oBAAoB,CAAC,KAAa,EAAE,SAAiB;QAC3D,OAAO,GAAG,KAAK,IAAI,SAAS,EAAE,CAAA;IAChC,CAAC;IAEO,sBAAsB,CAAC,GAAW;QACxC,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QACzC,IAAI,CAAC,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QACvD,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,EAAE,CAAA;IAC7D,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@message-queue-toolkit/kafka",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.1",
|
|
4
4
|
"engines": {
|
|
5
5
|
"node": ">= 22.14.0"
|
|
6
6
|
},
|
|
@@ -22,7 +22,11 @@
|
|
|
22
22
|
"notification",
|
|
23
23
|
"kafka"
|
|
24
24
|
],
|
|
25
|
-
"files": [
|
|
25
|
+
"files": [
|
|
26
|
+
"README.md",
|
|
27
|
+
"LICENSE",
|
|
28
|
+
"dist"
|
|
29
|
+
],
|
|
26
30
|
"maintainers": [
|
|
27
31
|
{
|
|
28
32
|
"name": "Igor Savin",
|
|
@@ -49,7 +53,7 @@
|
|
|
49
53
|
"dependencies": {
|
|
50
54
|
"@lokalise/node-core": "^14.2.0",
|
|
51
55
|
"@lokalise/universal-ts-utils": "^4.5.1",
|
|
52
|
-
"@platformatic/kafka": "^1.
|
|
56
|
+
"@platformatic/kafka": "^1.11.0"
|
|
53
57
|
},
|
|
54
58
|
"peerDependencies": {
|
|
55
59
|
"@message-queue-toolkit/core": ">=23.0.0",
|
|
@@ -57,9 +61,9 @@
|
|
|
57
61
|
"zod": ">=3.25.76 <5.0.0"
|
|
58
62
|
},
|
|
59
63
|
"devDependencies": {
|
|
60
|
-
"@biomejs/biome": "^
|
|
61
|
-
"@lokalise/biome-config": "^
|
|
62
|
-
"@lokalise/tsconfig": "^
|
|
64
|
+
"@biomejs/biome": "^2.2.0",
|
|
65
|
+
"@lokalise/biome-config": "^3.1.0",
|
|
66
|
+
"@lokalise/tsconfig": "^3.0.0",
|
|
63
67
|
"@message-queue-toolkit/core": ">=23.0.0",
|
|
64
68
|
"@message-queue-toolkit/schemas": ">=7.0.0",
|
|
65
69
|
"@types/node": "^24.0.3",
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import type { Message } from '@platformatic/kafka';
|
|
2
|
-
import type { ZodSchema } from 'zod/v4';
|
|
3
|
-
import type { RequestContext } from '../types.js';
|
|
4
|
-
export type KafkaHandler<MessageValue extends object, ExecutionContext> = (message: Message<string, MessageValue, string, string>, context: ExecutionContext, requestContext: RequestContext) => Promise<void> | void;
|
|
5
|
-
export declare class KafkaHandlerConfig<MessageValue extends object, ExecutionContext> {
|
|
6
|
-
readonly schema: ZodSchema<MessageValue, any>;
|
|
7
|
-
readonly handler: KafkaHandler<MessageValue, ExecutionContext>;
|
|
8
|
-
constructor(schema: ZodSchema<MessageValue, any>, handler: KafkaHandler<MessageValue, ExecutionContext>);
|
|
9
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"KafkaHandlerConfig.js","sourceRoot":"","sources":["../../lib/handler-container/KafkaHandlerConfig.ts"],"names":[],"mappings":"AAUA,MAAM,OAAO,kBAAkB;IAC7B,2EAA2E;IAC3D,MAAM,CAA8B;IACpC,OAAO,CAA8C;IAErE;IACE,2EAA2E;IAC3E,MAAoC,EACpC,OAAqD;QAErD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;QACpB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAA;IACxB,CAAC;CACF"}
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import type { SupportedMessageValuesForTopic, SupportedTopics, TopicConfig } from '../types.ts';
|
|
2
|
-
import type { KafkaHandlerConfig } from './KafkaHandlerConfig.ts';
|
|
3
|
-
import type { KafkaHandlerRouting } from './KafkaHandlerRoutingBuilder.ts';
|
|
4
|
-
export declare class KafkaHandlerContainer<TopicsConfig extends TopicConfig[], ExecutionContext> {
|
|
5
|
-
private readonly handlers;
|
|
6
|
-
private readonly messageTypeField?;
|
|
7
|
-
constructor(topicHandlers: KafkaHandlerRouting<TopicsConfig, ExecutionContext>, messageTypeField?: string);
|
|
8
|
-
private mapTopicHandlers;
|
|
9
|
-
resolveHandler<Topic extends SupportedTopics<TopicsConfig>>(topic: Topic, messageValue: SupportedMessageValuesForTopic<TopicsConfig, Topic>): KafkaHandlerConfig<SupportedMessageValuesForTopic<TopicsConfig, Topic>, ExecutionContext> | undefined;
|
|
10
|
-
get topics(): SupportedTopics<TopicsConfig>[];
|
|
11
|
-
}
|
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
const DEFAULT_HANDLER_KEY = Symbol('default-handler');
|
|
2
|
-
export class KafkaHandlerContainer {
|
|
3
|
-
handlers;
|
|
4
|
-
messageTypeField;
|
|
5
|
-
constructor(topicHandlers, messageTypeField) {
|
|
6
|
-
this.messageTypeField = messageTypeField;
|
|
7
|
-
this.handlers = this.mapTopicHandlers(topicHandlers);
|
|
8
|
-
}
|
|
9
|
-
mapTopicHandlers(topicHandlerRouting) {
|
|
10
|
-
const result = {};
|
|
11
|
-
for (const [topic, topicHandlers] of Object.entries(topicHandlerRouting)) {
|
|
12
|
-
if (!topicHandlers.length)
|
|
13
|
-
continue;
|
|
14
|
-
result[topic] = {};
|
|
15
|
-
for (const handler of topicHandlers) {
|
|
16
|
-
let handlerKey = this.messageTypeField
|
|
17
|
-
? // @ts-expect-error
|
|
18
|
-
handler.schema.shape[this.messageTypeField]?.value
|
|
19
|
-
: undefined;
|
|
20
|
-
handlerKey ??= DEFAULT_HANDLER_KEY;
|
|
21
|
-
if (result[topic][handlerKey]) {
|
|
22
|
-
throw new Error(`Duplicate handler for topic ${topic}`);
|
|
23
|
-
}
|
|
24
|
-
result[topic][handlerKey] = handler;
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
return result;
|
|
28
|
-
}
|
|
29
|
-
resolveHandler(topic, messageValue) {
|
|
30
|
-
const handlers = this.handlers[topic];
|
|
31
|
-
if (!handlers)
|
|
32
|
-
return undefined;
|
|
33
|
-
let messageValueType = undefined;
|
|
34
|
-
// @ts-expect-error
|
|
35
|
-
if (this.messageTypeField)
|
|
36
|
-
messageValueType = messageValue[this.messageTypeField];
|
|
37
|
-
return messageValueType
|
|
38
|
-
? (handlers[messageValueType] ?? handlers[DEFAULT_HANDLER_KEY])
|
|
39
|
-
: handlers[DEFAULT_HANDLER_KEY];
|
|
40
|
-
}
|
|
41
|
-
get topics() {
|
|
42
|
-
return Object.keys(this.handlers);
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
//# sourceMappingURL=KafkaHandlerContainer.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"KafkaHandlerContainer.js","sourceRoot":"","sources":["../../lib/handler-container/KafkaHandlerContainer.ts"],"names":[],"mappings":"AASA,MAAM,mBAAmB,GAAG,MAAM,CAAC,iBAAiB,CAAC,CAAA;AAUrD,MAAM,OAAO,qBAAqB;IACf,QAAQ,CAA0C;IAClD,gBAAgB,CAAS;IAE1C,YACE,aAAkE,EAClE,gBAAyB;QAEzB,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAA;QACxC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAA;IACtD,CAAC;IAEO,gBAAgB,CACtB,mBAAwE;QAExE,MAAM,MAAM,GAA6C,EAAE,CAAA;QAE3D,KAAK,MAAM,CAAC,KAAK,EAAE,aAAa,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,mBAAmB,CAAC,EAAE,CAAC;YACzE,IAAI,CAAC,aAAa,CAAC,MAAM;gBAAE,SAAQ;YACnC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAA;YAElB,KAAK,MAAM,OAAO,IAAI,aAAa,EAAE,CAAC;gBACpC,IAAI,UAAU,GAAG,IAAI,CAAC,gBAAgB;oBACpC,CAAC,CAAC,mBAAmB;wBACnB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,KAAK;oBACpD,CAAC,CAAC,SAAS,CAAA;gBACb,UAAU,KAAK,mBAAmB,CAAA;gBAClC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC9B,MAAM,IAAI,KAAK,CAAC,+BAA+B,KAAK,EAAE,CAAC,CAAA;gBACzD,CAAC;gBAED,MAAM,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,GAAG,OAAO,CAAA;YACrC,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;IAED,cAAc,CACZ,KAAY,EACZ,YAAiE;QAIjE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;QACrC,IAAI,CAAC,QAAQ;YAAE,OAAO,SAAS,CAAA;QAE/B,IAAI,gBAAgB,GAAuB,SAAS,CAAA;QACpD,mBAAmB;QACnB,IAAI,IAAI,CAAC,gBAAgB;YAAE,gBAAgB,GAAG,YAAY,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QAEjF,OAAO,gBAAgB;YACrB,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,QAAQ,CAAC,mBAAmB,CAAC,CAAC;YAC/D,CAAC,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAA;IACnC,CAAC;IAED,IAAI,MAAM;QACR,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACnC,CAAC;CACF"}
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import type { SupportedMessageValues, SupportedMessageValuesForTopic, SupportedTopics, TopicConfig } from '../types.ts';
|
|
2
|
-
import type { KafkaHandlerConfig } from './KafkaHandlerConfig.ts';
|
|
3
|
-
export type KafkaHandlerRouting<TopicsConfig extends TopicConfig[], ExecutionContext> = Record<string, KafkaHandlerConfig<SupportedMessageValues<TopicsConfig>, ExecutionContext>[]>;
|
|
4
|
-
export declare class KafkaHandlerRoutingBuilder<const TopicsConfig extends TopicConfig[], ExecutionContext> {
|
|
5
|
-
private readonly configs;
|
|
6
|
-
addConfig<Topic extends SupportedTopics<TopicsConfig>, MessageValue extends SupportedMessageValuesForTopic<TopicsConfig, Topic>>(topic: Topic, config: KafkaHandlerConfig<MessageValue, ExecutionContext>): this;
|
|
7
|
-
build(): KafkaHandlerRouting<TopicsConfig, ExecutionContext>;
|
|
8
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"KafkaHandlerRoutingBuilder.js","sourceRoot":"","sources":["../../lib/handler-container/KafkaHandlerRoutingBuilder.ts"],"names":[],"mappings":"AAaA,MAAM,OAAO,0BAA0B;IAIpB,OAAO,GAAwD,EAAE,CAAA;IAElF,SAAS,CAGP,KAAY,EAAE,MAA0D;QACxE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;QAC1B,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CACtB,MAAoF,CACrF,CAAA;QAED,OAAO,IAAI,CAAA;IACb,CAAC;IAED,KAAK;QACH,OAAO,IAAI,CAAC,OAAO,CAAA;IACrB,CAAC;CACF"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../lib/handler-container/index.ts"],"names":[],"mappings":"AAAA,cAAc,yBAAyB,CAAA;AACvC,OAAO,EAAE,0BAA0B,EAAE,MAAM,iCAAiC,CAAA"}
|
|
File without changes
|
|
File without changes
|