@message-queue-toolkit/kafka 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/dist/AbstractKafkaConsumer.d.ts +22 -0
- package/dist/AbstractKafkaConsumer.js +145 -0
- package/dist/AbstractKafkaConsumer.js.map +1 -0
- package/dist/AbstractKafkaPublisher.d.ts +18 -0
- package/dist/AbstractKafkaPublisher.js +95 -0
- package/dist/AbstractKafkaPublisher.js.map +1 -0
- package/dist/AbstractKafkaService.d.ts +36 -0
- package/dist/AbstractKafkaService.js +53 -0
- package/dist/AbstractKafkaService.js.map +1 -0
- package/dist/handler-container/KafkaHandlerConfig.d.ts +13 -0
- package/dist/handler-container/KafkaHandlerConfig.js +9 -0
- package/dist/handler-container/KafkaHandlerConfig.js.map +1 -0
- package/dist/handler-container/KafkaHandlerContainer.d.ts +11 -0
- package/dist/handler-container/KafkaHandlerContainer.js +44 -0
- package/dist/handler-container/KafkaHandlerContainer.js.map +1 -0
- package/dist/handler-container/KafkaHandlerRoutingBuilder.d.ts +8 -0
- package/dist/handler-container/KafkaHandlerRoutingBuilder.js +12 -0
- package/dist/handler-container/KafkaHandlerRoutingBuilder.js.map +1 -0
- package/dist/handler-container/index.d.ts +2 -0
- package/dist/handler-container/index.js +3 -0
- package/dist/handler-container/index.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -0
- package/dist/types.d.ts +23 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/package.json +74 -0
package/README.md
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { QueueConsumerDependencies } from '@message-queue-toolkit/core';
|
|
2
|
+
import { type ConsumeOptions, type ConsumerOptions } from '@platformatic/kafka';
|
|
3
|
+
import { AbstractKafkaService, type BaseKafkaOptions } from './AbstractKafkaService.ts';
|
|
4
|
+
import type { KafkaHandlerRouting } from './handler-container/KafkaHandlerRoutingBuilder.ts';
|
|
5
|
+
import type { KafkaDependencies, TopicConfig } from './types.ts';
|
|
6
|
+
export type KafkaConsumerDependencies = KafkaDependencies & Pick<QueueConsumerDependencies, 'transactionObservabilityManager'>;
|
|
7
|
+
export type KafkaConsumerOptions<TopicsConfig extends TopicConfig[]> = BaseKafkaOptions & Omit<ConsumerOptions<string, object, string, string>, 'deserializers' | 'autocommit' | 'bootstrapBrokers'> & Omit<ConsumeOptions<string, object, string, string>, 'topics'> & {
|
|
8
|
+
handlers: KafkaHandlerRouting<TopicsConfig>;
|
|
9
|
+
};
|
|
10
|
+
export declare abstract class AbstractKafkaConsumer<TopicsConfig extends TopicConfig[]> extends AbstractKafkaService<TopicsConfig, KafkaConsumerOptions<TopicsConfig>> {
|
|
11
|
+
private readonly consumer;
|
|
12
|
+
private consumerStream?;
|
|
13
|
+
private readonly transactionObservabilityManager;
|
|
14
|
+
private readonly handlerContainer;
|
|
15
|
+
constructor(dependencies: KafkaConsumerDependencies, options: KafkaConsumerOptions<TopicsConfig>);
|
|
16
|
+
init(): Promise<void>;
|
|
17
|
+
close(): Promise<void>;
|
|
18
|
+
private consume;
|
|
19
|
+
private tryToConsume;
|
|
20
|
+
private buildTransactionName;
|
|
21
|
+
private getRequestContext;
|
|
22
|
+
}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { setTimeout } from 'node:timers/promises';
|
|
3
|
+
import { InternalError, stringValueSerializer, } from '@lokalise/node-core';
|
|
4
|
+
import { Consumer, jsonDeserializer, stringDeserializer, } from '@platformatic/kafka';
|
|
5
|
+
import { AbstractKafkaService } from "./AbstractKafkaService.js";
|
|
6
|
+
import { KafkaHandlerContainer } from "./handler-container/KafkaHandlerContainer.js";
|
|
7
|
+
/*
|
|
8
|
+
TODO: Proper retry mechanism + DLQ -> https://lokalise.atlassian.net/browse/EDEXP-498
|
|
9
|
+
In the meantime, we will retry in memory up to 3 times
|
|
10
|
+
*/
|
|
11
|
+
const MAX_IN_MEMORY_RETRIES = 3;
|
|
12
|
+
export class AbstractKafkaConsumer extends AbstractKafkaService {
|
|
13
|
+
consumer;
|
|
14
|
+
consumerStream;
|
|
15
|
+
transactionObservabilityManager;
|
|
16
|
+
handlerContainer;
|
|
17
|
+
constructor(dependencies, options) {
|
|
18
|
+
super(dependencies, options);
|
|
19
|
+
this.transactionObservabilityManager = dependencies.transactionObservabilityManager;
|
|
20
|
+
this.handlerContainer = new KafkaHandlerContainer(options.handlers, options.messageTypeField);
|
|
21
|
+
this.consumer = new Consumer({
|
|
22
|
+
...this.options.kafka,
|
|
23
|
+
...this.options,
|
|
24
|
+
autocommit: false, // Handling commits manually
|
|
25
|
+
deserializers: {
|
|
26
|
+
key: stringDeserializer,
|
|
27
|
+
value: jsonDeserializer,
|
|
28
|
+
headerKey: stringDeserializer,
|
|
29
|
+
headerValue: stringDeserializer,
|
|
30
|
+
},
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
async init() {
|
|
34
|
+
if (this.consumerStream)
|
|
35
|
+
return Promise.resolve();
|
|
36
|
+
const topics = this.handlerContainer.topics;
|
|
37
|
+
if (topics.length === 0)
|
|
38
|
+
throw new Error('At least one topic must be defined');
|
|
39
|
+
try {
|
|
40
|
+
const { handlers, ...consumeOptions } = this.options; // Handlers cannot be passed to consume method
|
|
41
|
+
this.consumerStream = await this.consumer.consume({ ...consumeOptions, topics });
|
|
42
|
+
}
|
|
43
|
+
catch (error) {
|
|
44
|
+
throw new InternalError({
|
|
45
|
+
message: 'Consumer init failed',
|
|
46
|
+
errorCode: 'KAFKA_CONSUMER_INIT_ERROR',
|
|
47
|
+
cause: error,
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
this.consumerStream.on('data', (message) => this.consume(message));
|
|
51
|
+
this.consumerStream.on('error', (error) => this.handlerError(error));
|
|
52
|
+
}
|
|
53
|
+
async close() {
|
|
54
|
+
if (!this.consumerStream)
|
|
55
|
+
return Promise.resolve();
|
|
56
|
+
await new Promise((done) => this.consumerStream?.close(done));
|
|
57
|
+
this.consumerStream = undefined;
|
|
58
|
+
await this.consumer.close();
|
|
59
|
+
}
|
|
60
|
+
async consume(message) {
|
|
61
|
+
const handler = this.handlerContainer.resolveHandler(message.topic, message.value);
|
|
62
|
+
// if there is no handler for the message, we ignore it (simulating subscription)
|
|
63
|
+
if (!handler)
|
|
64
|
+
return message.commit();
|
|
65
|
+
/* v8 ignore next */
|
|
66
|
+
const transactionId = this.resolveMessageId(message.value) ?? randomUUID();
|
|
67
|
+
this.transactionObservabilityManager?.start(this.buildTransactionName(message), transactionId);
|
|
68
|
+
const parseResult = handler.schema.safeParse(message.value);
|
|
69
|
+
if (!parseResult.success) {
|
|
70
|
+
this.handlerError(parseResult.error, {
|
|
71
|
+
topic: message.topic,
|
|
72
|
+
message: stringValueSerializer(message.value),
|
|
73
|
+
});
|
|
74
|
+
this.handleMessageProcessed({
|
|
75
|
+
topic: message.topic,
|
|
76
|
+
message: message.value,
|
|
77
|
+
processingResult: { status: 'error', errorReason: 'invalidMessage' },
|
|
78
|
+
});
|
|
79
|
+
return message.commit();
|
|
80
|
+
}
|
|
81
|
+
const validatedMessage = parseResult.data;
|
|
82
|
+
const requestContext = this.getRequestContext(message);
|
|
83
|
+
let retries = 0;
|
|
84
|
+
let consumed = false;
|
|
85
|
+
do {
|
|
86
|
+
// exponential backoff -> 2^(retry-1)
|
|
87
|
+
if (retries > 0)
|
|
88
|
+
await setTimeout(Math.pow(2, retries - 1));
|
|
89
|
+
consumed = await this.tryToConsume({ ...message, value: validatedMessage }, handler.handler, requestContext);
|
|
90
|
+
if (consumed)
|
|
91
|
+
break;
|
|
92
|
+
retries++;
|
|
93
|
+
} while (retries < MAX_IN_MEMORY_RETRIES);
|
|
94
|
+
if (consumed) {
|
|
95
|
+
this.handleMessageProcessed({
|
|
96
|
+
topic: message.topic,
|
|
97
|
+
message: validatedMessage,
|
|
98
|
+
processingResult: { status: 'consumed' },
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
else {
|
|
102
|
+
this.handleMessageProcessed({
|
|
103
|
+
topic: message.topic,
|
|
104
|
+
message: validatedMessage,
|
|
105
|
+
processingResult: { status: 'error', errorReason: 'handlerError' },
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
this.transactionObservabilityManager?.stop(transactionId);
|
|
109
|
+
return message.commit();
|
|
110
|
+
}
|
|
111
|
+
async tryToConsume(message, handler, requestContext) {
|
|
112
|
+
try {
|
|
113
|
+
await handler(message, requestContext);
|
|
114
|
+
return true;
|
|
115
|
+
}
|
|
116
|
+
catch (error) {
|
|
117
|
+
this.handlerError(error, {
|
|
118
|
+
topic: message.topic,
|
|
119
|
+
message: stringValueSerializer(message.value),
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
return false;
|
|
123
|
+
}
|
|
124
|
+
buildTransactionName(message) {
|
|
125
|
+
const messageType = this.resolveMessageType(message.value);
|
|
126
|
+
let name = `kafka:${message.topic}`;
|
|
127
|
+
if (messageType?.trim().length)
|
|
128
|
+
name += `:${messageType.trim()}`;
|
|
129
|
+
return name;
|
|
130
|
+
}
|
|
131
|
+
getRequestContext(message) {
|
|
132
|
+
let reqId = message.headers.get(this.resolveHeaderRequestIdField());
|
|
133
|
+
if (!reqId || reqId.trim().length === 0)
|
|
134
|
+
reqId = randomUUID();
|
|
135
|
+
return {
|
|
136
|
+
reqId,
|
|
137
|
+
logger: this.logger.child({
|
|
138
|
+
'x-request-id': reqId,
|
|
139
|
+
topic: message.topic,
|
|
140
|
+
messageKey: message.key,
|
|
141
|
+
}),
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
//# sourceMappingURL=AbstractKafkaConsumer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AbstractKafkaConsumer.js","sourceRoot":"","sources":["../lib/AbstractKafkaConsumer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EACL,aAAa,EAEb,qBAAqB,GACtB,MAAM,qBAAqB,CAAA;AAE5B,OAAO,EAEL,QAAQ,EAIR,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,oBAAoB,EAAyB,MAAM,2BAA2B,CAAA;AACvF,OAAO,EAAE,qBAAqB,EAAE,MAAM,8CAA8C,CAAA;AAiBpF;;;GAGG;AACH,MAAM,qBAAqB,GAAG,CAAC,CAAA;AAE/B,MAAM,OAAgB,qBAEpB,SAAQ,oBAAsE;IAC7D,QAAQ,CAA0C;IAC3D,cAAc,CAAiD;IAEtD,+BAA+B,CAAiC;IAChE,gBAAgB,CAAqC;IAEtE,YACE,YAAuC,EACvC,OAA2C;QAE3C,KAAK,CAAC,YAAY,EAAE,OAAO,CAAC,CAAA;QAE5B,IAAI,CAAC,+BAA+B,GAAG,YAAY,CAAC,+BAA+B,CAAA;QACnF,IAAI,CAAC,gBAAgB,GAAG,IAAI,qBAAqB,CAC/C,OAAO,CAAC,QAAQ,EAChB,OAAO,CAAC,gBAAgB,CACzB,CAAA;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC;YAC3B,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK;YACrB,GAAG,IAAI,CAAC,OAAO;YACf,UAAU,EAAE,KAAK,EAAE,4BAA4B;YAC/C,aAAa,EAAE;gBACb,GAAG,EAAE,kBAAkB;gBACvB,KAAK,EAAE,gBAAgB;gBACvB,SAAS,EAAE,kBAAkB;gBAC7B,WAAW,EAAE,kBAAkB;aAChC;SACF,CAAC,CAAA;IACJ,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAA;QAC3C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QAE9E,IAAI,CAAC;YACH,MAAM,EAAE,QAAQ,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC,OAAO,CAAA,CAAC,8CAA8C;YACnG,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,GAAG,cAAc,EAAE,MAAM,EAAE,CAAC,CAAA;QAClF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,aAAa,CAAC;gBACtB,OAAO,EAAE,sBAAsB;gBAC/B,SAAS,EAAE,2BAA2B;gBACtC,KAAK,EAAE,KAAK;aACb,CAAC,CAAA;QACJ,CAAC;QAED,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAA;QAClE,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;IACtE,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,cAAc;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;QAElD,MAAM,IAAI,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;QAC7D,IAAI,CAAC,cAAc,GAAG,SAAS,CAAA;QAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;IAC7B,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,OAAgD;QACpE,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;QAClF,iFAAiF;QACjF,IAAI,CAAC,OAAO;YAAE,OAAO,OAAO,CAAC,MAAM,EAAE,CAAA;QAErC,oBAAoB;QACpB,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,UAAU,EAAE,CAAA;QAC1E,IAAI,CAAC,+BAA+B,EAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,EAAE,aAAa,CAAC,CAAA;QAE9F,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAC3D,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;YACzB,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,EAAE;gBACnC,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC;aAC9C,CAAC,CAAA;YACF,IAAI,CAAC,sBAAsB,CAAC;gBAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,OAAO,EAAE,OAAO,CAAC,KAAK;gBACtB,gBAAgB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE;aACrE,CAAC,CAAA;YAEF,OAAO,OAAO,CAAC,MAAM,EAAE,CAAA;QACzB,CAAC;QAED,MAAM,gBAAgB,GAAG,WAAW,CAAC,IAAI,CAAA;QAEzC,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAA;QAEtD,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,QAAQ,GAAG,KAAK,CAAA;QACpB,GAAG,CAAC;YACF,qCAAqC;YACrC,IAAI,OAAO,GAAG,CAAC;gBAAE,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAA;YAE3D,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAChC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAE,EACvC,OAAO,CAAC,OAAO,EACf,cAAc,CACf,CAAA;YACD,IAAI,QAAQ;gBAAE,MAAK;YAEnB,OAAO,EAAE,CAAA;QACX,CAAC,QAAQ,OAAO,GAAG,qBAAqB,EAAC;QAEzC,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,CAAC,sBAAsB,CAAC;gBAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,OAAO,EAAE,gBAAgB;gBACzB,gBAAgB,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE;aACzC,CAAC,CAAA;QACJ,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,sBAAsB,CAAC;gBAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,OAAO,EAAE,gBAAgB;gBACzB,gBAAgB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE;aACnE,CAAC,CAAA;QACJ,CAAC;QAED,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,aAAa,CAAC,CAAA;QAEzD,OAAO,OAAO,CAAC,MAAM,EAAE,CAAA;IACzB,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,OAAsD,EACtD,OAAmC,EACnC,cAA8B;QAE9B,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,OAAO,EAAE,cAAc,CAAC,CAAA;YACtC,OAAO,IAAI,CAAA;QACb,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE;gBACvB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC;aAC9C,CAAC,CAAA;QACJ,CAAC;QAED,OAAO,KAAK,CAAA;IACd,CAAC;IAEO,oBAAoB,CAAC,OAAgD;QAC3E,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAE1D,IAAI,IAAI,GAAG,SAAS,OAAO,CAAC,KAAK,EAAE,CAAA;QACnC,IAAI,WAAW,EAAE,IAAI,EAAE,CAAC,MAAM;YAAE,IAAI,IAAI,IAAI,WAAW,CAAC,IAAI,EAAE,EAAE,CAAA;QAEhE,OAAO,IAAI,CAAA;IACb,CAAC;IAEO,iBAAiB,CAAC,OAAgD;QACxE,IAAI,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC,CAAA;QACnE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,KAAK,GAAG,UAAU,EAAE,CAAA;QAE7D,OAAO;YACL,KAAK;YACL,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;gBACxB,cAAc,EAAE,KAAK;gBACrB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,UAAU,EAAE,OAAO,CAAC,GAAG;aACxB,CAAC;SACH,CAAA;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { type MessageToProduce, type ProduceOptions } from '@platformatic/kafka';
|
|
2
|
+
import { AbstractKafkaService, type BaseKafkaOptions } from './AbstractKafkaService.ts';
|
|
3
|
+
import type { RequestContext } from './handler-container/index.js';
|
|
4
|
+
import type { KafkaDependencies, SupportedMessageValuesInputForTopic, SupportedTopics, TopicConfig } from './types.ts';
|
|
5
|
+
export type KafkaPublisherOptions<TopicsConfig extends TopicConfig[]> = BaseKafkaOptions & Omit<ProduceOptions<string, object, string, string>, 'serializers'> & {
|
|
6
|
+
topicsConfig: TopicsConfig;
|
|
7
|
+
};
|
|
8
|
+
export type KafkaMessageOptions = Omit<MessageToProduce<string, object, string, string>, 'topic' | 'value'>;
|
|
9
|
+
export declare abstract class AbstractKafkaPublisher<TopicsConfig extends TopicConfig[]> extends AbstractKafkaService<TopicsConfig, KafkaPublisherOptions<TopicsConfig>> {
|
|
10
|
+
private readonly topicsConfig;
|
|
11
|
+
private readonly schemaContainers;
|
|
12
|
+
private readonly producer;
|
|
13
|
+
private isInitiated;
|
|
14
|
+
constructor(dependencies: KafkaDependencies, options: KafkaPublisherOptions<TopicsConfig>);
|
|
15
|
+
init(): Promise<void>;
|
|
16
|
+
close(): Promise<void>;
|
|
17
|
+
publish<Topic extends SupportedTopics<TopicsConfig>>(topic: Topic, message: SupportedMessageValuesInputForTopic<TopicsConfig, Topic>, requestContext?: RequestContext, options?: KafkaMessageOptions): Promise<void>;
|
|
18
|
+
}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { InternalError, stringValueSerializer } from '@lokalise/node-core';
|
|
2
|
+
import { MessageSchemaContainer } from '@message-queue-toolkit/core';
|
|
3
|
+
import { Producer, jsonSerializer, stringSerializer, } from '@platformatic/kafka';
|
|
4
|
+
import { AbstractKafkaService } from "./AbstractKafkaService.js";
|
|
5
|
+
export class AbstractKafkaPublisher extends AbstractKafkaService {
|
|
6
|
+
topicsConfig;
|
|
7
|
+
schemaContainers;
|
|
8
|
+
producer;
|
|
9
|
+
isInitiated;
|
|
10
|
+
constructor(dependencies, options) {
|
|
11
|
+
super(dependencies, options);
|
|
12
|
+
this.isInitiated = false;
|
|
13
|
+
this.topicsConfig = options.topicsConfig;
|
|
14
|
+
if (this.topicsConfig.length === 0)
|
|
15
|
+
throw new Error('At least one topic must be defined');
|
|
16
|
+
this.schemaContainers = {};
|
|
17
|
+
for (const { topic, schemas } of this.topicsConfig) {
|
|
18
|
+
this.schemaContainers[topic] = new MessageSchemaContainer({
|
|
19
|
+
messageSchemas: schemas,
|
|
20
|
+
messageTypeField: this.options.messageTypeField,
|
|
21
|
+
messageDefinitions: [],
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
this.producer = new Producer({
|
|
25
|
+
...this.options.kafka,
|
|
26
|
+
...this.options,
|
|
27
|
+
serializers: {
|
|
28
|
+
key: stringSerializer,
|
|
29
|
+
value: jsonSerializer,
|
|
30
|
+
headerKey: stringSerializer,
|
|
31
|
+
headerValue: stringSerializer,
|
|
32
|
+
},
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
async init() {
|
|
36
|
+
if (this.isInitiated)
|
|
37
|
+
return;
|
|
38
|
+
try {
|
|
39
|
+
await this.producer.listApis();
|
|
40
|
+
this.isInitiated = true;
|
|
41
|
+
}
|
|
42
|
+
catch (e) {
|
|
43
|
+
throw new InternalError({
|
|
44
|
+
message: 'Producer init failed',
|
|
45
|
+
errorCode: 'KAFKA_PRODUCER_INIT_ERROR',
|
|
46
|
+
cause: e,
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
async close() {
|
|
51
|
+
if (!this.isInitiated)
|
|
52
|
+
return;
|
|
53
|
+
await this.producer.close();
|
|
54
|
+
this.isInitiated = false;
|
|
55
|
+
}
|
|
56
|
+
async publish(topic, message, requestContext, options) {
|
|
57
|
+
const schemaResult = this.schemaContainers[topic]?.resolveSchema(message);
|
|
58
|
+
if (!schemaResult)
|
|
59
|
+
throw new Error(`Message schemas not found for topic: ${topic}`);
|
|
60
|
+
if (schemaResult.error)
|
|
61
|
+
throw schemaResult.error;
|
|
62
|
+
await this.init(); // lazy init
|
|
63
|
+
try {
|
|
64
|
+
const parsedMessage = schemaResult.result.parse(message);
|
|
65
|
+
const headers = {
|
|
66
|
+
...options?.headers,
|
|
67
|
+
[this.resolveHeaderRequestIdField()]: requestContext?.reqId ?? '',
|
|
68
|
+
};
|
|
69
|
+
// biome-ignore lint/style/noNonNullAssertion: Should always exist due to lazy init
|
|
70
|
+
await this.producer.send({
|
|
71
|
+
messages: [{ ...options, topic, value: parsedMessage, headers }],
|
|
72
|
+
});
|
|
73
|
+
this.handleMessageProcessed({
|
|
74
|
+
message: parsedMessage,
|
|
75
|
+
processingResult: { status: 'published' },
|
|
76
|
+
topic,
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
catch (error) {
|
|
80
|
+
const errorDetails = {
|
|
81
|
+
topic,
|
|
82
|
+
publisher: this.constructor.name,
|
|
83
|
+
message: stringValueSerializer(message),
|
|
84
|
+
};
|
|
85
|
+
this.handlerError(error, errorDetails);
|
|
86
|
+
throw new InternalError({
|
|
87
|
+
message: `Error while publishing to Kafka: ${error.message}`,
|
|
88
|
+
errorCode: 'KAFKA_PUBLISH_ERROR',
|
|
89
|
+
cause: error,
|
|
90
|
+
details: errorDetails,
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
//# sourceMappingURL=AbstractKafkaPublisher.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AbstractKafkaPublisher.js","sourceRoot":"","sources":["../lib/AbstractKafkaPublisher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAA;AAC1E,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAA;AACpE,OAAO,EAGL,QAAQ,EACR,cAAc,EACd,gBAAgB,GACjB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,oBAAoB,EAAyB,MAAM,2BAA2B,CAAA;AAoBvF,MAAM,OAAgB,sBAEpB,SAAQ,oBAAuE;IAC9D,YAAY,CAAc;IAC1B,gBAAgB,CAGhC;IAEgB,QAAQ,CAA0C;IAC3D,WAAW,CAAS;IAE5B,YAAY,YAA+B,EAAE,OAA4C;QACvF,KAAK,CAAC,YAAY,EAAE,OAAO,CAAC,CAAA;QAC5B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAA;QAExB,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAA;QACxC,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QAEzF,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAA;QAC1B,KAAK,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACnD,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,GAAG,IAAI,sBAAsB,CAAC;gBACxD,cAAc,EAAE,OAAO;gBACvB,gBAAgB,EAAE,IAAI,CAAC,OAAO,CAAC,gBAAgB;gBAC/C,kBAAkB,EAAE,EAAE;aACvB,CAAC,CAAA;QACJ,CAAC;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC;YAC3B,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK;YACrB,GAAG,IAAI,CAAC,OAAO;YACf,WAAW,EAAE;gBACX,GAAG,EAAE,gBAAgB;gBACrB,KAAK,EAAE,cAAc;gBACrB,SAAS,EAAE,gBAAgB;gBAC3B,WAAW,EAAE,gBAAgB;aAC9B;SACF,CAAC,CAAA;IACJ,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,WAAW;YAAE,OAAM;QAE5B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAA;YAC9B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;QACzB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,IAAI,aAAa,CAAC;gBACtB,OAAO,EAAE,sBAAsB;gBAC/B,SAAS,EAAE,2BAA2B;gBACtC,KAAK,EAAE,CAAC;aACT,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,WAAW;YAAE,OAAM;QAE7B,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;QAC3B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAA;IAC1B,CAAC;IAED,KAAK,CAAC,OAAO,CACX,KAAY,EACZ,OAAiE,EACjE,cAA+B,EAC/B,OAA6B;QAE7B,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,CAAA;QACzE,IAAI,CAAC,YAAY;YAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,KAAK,EAAE,CAAC,CAAA;QACnF,IAAI,YAAY,CAAC,KAAK;YAAE,MAAM,YAAY,CAAC,KAAK,CAAA;QAEhD,MAAM,IAAI,CAAC,IAAI,EAAE,CAAA,CAAC,YAAY;QAE9B,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;YAExD,MAAM,OAAO,GAAG;gBACd,GAAG,OAAO,EAAE,OAAO;gBACnB,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC,EAAE,cAAc,EAAE,KAAK,IAAI,EAAE;aAClE,CAAA;YAED,mFAAmF;YACnF,MAAM,IAAI,CAAC,QAAS,CAAC,IAAI,CAAC;gBACxB,QAAQ,EAAE,CAAC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC;aACjE,CAAC,CAAA;YAEF,IAAI,CAAC,sBAAsB,CAAC;gBAC1B,OAAO,EAAE,aAAa;gBACtB,gBAAgB,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE;gBACzC,KAAK;aACN,CAAC,CAAA;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG;gBACnB,KAAK;gBACL,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI;gBAChC,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC;aACxC,CAAA;YACD,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,YAAY,CAAC,CAAA;YACtC,MAAM,IAAI,aAAa,CAAC;gBACtB,OAAO,EAAE,oCAAqC,KAAe,CAAC,OAAO,EAAE;gBACvE,SAAS,EAAE,qBAAqB;gBAChC,KAAK,EAAE,KAAK;gBACZ,OAAO,EAAE,YAAY;aACtB,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { type CommonLogger, type ErrorReporter } from '@lokalise/node-core';
|
|
2
|
+
import { type HandlerSpy, type HandlerSpyParams, type MessageProcessingResult, type PublicHandlerSpy } from '@message-queue-toolkit/core';
|
|
3
|
+
import type { BaseOptions } from '@platformatic/kafka';
|
|
4
|
+
import type { KafkaConfig, KafkaDependencies, SupportedMessageValues, TopicConfig } from './types.ts';
|
|
5
|
+
export type BaseKafkaOptions = {
|
|
6
|
+
kafka: KafkaConfig;
|
|
7
|
+
messageTypeField?: string;
|
|
8
|
+
messageIdField?: string;
|
|
9
|
+
/**
|
|
10
|
+
* The field in the message headers that contains the request ID.
|
|
11
|
+
* This is used to correlate logs and transactions with the request.
|
|
12
|
+
* Defaults to 'x-request-id'.
|
|
13
|
+
*/
|
|
14
|
+
headerRequestIdField?: string;
|
|
15
|
+
handlerSpy?: HandlerSpy<object> | HandlerSpyParams | boolean;
|
|
16
|
+
logMessages?: boolean;
|
|
17
|
+
} & Omit<BaseOptions, keyof KafkaConfig>;
|
|
18
|
+
export declare abstract class AbstractKafkaService<TopicsConfig extends TopicConfig[], KafkaOptions extends BaseKafkaOptions> {
|
|
19
|
+
protected readonly errorReporter: ErrorReporter;
|
|
20
|
+
protected readonly logger: CommonLogger;
|
|
21
|
+
protected readonly options: KafkaOptions;
|
|
22
|
+
protected readonly _handlerSpy?: HandlerSpy<SupportedMessageValues<TopicsConfig>>;
|
|
23
|
+
constructor(dependencies: KafkaDependencies, options: KafkaOptions);
|
|
24
|
+
abstract init(): Promise<void>;
|
|
25
|
+
abstract close(): Promise<void>;
|
|
26
|
+
get handlerSpy(): PublicHandlerSpy<SupportedMessageValues<TopicsConfig>>;
|
|
27
|
+
protected resolveMessageType(message: SupportedMessageValues<TopicsConfig>): string | undefined;
|
|
28
|
+
protected resolveMessageId(message: SupportedMessageValues<TopicsConfig>): string | undefined;
|
|
29
|
+
protected resolveHeaderRequestIdField(): string;
|
|
30
|
+
protected handleMessageProcessed(params: {
|
|
31
|
+
message: SupportedMessageValues<TopicsConfig>;
|
|
32
|
+
processingResult: MessageProcessingResult;
|
|
33
|
+
topic: string;
|
|
34
|
+
}): void;
|
|
35
|
+
protected handlerError(error: unknown, context?: Record<string, unknown>): void;
|
|
36
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { types } from 'node:util';
|
|
2
|
+
import { resolveGlobalErrorLogObject, stringValueSerializer, } from '@lokalise/node-core';
|
|
3
|
+
import { resolveHandlerSpy, } from '@message-queue-toolkit/core';
|
|
4
|
+
export class AbstractKafkaService {
|
|
5
|
+
errorReporter;
|
|
6
|
+
logger;
|
|
7
|
+
options;
|
|
8
|
+
_handlerSpy;
|
|
9
|
+
constructor(dependencies, options) {
|
|
10
|
+
this.logger = dependencies.logger;
|
|
11
|
+
this.errorReporter = dependencies.errorReporter;
|
|
12
|
+
this.options = options;
|
|
13
|
+
this._handlerSpy = resolveHandlerSpy(options);
|
|
14
|
+
}
|
|
15
|
+
get handlerSpy() {
|
|
16
|
+
if (this._handlerSpy)
|
|
17
|
+
return this._handlerSpy;
|
|
18
|
+
throw new Error('HandlerSpy was not instantiated, please pass `handlerSpy` parameter during creation.');
|
|
19
|
+
}
|
|
20
|
+
resolveMessageType(message) {
|
|
21
|
+
if (!this.options.messageTypeField)
|
|
22
|
+
return undefined;
|
|
23
|
+
return message[this.options.messageTypeField];
|
|
24
|
+
}
|
|
25
|
+
resolveMessageId(message) {
|
|
26
|
+
if (!this.options.messageIdField)
|
|
27
|
+
return undefined;
|
|
28
|
+
return message[this.options.messageIdField];
|
|
29
|
+
}
|
|
30
|
+
resolveHeaderRequestIdField() {
|
|
31
|
+
return this.options.headerRequestIdField ?? 'x-request-id';
|
|
32
|
+
}
|
|
33
|
+
handleMessageProcessed(params) {
|
|
34
|
+
const { message, processingResult, topic } = params;
|
|
35
|
+
const messageId = this.resolveMessageId(message);
|
|
36
|
+
this._handlerSpy?.addProcessedMessage({ message, processingResult }, messageId);
|
|
37
|
+
if (this.options.logMessages) {
|
|
38
|
+
this.logger.debug({
|
|
39
|
+
message: stringValueSerializer(message),
|
|
40
|
+
topic,
|
|
41
|
+
processingResult,
|
|
42
|
+
messageId,
|
|
43
|
+
messageType: this.resolveMessageType(message),
|
|
44
|
+
}, `Finished processing message ${messageId}`);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
handlerError(error, context = {}) {
|
|
48
|
+
this.logger.error({ ...resolveGlobalErrorLogObject(error), ...context });
|
|
49
|
+
if (types.isNativeError(error))
|
|
50
|
+
this.errorReporter.report({ error, context });
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
//# sourceMappingURL=AbstractKafkaService.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AbstractKafkaService.js","sourceRoot":"","sources":["../lib/AbstractKafkaService.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAA;AACjC,OAAO,EAGL,2BAA2B,EAC3B,qBAAqB,GACtB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAKL,iBAAiB,GAClB,MAAM,6BAA6B,CAAA;AAuBpC,MAAM,OAAgB,oBAAoB;IAIrB,aAAa,CAAe;IAC5B,MAAM,CAAc;IAEpB,OAAO,CAAc;IACrB,WAAW,CAAmD;IAEjF,YAAY,YAA+B,EAAE,OAAqB;QAChE,IAAI,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAA;QACjC,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC,aAAa,CAAA;QAC/C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAA;QAEtB,IAAI,CAAC,WAAW,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAA;IAC/C,CAAC;IAKD,IAAI,UAAU;QACZ,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC,WAAW,CAAA;QAE7C,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAA;IACH,CAAC;IAES,kBAAkB,CAAC,OAA6C;QACxE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB;YAAE,OAAO,SAAS,CAAA;QACpD,OAAO,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAA;IAC/C,CAAC;IAES,gBAAgB,CAAC,OAA6C;QACtE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc;YAAE,OAAO,SAAS,CAAA;QAClD,OAAO,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;IAC7C,CAAC;IAES,2BAA2B;QACnC,OAAO,IAAI,CAAC,OAAO,CAAC,oBAAoB,IAAI,cAAc,CAAA;IAC5D,CAAC;IAES,sBAAsB,CAAC,MAIhC;QACC,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,KAAK,EAAE,GAAG,MAAM,CAAA;QACnD,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAA;QAEhD,IAAI,CAAC,WAAW,EAAE,mBAAmB,CAAC,EAAE,OAAO,EAAE,gBAAgB,EAAE,EAAE,SAAS,CAAC,CAAA;QAE/E,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,CACf;gBACE,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC;gBACvC,KAAK;gBACL,gBAAgB;gBAChB,SAAS;gBACT,WAAW,EAAE,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC;aAC9C,EACD,+BAA+B,SAAS,EAAE,CAC3C,CAAA;QACH,CAAC;IACH,CAAC;IAES,YAAY,CAAC,KAAc,EAAE,UAAmC,EAAE;QAC1E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,2BAA2B,CAAC,KAAK,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC,CAAA;QACxE,IAAI,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC;YAAE,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAA;IAC/E,CAAC;CACF"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { CommonLogger } from '@lokalise/node-core';
|
|
2
|
+
import type { Message } from '@platformatic/kafka';
|
|
3
|
+
import type { ZodSchema } from 'zod';
|
|
4
|
+
export interface RequestContext {
|
|
5
|
+
logger: CommonLogger;
|
|
6
|
+
reqId: string;
|
|
7
|
+
}
|
|
8
|
+
export type KafkaHandler<MessageValue extends object> = (message: Message<string, MessageValue, string, string>, requestContext: RequestContext) => Promise<void> | void;
|
|
9
|
+
export declare class KafkaHandlerConfig<MessageValue extends object> {
|
|
10
|
+
readonly schema: ZodSchema<MessageValue>;
|
|
11
|
+
readonly handler: KafkaHandler<MessageValue>;
|
|
12
|
+
constructor(schema: ZodSchema<MessageValue>, handler: KafkaHandler<MessageValue>);
|
|
13
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"KafkaHandlerConfig.js","sourceRoot":"","sources":["../../lib/handler-container/KafkaHandlerConfig.ts"],"names":[],"mappings":"AAcA,MAAM,OAAO,kBAAkB;IACb,MAAM,CAAyB;IAC/B,OAAO,CAA4B;IAEnD,YAAY,MAA+B,EAAE,OAAmC;QAC9E,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;QACpB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAA;IACxB,CAAC;CACF"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { SupportedMessageValuesForTopic, SupportedTopics, TopicConfig } from '../types.ts';
|
|
2
|
+
import type { KafkaHandlerConfig } from './KafkaHandlerConfig.ts';
|
|
3
|
+
import type { KafkaHandlerRouting } from './KafkaHandlerRoutingBuilder.ts';
|
|
4
|
+
export declare class KafkaHandlerContainer<TopicsConfig extends TopicConfig[]> {
|
|
5
|
+
private readonly handlers;
|
|
6
|
+
private readonly messageTypeField?;
|
|
7
|
+
constructor(topicHandlers: KafkaHandlerRouting<TopicsConfig>, messageTypeField?: string);
|
|
8
|
+
private mapTopicHandlers;
|
|
9
|
+
resolveHandler<Topic extends SupportedTopics<TopicsConfig>>(topic: Topic, messageValue: SupportedMessageValuesForTopic<TopicsConfig, Topic>): KafkaHandlerConfig<SupportedMessageValuesForTopic<TopicsConfig, Topic>> | undefined;
|
|
10
|
+
get topics(): SupportedTopics<TopicsConfig>[];
|
|
11
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
const DEFAULT_HANDLER_KEY = Symbol('default-handler');
|
|
2
|
+
export class KafkaHandlerContainer {
|
|
3
|
+
handlers;
|
|
4
|
+
messageTypeField;
|
|
5
|
+
constructor(topicHandlers, messageTypeField) {
|
|
6
|
+
this.messageTypeField = messageTypeField;
|
|
7
|
+
this.handlers = this.mapTopicHandlers(topicHandlers);
|
|
8
|
+
}
|
|
9
|
+
mapTopicHandlers(topicHandlerRouting) {
|
|
10
|
+
const result = {};
|
|
11
|
+
for (const [topic, topicHandlers] of Object.entries(topicHandlerRouting)) {
|
|
12
|
+
if (!topicHandlers.length)
|
|
13
|
+
continue;
|
|
14
|
+
result[topic] = {};
|
|
15
|
+
for (const handler of topicHandlers) {
|
|
16
|
+
let handlerKey = this.messageTypeField
|
|
17
|
+
? // @ts-expect-error
|
|
18
|
+
handler.schema.shape[this.messageTypeField]?.value
|
|
19
|
+
: undefined;
|
|
20
|
+
handlerKey ??= DEFAULT_HANDLER_KEY;
|
|
21
|
+
if (result[topic][handlerKey]) {
|
|
22
|
+
throw new Error(`Duplicate handler for topic ${topic}`);
|
|
23
|
+
}
|
|
24
|
+
result[topic][handlerKey] = handler;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
return result;
|
|
28
|
+
}
|
|
29
|
+
resolveHandler(topic, messageValue) {
|
|
30
|
+
const handlers = this.handlers[topic];
|
|
31
|
+
if (!handlers)
|
|
32
|
+
return undefined;
|
|
33
|
+
let messageValueType = undefined;
|
|
34
|
+
if (this.messageTypeField)
|
|
35
|
+
messageValueType = messageValue[this.messageTypeField];
|
|
36
|
+
return messageValueType
|
|
37
|
+
? (handlers[messageValueType] ?? handlers[DEFAULT_HANDLER_KEY])
|
|
38
|
+
: handlers[DEFAULT_HANDLER_KEY];
|
|
39
|
+
}
|
|
40
|
+
get topics() {
|
|
41
|
+
return Object.keys(this.handlers);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=KafkaHandlerContainer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"KafkaHandlerContainer.js","sourceRoot":"","sources":["../../lib/handler-container/KafkaHandlerContainer.ts"],"names":[],"mappings":"AASA,MAAM,mBAAmB,GAAG,MAAM,CAAC,iBAAiB,CAAC,CAAA;AAOrD,MAAM,OAAO,qBAAqB;IACf,QAAQ,CAAwB;IAChC,gBAAgB,CAAS;IAE1C,YAAY,aAAgD,EAAE,gBAAyB;QACrF,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAA;QACxC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAA;IACtD,CAAC;IAEO,gBAAgB,CACtB,mBAAsD;QAEtD,MAAM,MAAM,GAA2B,EAAE,CAAA;QAEzC,KAAK,MAAM,CAAC,KAAK,EAAE,aAAa,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,mBAAmB,CAAC,EAAE,CAAC;YACzE,IAAI,CAAC,aAAa,CAAC,MAAM;gBAAE,SAAQ;YACnC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAA;YAElB,KAAK,MAAM,OAAO,IAAI,aAAa,EAAE,CAAC;gBACpC,IAAI,UAAU,GAAG,IAAI,CAAC,gBAAgB;oBACpC,CAAC,CAAC,mBAAmB;wBACnB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,KAAK;oBACpD,CAAC,CAAC,SAAS,CAAA;gBACb,UAAU,KAAK,mBAAmB,CAAA;gBAClC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC9B,MAAM,IAAI,KAAK,CAAC,+BAA+B,KAAK,EAAE,CAAC,CAAA;gBACzD,CAAC;gBAED,MAAM,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,GAAG,OAAO,CAAA;YACrC,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAA;IACf,CAAC;IAED,cAAc,CACZ,KAAY,EACZ,YAAiE;QAEjE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;QACrC,IAAI,CAAC,QAAQ;YAAE,OAAO,SAAS,CAAA;QAE/B,IAAI,gBAAgB,GAAuB,SAAS,CAAA;QACpD,IAAI,IAAI,CAAC,gBAAgB;YAAE,gBAAgB,GAAG,YAAY,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QAEjF,OAAO,gBAAgB;YACrB,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,QAAQ,CAAC,mBAAmB,CAAC,CAAC;YAC/D,CAAC,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAA;IACnC,CAAC;IAED,IAAI,MAAM;QACR,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACnC,CAAC;CACF"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { SupportedMessageValues, SupportedMessageValuesForTopic, SupportedTopics, TopicConfig } from '../types.ts';
|
|
2
|
+
import type { KafkaHandlerConfig } from './KafkaHandlerConfig.ts';
|
|
3
|
+
export type KafkaHandlerRouting<TopicsConfig extends TopicConfig[], MessageValue extends SupportedMessageValues<TopicsConfig> = SupportedMessageValues<TopicsConfig>> = Record<string, KafkaHandlerConfig<MessageValue>[]>;
|
|
4
|
+
export declare class KafkaHandlerRoutingBuilder<TopicsConfig extends TopicConfig[]> {
|
|
5
|
+
private readonly configs;
|
|
6
|
+
addConfig<Topic extends SupportedTopics<TopicsConfig>, MessageValue extends SupportedMessageValuesForTopic<TopicsConfig, Topic>>(topic: Topic, config: KafkaHandlerConfig<MessageValue>): this;
|
|
7
|
+
build(): KafkaHandlerRouting<TopicsConfig>;
|
|
8
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export class KafkaHandlerRoutingBuilder {
|
|
2
|
+
configs = {};
|
|
3
|
+
addConfig(topic, config) {
|
|
4
|
+
this.configs[topic] ??= [];
|
|
5
|
+
this.configs[topic].push(config);
|
|
6
|
+
return this;
|
|
7
|
+
}
|
|
8
|
+
build() {
|
|
9
|
+
return this.configs;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=KafkaHandlerRoutingBuilder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"KafkaHandlerRoutingBuilder.js","sourceRoot":"","sources":["../../lib/handler-container/KafkaHandlerRoutingBuilder.ts"],"names":[],"mappings":"AAaA,MAAM,OAAO,0BAA0B;IACpB,OAAO,GAAsC,EAAE,CAAA;IAEhE,SAAS,CAGP,KAAY,EAAE,MAAwC;QACtD,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;QAC1B,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAEhC,OAAO,IAAI,CAAA;IACb,CAAC;IAED,KAAK;QACH,OAAO,IAAI,CAAC,OAAO,CAAA;IACrB,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../lib/handler-container/index.ts"],"names":[],"mappings":"AAAA,cAAc,yBAAyB,CAAA;AACvC,OAAO,EAAE,0BAA0B,EAAE,MAAM,iCAAiC,CAAA"}
|
package/dist/index.d.ts
ADDED
package/dist/index.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../lib/index.ts"],"names":[],"mappings":"AAAA,cAAc,8BAA8B,CAAA;AAC5C,cAAc,YAAY,CAAA;AAC1B,cAAc,6BAA6B,CAAA"}
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { QueueDependencies } from '@message-queue-toolkit/core';
|
|
2
|
+
import type { ConnectionOptions } from '@platformatic/kafka';
|
|
3
|
+
import type { ZodSchema } from 'zod';
|
|
4
|
+
import type z from 'zod/v3';
|
|
5
|
+
export type KafkaDependencies = Omit<QueueDependencies, 'messageMetricsManager'>;
|
|
6
|
+
export type KafkaConfig = {
|
|
7
|
+
bootstrapBrokers: string[];
|
|
8
|
+
clientId: string;
|
|
9
|
+
} & ConnectionOptions;
|
|
10
|
+
export type TopicConfig<Topic extends string = string> = {
|
|
11
|
+
topic: Topic;
|
|
12
|
+
schemas: ZodSchema[];
|
|
13
|
+
};
|
|
14
|
+
export type SupportedTopics<TopicsConfig extends TopicConfig[]> = TopicsConfig[number]['topic'];
|
|
15
|
+
type MessageSchemasForTopic<TopicsConfig extends TopicConfig[], Topic extends SupportedTopics<TopicsConfig>> = Extract<TopicsConfig[number], {
|
|
16
|
+
topic: Topic;
|
|
17
|
+
}>['schemas'][number];
|
|
18
|
+
export type SupportedMessageValuesInputForTopic<TopicsConfig extends TopicConfig[], Topic extends SupportedTopics<TopicsConfig>> = z.input<MessageSchemasForTopic<TopicsConfig, Topic>>;
|
|
19
|
+
export type SupportedMessageValuesForTopic<TopicsConfig extends TopicConfig[], Topic extends SupportedTopics<TopicsConfig>> = z.infer<MessageSchemasForTopic<TopicsConfig, Topic>>;
|
|
20
|
+
type MessageSchemas<TopicsConfig extends TopicConfig[]> = TopicsConfig[number]['schemas'][number];
|
|
21
|
+
export type SupportedMessageValuesInput<TopicsConfig extends TopicConfig[]> = z.input<MessageSchemas<TopicsConfig>>;
|
|
22
|
+
export type SupportedMessageValues<TopicsConfig extends TopicConfig[]> = z.input<MessageSchemas<TopicsConfig>>;
|
|
23
|
+
export {};
|
package/dist/types.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../lib/types.ts"],"names":[],"mappings":""}
|
package/package.json
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@message-queue-toolkit/kafka",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"engines": {
|
|
5
|
+
"node": ">= 22.14.0"
|
|
6
|
+
},
|
|
7
|
+
"private": false,
|
|
8
|
+
"license": "MIT",
|
|
9
|
+
"homepage": "https://github.com/kibertoad/message-queue-toolkit",
|
|
10
|
+
"repository": {
|
|
11
|
+
"type": "git",
|
|
12
|
+
"url": "git://github.com/kibertoad/message-queue-toolkit.git"
|
|
13
|
+
},
|
|
14
|
+
"description": "Kafka adapter for message-queue-toolkit",
|
|
15
|
+
"keywords": [
|
|
16
|
+
"message",
|
|
17
|
+
"queue",
|
|
18
|
+
"queues",
|
|
19
|
+
"abstract",
|
|
20
|
+
"common",
|
|
21
|
+
"utils",
|
|
22
|
+
"notification",
|
|
23
|
+
"kafka"
|
|
24
|
+
],
|
|
25
|
+
"files": ["README.md", "LICENSE", "dist"],
|
|
26
|
+
"maintainers": [
|
|
27
|
+
{
|
|
28
|
+
"name": "Igor Savin",
|
|
29
|
+
"email": "kibertoad@gmail.com"
|
|
30
|
+
}
|
|
31
|
+
],
|
|
32
|
+
"type": "module",
|
|
33
|
+
"main": "./dist/index.js",
|
|
34
|
+
"exports": {
|
|
35
|
+
".": "./dist/index.js",
|
|
36
|
+
"./package.json": "./package.json"
|
|
37
|
+
},
|
|
38
|
+
"scripts": {
|
|
39
|
+
"build": "npm run clean && tsc --project tsconfig.build.json",
|
|
40
|
+
"clean": "rimraf dist",
|
|
41
|
+
"test": "vitest run --typecheck",
|
|
42
|
+
"test:coverage": "npm run test -- --coverage",
|
|
43
|
+
"lint": "biome check . && tsc",
|
|
44
|
+
"lint:fix": "biome check --write .",
|
|
45
|
+
"docker:start": "docker compose up -d kafka",
|
|
46
|
+
"docker:stop": "docker compose down",
|
|
47
|
+
"prepublishOnly": "npm run lint && npm run build"
|
|
48
|
+
},
|
|
49
|
+
"dependencies": {
|
|
50
|
+
"@confluentinc/kafka-javascript": "^1.3.0",
|
|
51
|
+
"@lokalise/node-core": "^14.1.0",
|
|
52
|
+
"@lokalise/universal-ts-utils": "^4.4.1",
|
|
53
|
+
"@platformatic/kafka": "^1.3.0",
|
|
54
|
+
"zod": "^3.25.7"
|
|
55
|
+
},
|
|
56
|
+
"peerDependencies": {
|
|
57
|
+
"@message-queue-toolkit/core": ">=21.3.0",
|
|
58
|
+
"@message-queue-toolkit/schemas": ">=6.0.0"
|
|
59
|
+
},
|
|
60
|
+
"devDependencies": {
|
|
61
|
+
"@biomejs/biome": "1.9.4",
|
|
62
|
+
"@lokalise/biome-config": "^2.0.0",
|
|
63
|
+
"@lokalise/tsconfig": "^1.3.0",
|
|
64
|
+
"@message-queue-toolkit/core": "*",
|
|
65
|
+
"@message-queue-toolkit/schemas": "*",
|
|
66
|
+
"@types/node": "^22.7.5",
|
|
67
|
+
"@vitest/coverage-v8": "^3.0.7",
|
|
68
|
+
"awilix": "^12.0.1",
|
|
69
|
+
"awilix-manager": "^6.0.0",
|
|
70
|
+
"rimraf": "^6.0.1",
|
|
71
|
+
"typescript": "^5.7.2",
|
|
72
|
+
"vitest": "^3.0.7"
|
|
73
|
+
}
|
|
74
|
+
}
|