@loipv/nestjs-kafka 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -0
- package/dist/interfaces/consumer-options.interface.d.ts +1 -0
- package/dist/interfaces/message.interface.d.ts +5 -2
- package/dist/interfaces/message.interface.js +35 -0
- package/dist/interfaces/message.interface.js.map +1 -1
- package/dist/kafka.module.js +16 -3
- package/dist/kafka.module.js.map +1 -1
- package/dist/services/batch-processor.service.d.ts +3 -3
- package/dist/services/batch-processor.service.js +5 -4
- package/dist/services/batch-processor.service.js.map +1 -1
- package/dist/services/consumer-registry.service.js +11 -4
- package/dist/services/consumer-registry.service.js.map +1 -1
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/package.json +2 -3
package/README.md
CHANGED
|
@@ -228,6 +228,31 @@ async handleOrder(message: KafkaMessage) {
|
|
|
228
228
|
|
|
229
229
|
Use this to temporarily disable a consumer without removing the code.
|
|
230
230
|
|
|
231
|
+
### Auto-Deserialization
|
|
232
|
+
|
|
233
|
+
Messages are automatically deserialized by default:
|
|
234
|
+
|
|
235
|
+
- **JSON**: Parsed automatically if valid JSON
|
|
236
|
+
- **String**: Falls back to UTF-8 string
|
|
237
|
+
- **Key**: Buffer converted to string
|
|
238
|
+
|
|
239
|
+
```typescript
|
|
240
|
+
// With auto-deserialization (default)
|
|
241
|
+
@Consumer('orders')
|
|
242
|
+
async handleOrder(message: KafkaMessagePayload<Order>) {
|
|
243
|
+
// message.value is already parsed as Order object
|
|
244
|
+
// message.key is string (not Buffer)
|
|
245
|
+
console.log(message.value.orderId);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// Disable auto-deserialization for raw Buffer access
|
|
249
|
+
@Consumer('binary-data', { deserialize: false })
|
|
250
|
+
async handleBinary(message: KafkaMessage) {
|
|
251
|
+
// message.value is Buffer
|
|
252
|
+
const raw = message.value.toString('hex');
|
|
253
|
+
}
|
|
254
|
+
```
|
|
255
|
+
|
|
231
256
|
### All Consumer Options
|
|
232
257
|
|
|
233
258
|
```typescript
|
|
@@ -235,6 +260,9 @@ interface ConsumerOptions {
|
|
|
235
260
|
// Enable/disable consumer
|
|
236
261
|
disabled?: boolean; // Default: false (skip registration when true)
|
|
237
262
|
|
|
263
|
+
// Message deserialization
|
|
264
|
+
deserialize?: boolean; // Default: true (auto JSON parse/string)
|
|
265
|
+
|
|
238
266
|
// Consumer group settings
|
|
239
267
|
groupId?: string;
|
|
240
268
|
sessionTimeout?: number; // Default: 30000
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { IHeaders } from 'kafkajs';
|
|
1
|
+
import { IHeaders, KafkaMessage } from 'kafkajs';
|
|
2
2
|
export interface KafkaMessagePayload<T = any> {
|
|
3
|
-
key?: string |
|
|
3
|
+
key?: string | null;
|
|
4
4
|
value: T;
|
|
5
5
|
headers?: IHeaders;
|
|
6
6
|
partition?: number;
|
|
@@ -8,6 +8,9 @@ export interface KafkaMessagePayload<T = any> {
|
|
|
8
8
|
offset?: string;
|
|
9
9
|
topic?: string;
|
|
10
10
|
}
|
|
11
|
+
export declare function deserializeMessageValue(value: Buffer | null): any;
|
|
12
|
+
export declare function deserializeMessageKey(key: Buffer | string | null | undefined): string | null;
|
|
13
|
+
export declare function deserializeMessage<T = any>(message: KafkaMessage, topic?: string, partition?: number): KafkaMessagePayload<T>;
|
|
11
14
|
export interface ProducerMessage {
|
|
12
15
|
key?: string | Buffer | null;
|
|
13
16
|
value: string | Buffer | object | null;
|
|
@@ -1,3 +1,38 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.deserializeMessageValue = deserializeMessageValue;
|
|
4
|
+
exports.deserializeMessageKey = deserializeMessageKey;
|
|
5
|
+
exports.deserializeMessage = deserializeMessage;
|
|
6
|
+
function deserializeMessageValue(value) {
|
|
7
|
+
if (value === null) {
|
|
8
|
+
return null;
|
|
9
|
+
}
|
|
10
|
+
const stringValue = value.toString('utf-8');
|
|
11
|
+
try {
|
|
12
|
+
return JSON.parse(stringValue);
|
|
13
|
+
}
|
|
14
|
+
catch {
|
|
15
|
+
return stringValue;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
function deserializeMessageKey(key) {
|
|
19
|
+
if (key === null || key === undefined) {
|
|
20
|
+
return null;
|
|
21
|
+
}
|
|
22
|
+
if (Buffer.isBuffer(key)) {
|
|
23
|
+
return key.toString('utf-8');
|
|
24
|
+
}
|
|
25
|
+
return key;
|
|
26
|
+
}
|
|
27
|
+
function deserializeMessage(message, topic, partition) {
|
|
28
|
+
return {
|
|
29
|
+
key: deserializeMessageKey(message.key),
|
|
30
|
+
value: deserializeMessageValue(message.value),
|
|
31
|
+
headers: message.headers,
|
|
32
|
+
timestamp: message.timestamp,
|
|
33
|
+
offset: message.offset,
|
|
34
|
+
topic,
|
|
35
|
+
partition,
|
|
36
|
+
};
|
|
37
|
+
}
|
|
3
38
|
//# sourceMappingURL=message.interface.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"message.interface.js","sourceRoot":"","sources":["../../lib/interfaces/message.interface.ts"],"names":[],"mappings":""}
|
|
1
|
+
{"version":3,"file":"message.interface.js","sourceRoot":"","sources":["../../lib/interfaces/message.interface.ts"],"names":[],"mappings":";;AAkBA,0DAYC;AAOD,sDAUC;AAKD,gDAeC;AAjDD,SAAgB,uBAAuB,CAAC,KAAoB;IAC1D,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,WAAW,GAAG,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAE5C,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IACjC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,WAAW,CAAC;IACrB,CAAC;AACH,CAAC;AAOD,SAAgB,qBAAqB,CACnC,GAAuC;IAEvC,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,OAAO,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAKD,SAAgB,kBAAkB,CAChC,OAAqB,EACrB,KAAc,EACd,SAAkB;IAElB,OAAO;QACL,GAAG,EAAE,qBAAqB,CAAC,OAAO,CAAC,GAAG,CAAC;QAEvC,KAAK,EAAE,uBAAuB,CAAC,OAAO,CAAC,KAAK,CAAC;QAC7C,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,KAAK;QACL,SAAS;KACV,CAAC;AACJ,CAAC"}
|
package/dist/kafka.module.js
CHANGED
|
@@ -20,6 +20,7 @@ const idempotency_service_1 = require("./services/idempotency.service");
|
|
|
20
20
|
const pressure_manager_service_1 = require("./services/pressure-manager.service");
|
|
21
21
|
const dlq_service_1 = require("./services/dlq.service");
|
|
22
22
|
const kafka_health_indicator_1 = require("./health/kafka-health-indicator");
|
|
23
|
+
const terminus_1 = require("@nestjs/terminus");
|
|
23
24
|
let KafkaModule = KafkaModule_1 = class KafkaModule {
|
|
24
25
|
static forRoot(options) {
|
|
25
26
|
const optionsProvider = {
|
|
@@ -28,7 +29,7 @@ let KafkaModule = KafkaModule_1 = class KafkaModule {
|
|
|
28
29
|
};
|
|
29
30
|
return {
|
|
30
31
|
module: KafkaModule_1,
|
|
31
|
-
imports: [core_1.DiscoveryModule],
|
|
32
|
+
imports: [core_1.DiscoveryModule, terminus_1.TerminusModule],
|
|
32
33
|
providers: [
|
|
33
34
|
optionsProvider,
|
|
34
35
|
kafka_core_service_1.KafkaCoreService,
|
|
@@ -41,7 +42,13 @@ let KafkaModule = KafkaModule_1 = class KafkaModule {
|
|
|
41
42
|
dlq_service_1.DlqService,
|
|
42
43
|
kafka_health_indicator_1.KafkaHealthIndicator,
|
|
43
44
|
],
|
|
44
|
-
exports: [
|
|
45
|
+
exports: [
|
|
46
|
+
kafka_client_service_1.KafkaClient,
|
|
47
|
+
kafka_health_indicator_1.KafkaHealthIndicator,
|
|
48
|
+
consumer_discovery_service_1.ConsumerDiscoveryService,
|
|
49
|
+
consumer_registry_service_1.ConsumerRegistryService,
|
|
50
|
+
interfaces_1.KAFKA_MODULE_OPTIONS,
|
|
51
|
+
],
|
|
45
52
|
};
|
|
46
53
|
}
|
|
47
54
|
static forRootAsync(options) {
|
|
@@ -61,7 +68,13 @@ let KafkaModule = KafkaModule_1 = class KafkaModule {
|
|
|
61
68
|
dlq_service_1.DlqService,
|
|
62
69
|
kafka_health_indicator_1.KafkaHealthIndicator,
|
|
63
70
|
],
|
|
64
|
-
exports: [
|
|
71
|
+
exports: [
|
|
72
|
+
kafka_client_service_1.KafkaClient,
|
|
73
|
+
kafka_health_indicator_1.KafkaHealthIndicator,
|
|
74
|
+
consumer_discovery_service_1.ConsumerDiscoveryService,
|
|
75
|
+
consumer_registry_service_1.ConsumerRegistryService,
|
|
76
|
+
interfaces_1.KAFKA_MODULE_OPTIONS,
|
|
77
|
+
],
|
|
65
78
|
global: options.global ?? true,
|
|
66
79
|
};
|
|
67
80
|
}
|
package/dist/kafka.module.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"kafka.module.js","sourceRoot":"","sources":["../lib/kafka.module.ts"],"names":[],"mappings":";;;;;;;;;;AAAA,2CAAyE;AACzE,uCAA+C;AAC/C,6CAKsB;AACtB,sEAAiE;AACjE,0EAA8D;AAC9D,uFAAkF;AAClF,oFAA+E;AAC/E,gFAA2E;AAC3E,wEAAoE;AACpE,kFAA6E;AAC7E,wDAAoD;AACpD,4EAAuE;
|
|
1
|
+
{"version":3,"file":"kafka.module.js","sourceRoot":"","sources":["../lib/kafka.module.ts"],"names":[],"mappings":";;;;;;;;;;AAAA,2CAAyE;AACzE,uCAA+C;AAC/C,6CAKsB;AACtB,sEAAiE;AACjE,0EAA8D;AAC9D,uFAAkF;AAClF,oFAA+E;AAC/E,gFAA2E;AAC3E,wEAAoE;AACpE,kFAA6E;AAC7E,wDAAoD;AACpD,4EAAuE;AACvE,+CAAkD;AAI3C,IAAM,WAAW,mBAAjB,MAAM,WAAW;IACtB,MAAM,CAAC,OAAO,CAAC,OAA2B;QACxC,MAAM,eAAe,GAAa;YAChC,OAAO,EAAE,iCAAoB;YAC7B,QAAQ,EAAE,OAAO;SAClB,CAAC;QAEF,OAAO;YACL,MAAM,EAAE,aAAW;YACnB,OAAO,EAAE,CAAC,sBAAe,EAAE,yBAAc,CAAC;YAC1C,SAAS,EAAE;gBACT,eAAe;gBACf,qCAAgB;gBAChB,kCAAW;gBACX,qDAAwB;gBACxB,mDAAuB;gBACvB,+CAAqB;gBACrB,wCAAkB;gBAClB,iDAAsB;gBACtB,wBAAU;gBACV,6CAAoB;aACrB;YACD,OAAO,EAAE;gBACP,kCAAW;gBACX,6CAAoB;gBACpB,qDAAwB;gBACxB,mDAAuB;gBACvB,iCAAoB;aACrB;SACF,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,YAAY,CAAC,OAAgC;QAClD,MAAM,cAAc,GAAG,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC;QAE1D,OAAO;YACL,MAAM,EAAE,aAAW;YACnB,OAAO,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,sBAAe,CAAC;YACtD,SAAS,EAAE;gBACT,GAAG,cAAc;gBACjB,qCAAgB;gBAChB,kCAAW;gBACX,qDAAwB;gBACxB,mDAAuB;gBACvB,+CAAqB;gBACrB,wCAAkB;gBAClB,iDAAsB;gBACtB,wBAAU;gBACV,6CAAoB;aACrB;YACD,OAAO,EAAE;gBACP,kCAAW;gBACX,6CAAoB;gBACpB,qDAAwB;gBACxB,mDAAuB;gBACvB,iCAAoB;aACrB;YACD,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI;SAC/B,CAAC;IACJ,CAAC;IAEO,MAAM,CAAC,oBAAoB,CACjC,OAAgC;QAEhC,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACvB,OAAO;gBACL;oBACE,OAAO,EAAE,iCAAoB;oBAC7B,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,EAAE;iBAC7B;aACF,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,WAAW,CAAC;QACzD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACrD,CAAC;QAED,OAAO;YACL;gBACE,OAAO,EAAE,iCAAoB;gBAC7B,UAAU,EAAE,KAAK,EAAE,cAAmC,EAAE,EAAE,CACxD,MAAM,cAAc,CAAC,kBAAkB,EAAE;gBAC3C,MAAM,EAAE,CAAC,QAAQ,CAAC;aACnB;YACD,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SAC/D,CAAC;IACJ,CAAC;CACF,CAAA;AAzFY,kCAAW;sBAAX,WAAW;IAFvB,IAAA,eAAM,GAAE;IACR,IAAA,eAAM,EAAC,EAAE,CAAC;GACE,WAAW,CAyFvB"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { KafkaMessage, EachBatchPayload } from 'kafkajs';
|
|
2
|
-
import { ConsumerOptions, GroupedBatch } from '../interfaces';
|
|
2
|
+
import { ConsumerOptions, GroupedBatch, KafkaMessagePayload } from '../interfaces';
|
|
3
3
|
interface BatchAccumulator {
|
|
4
4
|
add: (message: KafkaMessage) => Promise<void>;
|
|
5
5
|
onFlush: (callback: (messages: KafkaMessage[]) => Promise<void>) => void;
|
|
@@ -9,8 +9,8 @@ interface BatchAccumulator {
|
|
|
9
9
|
export declare class BatchProcessorService {
|
|
10
10
|
private readonly logger;
|
|
11
11
|
createBatchAccumulator(options: ConsumerOptions): BatchAccumulator;
|
|
12
|
-
groupMessagesByKey<T>(messages: KafkaMessage
|
|
12
|
+
groupMessagesByKey<T>(messages: Array<KafkaMessage | KafkaMessagePayload<T>>): GroupedBatch<T>[];
|
|
13
13
|
processBatch<T>(messages: KafkaMessage[], options: ConsumerOptions, handler: (messages: KafkaMessage[] | GroupedBatch<T>[]) => Promise<void>): Promise<void>;
|
|
14
|
-
createEachBatchHandler(options: ConsumerOptions, handler: (messages: KafkaMessage[]) => Promise<void>): (payload: EachBatchPayload) => Promise<void>;
|
|
14
|
+
createEachBatchHandler(options: ConsumerOptions, handler: (messages: KafkaMessage[], topic: string, partition: number) => Promise<void>): (payload: EachBatchPayload) => Promise<void>;
|
|
15
15
|
}
|
|
16
16
|
export {};
|
|
@@ -78,11 +78,12 @@ let BatchProcessorService = BatchProcessorService_1 = class BatchProcessorServic
|
|
|
78
78
|
}
|
|
79
79
|
}
|
|
80
80
|
createEachBatchHandler(options, handler) {
|
|
81
|
-
const accumulator = this.createBatchAccumulator(options);
|
|
82
|
-
accumulator.onFlush(async (messages) => {
|
|
83
|
-
await this.processBatch(messages, options, handler);
|
|
84
|
-
});
|
|
85
81
|
const batchHandler = async (payload) => {
|
|
82
|
+
const { topic, partition } = payload.batch;
|
|
83
|
+
const accumulator = this.createBatchAccumulator(options);
|
|
84
|
+
accumulator.onFlush(async (messages) => {
|
|
85
|
+
await handler(messages, topic, partition);
|
|
86
|
+
});
|
|
86
87
|
for (const message of payload.batch.messages) {
|
|
87
88
|
if (!payload.isRunning() || payload.isStale())
|
|
88
89
|
break;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batch-processor.service.js","sourceRoot":"","sources":["../../lib/services/batch-processor.service.ts"],"names":[],"mappings":";;;;;;;;;;AAAA,2CAAoD;AAgB7C,IAAM,qBAAqB,6BAA3B,MAAM,qBAAqB;IACf,MAAM,GAAG,IAAI,eAAM,CAAC,uBAAqB,CAAC,IAAI,CAAC,CAAC;IAEjE,sBAAsB,CAAC,OAAwB;QAC7C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC;QAC3C,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;QAElD,IAAI,MAAM,GAAmB,EAAE,CAAC;QAChC,IAAI,KAAK,GAA0B,IAAI,CAAC;QACxC,IAAI,aAAa,GACf,IAAI,CAAC;QAEP,MAAM,KAAK,GAAG,KAAK,IAAI,EAAE;YACvB,IAAI,KAAK,EAAE,CAAC;gBACV,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,KAAK,GAAG,IAAI,CAAC;YACf,CAAC;YAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAEhC,MAAM,QAAQ,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;YAC7B,MAAM,GAAG,EAAE,CAAC;YAEZ,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC;YAChC,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,aAAa,GAAG,GAAG,EAAE;YACzB,IAAI,KAAK;gBAAE,OAAO;YAClB,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBACtB,KAAK,KAAK,EAAE,CAAC;YACf,CAAC,EAAE,YAAY,CAAC,CAAC;QACnB,CAAC,CAAC;QAEF,OAAO;YACL,GAAG,EAAE,KAAK,EAAE,OAAqB,EAAE,EAAE;gBACnC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAErB,IAAI,MAAM,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;oBAC/B,MAAM,KAAK,EAAE,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACN,aAAa,EAAE,CAAC;gBAClB,CAAC;YACH,CAAC;YAED,OAAO,EAAE,CAAC,QAAqD,EAAE,EAAE;gBACjE,aAAa,GAAG,QAAQ,CAAC;YAC3B,CAAC;YAED,KAAK;YAEL,IAAI,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM;SAC1B,CAAC;IACJ,CAAC;IAED,kBAAkB,
|
|
1
|
+
{"version":3,"file":"batch-processor.service.js","sourceRoot":"","sources":["../../lib/services/batch-processor.service.ts"],"names":[],"mappings":";;;;;;;;;;AAAA,2CAAoD;AAgB7C,IAAM,qBAAqB,6BAA3B,MAAM,qBAAqB;IACf,MAAM,GAAG,IAAI,eAAM,CAAC,uBAAqB,CAAC,IAAI,CAAC,CAAC;IAEjE,sBAAsB,CAAC,OAAwB;QAC7C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC;QAC3C,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;QAElD,IAAI,MAAM,GAAmB,EAAE,CAAC;QAChC,IAAI,KAAK,GAA0B,IAAI,CAAC;QACxC,IAAI,aAAa,GACf,IAAI,CAAC;QAEP,MAAM,KAAK,GAAG,KAAK,IAAI,EAAE;YACvB,IAAI,KAAK,EAAE,CAAC;gBACV,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,KAAK,GAAG,IAAI,CAAC;YACf,CAAC;YAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAEhC,MAAM,QAAQ,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;YAC7B,MAAM,GAAG,EAAE,CAAC;YAEZ,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC;YAChC,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,aAAa,GAAG,GAAG,EAAE;YACzB,IAAI,KAAK;gBAAE,OAAO;YAClB,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBACtB,KAAK,KAAK,EAAE,CAAC;YACf,CAAC,EAAE,YAAY,CAAC,CAAC;QACnB,CAAC,CAAC;QAEF,OAAO;YACL,GAAG,EAAE,KAAK,EAAE,OAAqB,EAAE,EAAE;gBACnC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAErB,IAAI,MAAM,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;oBAC/B,MAAM,KAAK,EAAE,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACN,aAAa,EAAE,CAAC;gBAClB,CAAC;YACH,CAAC;YAED,OAAO,EAAE,CAAC,QAAqD,EAAE,EAAE;gBACjE,aAAa,GAAG,QAAQ,CAAC;YAC3B,CAAC;YAED,KAAK;YAEL,IAAI,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM;SAC1B,CAAC;IACJ,CAAC;IAED,kBAAkB,CAChB,QAAsD;QAEtD,MAAM,OAAO,GAAG,IAAI,GAAG,EAGpB,CAAC;QAEJ,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,QAAQ,EAAE,IAAI,cAAc,CAAC;YAEtD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBACtB,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACvB,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAClC,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;YACzD,GAAG;YACH,QAAQ,EAAE,IAAgC;SAC3C,CAAC,CAAC,CAAC;IACN,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,QAAwB,EACxB,OAAwB,EACxB,OAAwE;QAExE,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACvB,MAAM,OAAO,GAAG,IAAI,CAAC,kBAAkB,CAAI,QAAQ,CAAC,CAAC;YACrD,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,sBAAsB,CACpB,OAAwB,EACxB,OAIkB;QAElB,MAAM,YAAY,GAAG,KAAK,EAAE,OAAyB,EAAiB,EAAE;YACtE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC;YAC3C,MAAM,WAAW,GAAG,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC;YAEzD,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;gBACrC,MAAM,OAAO,CAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;YAC5C,CAAC,CAAC,CAAC;YAEH,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;gBAC7C,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,OAAO,CAAC,OAAO,EAAE;oBAAE,MAAM;gBAErD,MAAM,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBAC/B,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBACtC,MAAM,OAAO,CAAC,SAAS,EAAE,CAAC;YAC5B,CAAC;YAED,MAAM,WAAW,CAAC,KAAK,EAAE,CAAC;QAC5B,CAAC,CAAC;QAEF,OAAO,YAAY,CAAC;IACtB,CAAC;CACF,CAAA;AAzHY,sDAAqB;gCAArB,qBAAqB;IADjC,IAAA,mBAAU,GAAE;GACA,qBAAqB,CAyHjC"}
|
|
@@ -17,6 +17,7 @@ const batch_processor_service_1 = require("./batch-processor.service");
|
|
|
17
17
|
const idempotency_service_1 = require("./idempotency.service");
|
|
18
18
|
const pressure_manager_service_1 = require("./pressure-manager.service");
|
|
19
19
|
const dlq_service_1 = require("./dlq.service");
|
|
20
|
+
const interfaces_1 = require("../interfaces");
|
|
20
21
|
let ConsumerRegistryService = ConsumerRegistryService_1 = class ConsumerRegistryService {
|
|
21
22
|
kafkaCore;
|
|
22
23
|
batchProcessor;
|
|
@@ -107,7 +108,10 @@ let ConsumerRegistryService = ConsumerRegistryService_1 = class ConsumerRegistry
|
|
|
107
108
|
}
|
|
108
109
|
}
|
|
109
110
|
try {
|
|
110
|
-
|
|
111
|
+
const processedMessage = options.deserialize !== false
|
|
112
|
+
? (0, interfaces_1.deserializeMessage)(message, topic, partition)
|
|
113
|
+
: message;
|
|
114
|
+
await handler(processedMessage);
|
|
111
115
|
if (options.idempotencyKey) {
|
|
112
116
|
this.idempotencyService.markProcessed(message, options.idempotencyKey);
|
|
113
117
|
}
|
|
@@ -123,19 +127,22 @@ let ConsumerRegistryService = ConsumerRegistryService_1 = class ConsumerRegistry
|
|
|
123
127
|
const { metadata, consumer } = registered;
|
|
124
128
|
const { options, target, methodName } = metadata;
|
|
125
129
|
const handler = target[methodName].bind(target);
|
|
126
|
-
const eachBatchHandler = this.batchProcessor.createEachBatchHandler(options, async (messages) => {
|
|
130
|
+
const eachBatchHandler = this.batchProcessor.createEachBatchHandler(options, async (messages, topic, partition) => {
|
|
127
131
|
if (this.isShuttingDown)
|
|
128
132
|
return;
|
|
129
133
|
let processableMessages = messages;
|
|
130
134
|
if (options.idempotencyKey) {
|
|
131
135
|
processableMessages = this.idempotencyService.filterDuplicates(messages, options.idempotencyKey);
|
|
132
136
|
}
|
|
137
|
+
const deserializedMessages = options.deserialize !== false
|
|
138
|
+
? processableMessages.map((msg) => (0, interfaces_1.deserializeMessage)(msg, topic, partition))
|
|
139
|
+
: processableMessages;
|
|
133
140
|
if (options.groupByKey) {
|
|
134
|
-
const grouped = this.batchProcessor.groupMessagesByKey(
|
|
141
|
+
const grouped = this.batchProcessor.groupMessagesByKey(deserializedMessages);
|
|
135
142
|
await handler(grouped);
|
|
136
143
|
}
|
|
137
144
|
else {
|
|
138
|
-
await handler(
|
|
145
|
+
await handler(deserializedMessages);
|
|
139
146
|
}
|
|
140
147
|
if (options.idempotencyKey) {
|
|
141
148
|
for (const msg of processableMessages) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"consumer-registry.service.js","sourceRoot":"","sources":["../../lib/services/consumer-registry.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"consumer-registry.service.js","sourceRoot":"","sources":["../../lib/services/consumer-registry.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAIA,2CAA2E;AAE3E,6DAAwD;AACxD,uEAAkE;AAClE,+DAA2D;AAC3D,yEAAoE;AACpE,+CAA2C;AAC3C,8CAAqE;AAS9D,IAAM,uBAAuB,+BAA7B,MAAM,uBAAuB;IAMf;IACA;IACA;IACA;IACA;IATF,MAAM,GAAG,IAAI,eAAM,CAAC,yBAAuB,CAAC,IAAI,CAAC,CAAC;IAC3D,mBAAmB,GAAG,IAAI,GAAG,EAA8B,CAAC;IAC5D,cAAc,GAAG,KAAK,CAAC;IAE/B,YACmB,SAA2B,EAC3B,cAAqC,EACrC,kBAAsC,EACtC,eAAuC,EACvC,UAAsB;QAJtB,cAAS,GAAT,SAAS,CAAkB;QAC3B,mBAAc,GAAd,cAAc,CAAuB;QACrC,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,oBAAe,GAAf,eAAe,CAAwB;QACvC,eAAU,GAAV,UAAU,CAAY;IACtC,CAAC;IAEJ,iBAAiB,CAAC,SAA6B;QAC7C,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAEO,gBAAgB,CAAC,QAA0B;QACjD,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC;QACpC,MAAM,UAAU,GAAG,GAAG,KAAK,IAAI,OAAO,CAAC,OAAO,IAAI,SAAS,EAAE,CAAC;QAE9D,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC;YAClD,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,GAAG,KAAK,QAAQ;YAC5C,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;YAC5C,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;YAC1C,oBAAoB,EAAE,OAAO;SAC9B,CAAC,CAAC;QAEH,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,UAAU,EAAE,QAAQ,EAAE;YAClD,qBAAqB,EAAE,OAAO,CAAC,qBAAqB,IAAI,EAAE;YAC1D,eAAe,EAAE,EAAE;YACnB,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI;YAC1C,eAAe,EAAE,IAAI;SACtB,CAAC,CAAC;QAEH,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,UAAU,EAAE;YACvC,QAAQ;YACR,QAAQ;YACR,SAAS,EAAE,KAAK;SACjB,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,wBAAwB,UAAU,EAAE,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CACrE,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAC/C,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IACnC,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,UAA8B;QACxD,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,UAAU,CAAC;QAC1C,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC;QAEpC,IAAI,CAAC;YACH,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;YAEzB,MAAM,QAAQ,CAAC,SAAS,CAAC;gBACvB,KAAK;gBACL,aAAa,EAAE,OAAO,CAAC,aAAa;aACrC,CAAC,CAAC;YAEH,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBAClB,MAAM,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;YAC5C,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,oBAAoB,CAAC,UAAU,CAAC,CAAC;YAC9C,CAAC;YAED,UAAU,CAAC,SAAS,GAAG,IAAI,CAAC;YAC5B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,+BAA+B,KAAK,EAAE,CAAC,CAAC;QAC1D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,uCAAuC,KAAK,EAAE,EAAE,KAAK,CAAC,CAAC;YACzE,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAChC,UAA8B;QAE9B,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,UAAU,CAAC;QAE1C,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,QAAQ,CAAC;QAExD,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhD,MAAM,QAAQ,CAAC,GAAG,CAAC;YACjB,UAAU,EAAE,OAAO,CAAC,UAAU,KAAK,KAAK;YACxC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;YAC9C,mBAAmB,EAAE,OAAO,CAAC,mBAAmB;YAChD,WAAW,EAAE,KAAK,EAAE,OAA2B,EAAE,EAAE;gBACjD,IAAI,IAAI,CAAC,cAAc;oBAAE,OAAO;gBAEhC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;gBAEvC,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;oBAC3B,IACE,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,cAAc,CAAC,EACpE,CAAC;wBACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;wBAChD,OAAO;oBACT,CAAC;gBACH,CAAC;gBAED,IAAI,CAAC;oBAEH,MAAM,gBAAgB,GACpB,OAAO,CAAC,WAAW,KAAK,KAAK;wBAC3B,CAAC,CAAC,IAAA,+BAAkB,EAAC,OAAO,EAAE,KAAK,EAAE,SAAS,CAAC;wBAC/C,CAAC,CAAC,OAAO,CAAC;oBAEd,MAAM,OAAO,CAAC,gBAAgB,CAAC,CAAC;oBAEhC,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;wBAC3B,IAAI,CAAC,kBAAkB,CAAC,aAAa,CACnC,OAAO,EACP,OAAO,CAAC,cAAc,CACvB,CAAC;oBACJ,CAAC;oBAED,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;gBAC7D,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,KAAc,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;gBACvE,CAAC;YACH,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,UAA8B;QAE9B,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,UAAU,CAAC;QAE1C,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,QAAQ,CAAC;QAEjD,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhD,MAAM,gBAAgB,GAAG,IAAI,CAAC,cAAc,CAAC,sBAAsB,CACjE,OAAO,EACP,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE;YACnC,IAAI,IAAI,CAAC,cAAc;gBAAE,OAAO;YAEhC,IAAI,mBAAmB,GAAG,QAAQ,CAAC;YACnC,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;gBAC3B,mBAAmB,GAAG,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAC5D,QAAQ,EACR,OAAO,CAAC,cAAc,CACvB,CAAC;YACJ,CAAC;YAGD,MAAM,oBAAoB,GACxB,OAAO,CAAC,WAAW,KAAK,KAAK;gBAC3B,CAAC,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAC9B,IAAA,+BAAkB,EAAC,GAAG,EAAE,KAAK,EAAE,SAAS,CAAC,CAC1C;gBACH,CAAC,CAAC,mBAAmB,CAAC;YAE1B,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACvB,MAAM,OAAO,GACX,IAAI,CAAC,cAAc,CAAC,kBAAkB,CAAC,oBAAoB,CAAC,CAAC;gBAE/D,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,MAAM,OAAO,CAAC,oBAAoB,CAAC,CAAC;YACtC,CAAC;YAED,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;gBAC3B,KAAK,MAAM,GAAG,IAAI,mBAAmB,EAAE,CAAC;oBACtC,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC;gBACrE,CAAC;YACH,CAAC;QACH,CAAC,CACF,CAAC;QAEF,MAAM,QAAQ,CAAC,GAAG,CAAC;YACjB,UAAU,EAAE,KAAK;YACjB,SAAS,EAAE,gBAAgB;SAC5B,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,OAAY,EACZ,KAAY,EACZ,QAA0B,EAC1B,SAAkB;QAElB,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC;QAEpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iCAAiC,KAAK,EAAE,EAAE,KAAK,CAAC,CAAC;QAEnE,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;YAChB,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CACrD,OAAO,EACP,KAAK,EACL,OAAO,CAAC,GAAG,EACX,KAAK,EACL,SAAS,CACV,CAAC;YAEF,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,gBAAgB;QACpB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;QAE9D,MAAM,gBAAgB,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CACxE,KAAK,EAAE,UAAU,EAAE,EAAE;YACnB,IAAI,CAAC;gBACH,IAAI,UAAU,CAAC,SAAS,EAAE,CAAC;oBACzB,MAAM,UAAU,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;oBACjC,MAAM,UAAU,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;gBACzC,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC,CACF,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QACpC,IAAI,CAAC,kBAAkB,CAAC,WAAW,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAChC,CAAC;CACF,CAAA;AA7OY,0DAAuB;kCAAvB,uBAAuB;IADnC,IAAA,mBAAU,GAAE;qCAOmB,qCAAgB;QACX,+CAAqB;QACjB,wCAAkB;QACrB,iDAAsB;QAC3B,wBAAU;GAV9B,uBAAuB,CA6OnC"}
|