@loipv/nestjs-kafka 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +377 -0
- package/dist/consumer.module.d.ts +10 -0
- package/dist/consumer.module.js +38 -0
- package/dist/consumer.module.js.map +1 -0
- package/dist/decorators/constants.d.ts +1 -0
- package/dist/decorators/constants.js +5 -0
- package/dist/decorators/constants.js.map +1 -0
- package/dist/decorators/consumer.decorator.d.ts +8 -0
- package/dist/decorators/consumer.decorator.js +16 -0
- package/dist/decorators/consumer.decorator.js.map +1 -0
- package/dist/decorators/index.d.ts +2 -0
- package/dist/decorators/index.js +21 -0
- package/dist/decorators/index.js.map +1 -0
- package/dist/discovery/consumer-discovery.service.d.ts +14 -0
- package/dist/discovery/consumer-discovery.service.js +82 -0
- package/dist/discovery/consumer-discovery.service.js.map +1 -0
- package/dist/discovery/index.d.ts +1 -0
- package/dist/discovery/index.js +18 -0
- package/dist/discovery/index.js.map +1 -0
- package/dist/health/index.d.ts +1 -0
- package/dist/health/index.js +18 -0
- package/dist/health/index.js.map +1 -0
- package/dist/health/kafka-health-indicator.d.ts +12 -0
- package/dist/health/kafka-health-indicator.js +98 -0
- package/dist/health/kafka-health-indicator.js.map +1 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.js +29 -0
- package/dist/index.js.map +1 -0
- package/dist/interfaces/consumer-options.interface.d.ts +52 -0
- package/dist/interfaces/consumer-options.interface.js +3 -0
- package/dist/interfaces/consumer-options.interface.js.map +1 -0
- package/dist/interfaces/index.d.ts +3 -0
- package/dist/interfaces/index.js +20 -0
- package/dist/interfaces/index.js.map +1 -0
- package/dist/interfaces/kafka-module-options.interface.d.ts +43 -0
- package/dist/interfaces/kafka-module-options.interface.js +5 -0
- package/dist/interfaces/kafka-module-options.interface.js.map +1 -0
- package/dist/interfaces/message.interface.d.ts +26 -0
- package/dist/interfaces/message.interface.js +3 -0
- package/dist/interfaces/message.interface.js.map +1 -0
- package/dist/kafka.module.d.ts +7 -0
- package/dist/kafka.module.js +97 -0
- package/dist/kafka.module.js.map +1 -0
- package/dist/services/batch-processor.service.d.ts +16 -0
- package/dist/services/batch-processor.service.js +102 -0
- package/dist/services/batch-processor.service.js.map +1 -0
- package/dist/services/consumer-registry.service.d.ts +27 -0
- package/dist/services/consumer-registry.service.js +195 -0
- package/dist/services/consumer-registry.service.js.map +1 -0
- package/dist/services/dlq.service.d.ts +14 -0
- package/dist/services/dlq.service.js +90 -0
- package/dist/services/dlq.service.js.map +1 -0
- package/dist/services/idempotency.service.d.ts +16 -0
- package/dist/services/idempotency.service.js +76 -0
- package/dist/services/idempotency.service.js.map +1 -0
- package/dist/services/index.d.ts +7 -0
- package/dist/services/index.js +24 -0
- package/dist/services/index.js.map +1 -0
- package/dist/services/kafka-client.service.d.ts +31 -0
- package/dist/services/kafka-client.service.js +183 -0
- package/dist/services/kafka-client.service.js.map +1 -0
- package/dist/services/kafka-core.service.d.ts +13 -0
- package/dist/services/kafka-core.service.js +87 -0
- package/dist/services/kafka-core.service.js.map +1 -0
- package/dist/services/pressure-manager.service.d.ts +14 -0
- package/dist/services/pressure-manager.service.js +75 -0
- package/dist/services/pressure-manager.service.js.map +1 -0
- package/dist/tsconfig.build.tsbuildinfo +1 -0
- package/package.json +95 -0
package/README.md
ADDED
|
@@ -0,0 +1,377 @@
|
|
|
1
|
+
# @loipv/nestjs-kafka
|
|
2
|
+
|
|
3
|
+
A production-ready NestJS module for Kafka client and consumer functionality built on top of [KafkaJS](https://kafka.js.org/). This library provides enterprise-grade features including intelligent batch processing, idempotency guarantees, key-based grouping, and automatic pressure management.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- **Producer (KafkaClient)**: High-performance Kafka producer with `send()`, `sendBatch()`, `sendQueued()` methods
|
|
8
|
+
- **Consumer**: Method decorator-based consumer with auto-discovery
|
|
9
|
+
- **Batch Processing**: Intelligent batching with configurable size and timeout
|
|
10
|
+
- **Key-Based Grouping**: Group messages by key within batches for ordered processing
|
|
11
|
+
- **Back Pressure**: Automatic pause/resume when consumers are overwhelmed
|
|
12
|
+
- **Idempotency**: In-memory duplicate prevention with TTL
|
|
13
|
+
- **Dead Letter Queue (DLQ)**: Automatic retry with exponential backoff
|
|
14
|
+
- **Health Checks**: Integration with `@nestjs/terminus`
|
|
15
|
+
- **Graceful Shutdown**: Proper cleanup on application shutdown
|
|
16
|
+
|
|
17
|
+
## Installation
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
npm install @loipv/nestjs-kafka kafkajs
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
### Peer Dependencies
|
|
24
|
+
|
|
25
|
+
Make sure you have the following peer dependencies installed:
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
npm install @nestjs/common @nestjs/core @nestjs/terminus reflect-metadata rxjs
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## Quick Start
|
|
32
|
+
|
|
33
|
+
### 1. Import KafkaModule
|
|
34
|
+
|
|
35
|
+
```typescript
|
|
36
|
+
import { Module } from '@nestjs/common';
|
|
37
|
+
import { KafkaModule, ConsumerModule } from '@loipv/nestjs-kafka';
|
|
38
|
+
import { OrderConsumer } from './order.consumer';
|
|
39
|
+
|
|
40
|
+
@Module({
|
|
41
|
+
imports: [
|
|
42
|
+
KafkaModule.forRoot({
|
|
43
|
+
clientId: 'my-app',
|
|
44
|
+
brokers: ['localhost:9092'],
|
|
45
|
+
}),
|
|
46
|
+
ConsumerModule,
|
|
47
|
+
],
|
|
48
|
+
providers: [OrderConsumer],
|
|
49
|
+
})
|
|
50
|
+
export class AppModule {}
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
### 2. Create a Consumer
|
|
54
|
+
|
|
55
|
+
```typescript
|
|
56
|
+
import { Injectable } from '@nestjs/common';
|
|
57
|
+
import { Consumer } from '@loipv/nestjs-kafka';
|
|
58
|
+
import { KafkaMessage } from 'kafkajs';
|
|
59
|
+
|
|
60
|
+
@Injectable()
|
|
61
|
+
export class OrderConsumer {
|
|
62
|
+
@Consumer('orders')
|
|
63
|
+
async handleOrder(message: KafkaMessage) {
|
|
64
|
+
const order = JSON.parse(message.value.toString());
|
|
65
|
+
console.log('Processing order:', order);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
### 3. Use the Producer
|
|
71
|
+
|
|
72
|
+
```typescript
|
|
73
|
+
import { Injectable } from '@nestjs/common';
|
|
74
|
+
import { KafkaClient } from '@loipv/nestjs-kafka';
|
|
75
|
+
|
|
76
|
+
@Injectable()
|
|
77
|
+
export class OrderService {
|
|
78
|
+
constructor(private readonly kafka: KafkaClient) {}
|
|
79
|
+
|
|
80
|
+
async createOrder(order: Order) {
|
|
81
|
+
await this.kafka.send('orders', {
|
|
82
|
+
key: order.customerId,
|
|
83
|
+
value: order, // Automatically serialized to JSON
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## Configuration
|
|
90
|
+
|
|
91
|
+
### KafkaModule Options
|
|
92
|
+
|
|
93
|
+
```typescript
|
|
94
|
+
KafkaModule.forRoot({
|
|
95
|
+
// Required
|
|
96
|
+
clientId: 'my-app',
|
|
97
|
+
brokers: ['localhost:9092'],
|
|
98
|
+
|
|
99
|
+
// Optional - SSL/SASL
|
|
100
|
+
ssl: true,
|
|
101
|
+
sasl: {
|
|
102
|
+
mechanism: 'scram-sha-256',
|
|
103
|
+
username: process.env.KAFKA_USERNAME,
|
|
104
|
+
password: process.env.KAFKA_PASSWORD,
|
|
105
|
+
},
|
|
106
|
+
|
|
107
|
+
// Optional - Connection settings
|
|
108
|
+
connectionTimeout: 3000,
|
|
109
|
+
requestTimeout: 30000,
|
|
110
|
+
|
|
111
|
+
// Optional - Retry configuration
|
|
112
|
+
retry: {
|
|
113
|
+
initialRetryTime: 100,
|
|
114
|
+
retries: 8,
|
|
115
|
+
maxRetryTime: 30000,
|
|
116
|
+
},
|
|
117
|
+
|
|
118
|
+
// Optional - Logging
|
|
119
|
+
logLevel: 'INFO', // 'NOTHING' | 'ERROR' | 'WARN' | 'INFO' | 'DEBUG'
|
|
120
|
+
});
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
### Async Configuration
|
|
124
|
+
|
|
125
|
+
```typescript
|
|
126
|
+
KafkaModule.forRootAsync({
|
|
127
|
+
imports: [ConfigModule],
|
|
128
|
+
useFactory: (config: ConfigService) => ({
|
|
129
|
+
clientId: config.get('KAFKA_CLIENT_ID'),
|
|
130
|
+
brokers: config.get('KAFKA_BROKERS').split(','),
|
|
131
|
+
}),
|
|
132
|
+
inject: [ConfigService],
|
|
133
|
+
});
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
## Consumer Options
|
|
137
|
+
|
|
138
|
+
### Basic Consumer
|
|
139
|
+
|
|
140
|
+
```typescript
|
|
141
|
+
@Consumer('topic-name')
|
|
142
|
+
async handleMessage(message: KafkaMessage) {
|
|
143
|
+
// Process single message
|
|
144
|
+
}
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
### Batch Consumer
|
|
148
|
+
|
|
149
|
+
```typescript
|
|
150
|
+
@Consumer('orders', {
|
|
151
|
+
batch: true,
|
|
152
|
+
batchSize: 100, // Max messages per batch (default: 100)
|
|
153
|
+
batchTimeout: 5000, // Max wait time in ms (default: 5000)
|
|
154
|
+
})
|
|
155
|
+
async handleBatch(messages: KafkaMessage[]) {
|
|
156
|
+
// Process batch of messages
|
|
157
|
+
}
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
### Batch with Key Grouping
|
|
161
|
+
|
|
162
|
+
```typescript
|
|
163
|
+
@Consumer('orders', {
|
|
164
|
+
batch: true,
|
|
165
|
+
batchSize: 100,
|
|
166
|
+
groupByKey: true, // Group messages by key within batch
|
|
167
|
+
})
|
|
168
|
+
async handleBatch(groupedMessages: GroupedBatch[]) {
|
|
169
|
+
// groupedMessages = [{ key: 'customer-1', messages: [...] }, ...]
|
|
170
|
+
for (const group of groupedMessages) {
|
|
171
|
+
console.log(`Processing ${group.messages.length} orders for ${group.key}`);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
### Consumer with DLQ
|
|
177
|
+
|
|
178
|
+
```typescript
|
|
179
|
+
@Consumer('payments', {
|
|
180
|
+
dlq: {
|
|
181
|
+
topic: 'payments-dlq',
|
|
182
|
+
maxRetries: 3, // Retry 3 times before DLQ
|
|
183
|
+
retryDelay: 1000, // Initial delay: 1 second
|
|
184
|
+
retryBackoffMultiplier: 2, // Exponential backoff
|
|
185
|
+
includeErrorInfo: true, // Include error in DLQ headers
|
|
186
|
+
},
|
|
187
|
+
})
|
|
188
|
+
async handlePayment(message: KafkaMessage) {
|
|
189
|
+
// If this throws, message will be retried then sent to DLQ
|
|
190
|
+
}
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
### Consumer with Idempotency
|
|
194
|
+
|
|
195
|
+
```typescript
|
|
196
|
+
@Consumer('events', {
|
|
197
|
+
idempotencyKey: (msg) => msg.headers?.['event-id']?.toString(),
|
|
198
|
+
idempotencyTtl: 3600000, // 1 hour
|
|
199
|
+
})
|
|
200
|
+
async handleEvent(message: KafkaMessage) {
|
|
201
|
+
// Duplicate messages (same event-id) will be skipped
|
|
202
|
+
}
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
### Consumer with Back Pressure
|
|
206
|
+
|
|
207
|
+
```typescript
|
|
208
|
+
@Consumer('high-volume', {
|
|
209
|
+
batch: true,
|
|
210
|
+
backPressureThreshold: 80, // Pause at 80% capacity
|
|
211
|
+
maxQueueSize: 1000,
|
|
212
|
+
})
|
|
213
|
+
async handleHighVolume(messages: KafkaMessage[]) {
|
|
214
|
+
// Consumer will auto-pause when overwhelmed
|
|
215
|
+
}
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
### Disabled Consumer
|
|
219
|
+
|
|
220
|
+
```typescript
|
|
221
|
+
@Consumer('orders', {
|
|
222
|
+
disabled: true, // Consumer will be skipped during registration
|
|
223
|
+
})
|
|
224
|
+
async handleOrder(message: KafkaMessage) {
|
|
225
|
+
// This handler will not be registered
|
|
226
|
+
}
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
Use this to temporarily disable a consumer without removing the code.
|
|
230
|
+
|
|
231
|
+
### All Consumer Options
|
|
232
|
+
|
|
233
|
+
```typescript
|
|
234
|
+
interface ConsumerOptions {
|
|
235
|
+
// Enable/disable consumer
|
|
236
|
+
disabled?: boolean; // Default: false (skip registration when true)
|
|
237
|
+
|
|
238
|
+
// Consumer group settings
|
|
239
|
+
groupId?: string;
|
|
240
|
+
sessionTimeout?: number; // Default: 30000
|
|
241
|
+
heartbeatInterval?: number; // Default: 3000
|
|
242
|
+
rebalanceTimeout?: number;
|
|
243
|
+
|
|
244
|
+
// Batch processing
|
|
245
|
+
batch?: boolean;
|
|
246
|
+
batchSize?: number; // Default: 100
|
|
247
|
+
batchTimeout?: number; // Default: 5000
|
|
248
|
+
groupByKey?: boolean;
|
|
249
|
+
|
|
250
|
+
// Pressure management
|
|
251
|
+
backPressureThreshold?: number; // Default: 80
|
|
252
|
+
maxQueueSize?: number; // Default: 1000
|
|
253
|
+
|
|
254
|
+
// Idempotency
|
|
255
|
+
idempotencyKey?: (message: KafkaMessage) => string | undefined;
|
|
256
|
+
idempotencyTtl?: number; // Default: 3600000 (1 hour)
|
|
257
|
+
|
|
258
|
+
// Dead Letter Queue
|
|
259
|
+
dlq?: {
|
|
260
|
+
topic: string;
|
|
261
|
+
maxRetries?: number; // Default: 3
|
|
262
|
+
retryDelay?: number; // Default: 1000
|
|
263
|
+
retryBackoffMultiplier?: number; // Default: 2
|
|
264
|
+
};
|
|
265
|
+
|
|
266
|
+
// Commit settings
|
|
267
|
+
autoCommit?: boolean; // Default: true
|
|
268
|
+
autoCommitInterval?: number;
|
|
269
|
+
fromBeginning?: boolean; // Default: false
|
|
270
|
+
}
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
## Producer API
|
|
274
|
+
|
|
275
|
+
### KafkaClient Methods
|
|
276
|
+
|
|
277
|
+
```typescript
|
|
278
|
+
// Send single message
|
|
279
|
+
await kafka.send('topic', {
|
|
280
|
+
key: 'message-key',
|
|
281
|
+
value: { data: 'value' }, // Auto-serialized
|
|
282
|
+
headers: { 'correlation-id': '123' },
|
|
283
|
+
});
|
|
284
|
+
|
|
285
|
+
// Send batch to single topic
|
|
286
|
+
await kafka.sendBatch('topic', [
|
|
287
|
+
{ key: 'key1', value: 'value1' },
|
|
288
|
+
{ key: 'key2', value: 'value2' },
|
|
289
|
+
]);
|
|
290
|
+
|
|
291
|
+
// Send to multiple topics
|
|
292
|
+
await kafka.sendMultiTopicBatch([
|
|
293
|
+
{ topic: 'topic1', messages: [{ value: 'msg1' }] },
|
|
294
|
+
{ topic: 'topic2', messages: [{ value: 'msg2' }] },
|
|
295
|
+
]);
|
|
296
|
+
|
|
297
|
+
// Queue message for auto-batching
|
|
298
|
+
await kafka.sendQueued('topic', { value: 'message' });
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
### Send Options
|
|
302
|
+
|
|
303
|
+
```typescript
|
|
304
|
+
await kafka.send('topic', message, {
|
|
305
|
+
acks: -1, // -1 (all), 0 (none), 1 (leader only)
|
|
306
|
+
timeout: 30000,
|
|
307
|
+
compression: 1, // 0=None, 1=GZIP, 2=Snappy, 3=LZ4, 4=ZSTD
|
|
308
|
+
});
|
|
309
|
+
```
|
|
310
|
+
|
|
311
|
+
## Health Checks
|
|
312
|
+
|
|
313
|
+
Integrate with `@nestjs/terminus`:
|
|
314
|
+
|
|
315
|
+
```typescript
|
|
316
|
+
import { Controller, Get } from '@nestjs/common';
|
|
317
|
+
import { HealthCheck, HealthCheckService } from '@nestjs/terminus';
|
|
318
|
+
import { KafkaHealthIndicator } from '@loipv/nestjs-kafka';
|
|
319
|
+
|
|
320
|
+
@Controller('health')
|
|
321
|
+
export class HealthController {
|
|
322
|
+
constructor(
|
|
323
|
+
private health: HealthCheckService,
|
|
324
|
+
private kafkaHealth: KafkaHealthIndicator,
|
|
325
|
+
) {}
|
|
326
|
+
|
|
327
|
+
@Get()
|
|
328
|
+
@HealthCheck()
|
|
329
|
+
check() {
|
|
330
|
+
return this.health.check([
|
|
331
|
+
() => this.kafkaHealth.isHealthy('kafka'),
|
|
332
|
+
]);
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
@Get('kafka/brokers')
|
|
336
|
+
@HealthCheck()
|
|
337
|
+
checkBrokers() {
|
|
338
|
+
return this.health.check([
|
|
339
|
+
() => this.kafkaHealth.checkBrokers('kafka-brokers'),
|
|
340
|
+
]);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
```
|
|
344
|
+
|
|
345
|
+
## API Reference
|
|
346
|
+
|
|
347
|
+
### Exports
|
|
348
|
+
|
|
349
|
+
```typescript
|
|
350
|
+
// Modules
|
|
351
|
+
export { KafkaModule } from './kafka.module';
|
|
352
|
+
export { ConsumerModule } from './consumer.module';
|
|
353
|
+
|
|
354
|
+
// Decorators
|
|
355
|
+
export { Consumer } from './decorators';
|
|
356
|
+
|
|
357
|
+
// Services
|
|
358
|
+
export { KafkaClient } from './services/kafka-client.service';
|
|
359
|
+
|
|
360
|
+
// Health
|
|
361
|
+
export { KafkaHealthIndicator } from './health/kafka-health-indicator';
|
|
362
|
+
|
|
363
|
+
// Interfaces
|
|
364
|
+
export {
|
|
365
|
+
KafkaModuleOptions,
|
|
366
|
+
KafkaModuleAsyncOptions,
|
|
367
|
+
ConsumerOptions,
|
|
368
|
+
DlqOptions,
|
|
369
|
+
ProducerMessage,
|
|
370
|
+
SendOptions,
|
|
371
|
+
GroupedBatch,
|
|
372
|
+
} from './interfaces';
|
|
373
|
+
```
|
|
374
|
+
|
|
375
|
+
## License
|
|
376
|
+
|
|
377
|
+
MIT
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { OnModuleInit, OnApplicationShutdown } from '@nestjs/common';
|
|
2
|
+
import { ConsumerDiscoveryService } from './discovery/consumer-discovery.service';
|
|
3
|
+
import { ConsumerRegistryService } from './services/consumer-registry.service';
|
|
4
|
+
export declare class ConsumerModule implements OnModuleInit, OnApplicationShutdown {
|
|
5
|
+
private readonly discoveryService;
|
|
6
|
+
private readonly registryService;
|
|
7
|
+
constructor(discoveryService: ConsumerDiscoveryService, registryService: ConsumerRegistryService);
|
|
8
|
+
onModuleInit(): Promise<void>;
|
|
9
|
+
onApplicationShutdown(): Promise<void>;
|
|
10
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.ConsumerModule = void 0;
|
|
13
|
+
const common_1 = require("@nestjs/common");
|
|
14
|
+
const consumer_discovery_service_1 = require("./discovery/consumer-discovery.service");
|
|
15
|
+
const consumer_registry_service_1 = require("./services/consumer-registry.service");
|
|
16
|
+
let ConsumerModule = class ConsumerModule {
|
|
17
|
+
discoveryService;
|
|
18
|
+
registryService;
|
|
19
|
+
constructor(discoveryService, registryService) {
|
|
20
|
+
this.discoveryService = discoveryService;
|
|
21
|
+
this.registryService = registryService;
|
|
22
|
+
}
|
|
23
|
+
async onModuleInit() {
|
|
24
|
+
const consumers = this.discoveryService.discoverConsumers();
|
|
25
|
+
this.registryService.registerConsumers(consumers);
|
|
26
|
+
await this.registryService.startAll();
|
|
27
|
+
}
|
|
28
|
+
async onApplicationShutdown() {
|
|
29
|
+
await this.registryService.gracefulShutdown();
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
exports.ConsumerModule = ConsumerModule;
|
|
33
|
+
exports.ConsumerModule = ConsumerModule = __decorate([
|
|
34
|
+
(0, common_1.Module)({}),
|
|
35
|
+
__metadata("design:paramtypes", [consumer_discovery_service_1.ConsumerDiscoveryService,
|
|
36
|
+
consumer_registry_service_1.ConsumerRegistryService])
|
|
37
|
+
], ConsumerModule);
|
|
38
|
+
//# sourceMappingURL=consumer.module.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"consumer.module.js","sourceRoot":"","sources":["../lib/consumer.module.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2CAA6E;AAC7E,uFAAkF;AAClF,oFAA+E;AAGxE,IAAM,cAAc,GAApB,MAAM,cAAc;IAEN;IACA;IAFnB,YACmB,gBAA0C,EAC1C,eAAwC;QADxC,qBAAgB,GAAhB,gBAAgB,CAA0B;QAC1C,oBAAe,GAAf,eAAe,CAAyB;IACxD,CAAC;IAEJ,KAAK,CAAC,YAAY;QAChB,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,CAAC;QAE5D,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QAClD,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,qBAAqB;QACzB,MAAM,IAAI,CAAC,eAAe,CAAC,gBAAgB,EAAE,CAAC;IAChD,CAAC;CACF,CAAA;AAhBY,wCAAc;yBAAd,cAAc;IAD1B,IAAA,eAAM,EAAC,EAAE,CAAC;qCAG4B,qDAAwB;QACzB,mDAAuB;GAHhD,cAAc,CAgB1B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const KAFKA_CONSUMER_METADATA: unique symbol;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../lib/decorators/constants.ts"],"names":[],"mappings":";;;AAAa,QAAA,uBAAuB,GAAG,MAAM,CAAC,yBAAyB,CAAC,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { ConsumerOptions } from '../interfaces';
|
|
2
|
+
export interface ConsumerMethodMetadata {
|
|
3
|
+
topic: string;
|
|
4
|
+
options: Partial<ConsumerOptions>;
|
|
5
|
+
}
|
|
6
|
+
export declare function Consumer(topic: string, options?: Partial<ConsumerOptions>): MethodDecorator;
|
|
7
|
+
export type MessageHandler<T = any> = (message: T) => Promise<void>;
|
|
8
|
+
export type BatchHandler<T = any> = (messages: T[]) => Promise<void>;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.Consumer = Consumer;
|
|
4
|
+
const common_1 = require("@nestjs/common");
|
|
5
|
+
const constants_1 = require("./constants");
|
|
6
|
+
function Consumer(topic, options) {
|
|
7
|
+
return (target, propertyKey, descriptor) => {
|
|
8
|
+
const metadata = {
|
|
9
|
+
topic,
|
|
10
|
+
options: options || {},
|
|
11
|
+
};
|
|
12
|
+
(0, common_1.SetMetadata)(constants_1.KAFKA_CONSUMER_METADATA, metadata)(target, propertyKey, descriptor);
|
|
13
|
+
return descriptor;
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
//# sourceMappingURL=consumer.decorator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"consumer.decorator.js","sourceRoot":"","sources":["../../lib/decorators/consumer.decorator.ts"],"names":[],"mappings":";;AASA,4BAsBC;AA/BD,2CAA6C;AAE7C,2CAAsD;AAOtD,SAAgB,QAAQ,CACtB,KAAa,EACb,OAAkC;IAElC,OAAO,CACL,MAAc,EACd,WAA4B,EAC5B,UAA8B,EAC9B,EAAE;QACF,MAAM,QAAQ,GAA2B;YACvC,KAAK;YACL,OAAO,EAAE,OAAO,IAAI,EAAE;SACvB,CAAC;QAEF,IAAA,oBAAW,EAAC,mCAAuB,EAAE,QAAQ,CAAC,CAC5C,MAAM,EACN,WAAW,EACX,UAAU,CACX,CAAC;QAEF,OAAO,UAAU,CAAC;IACpB,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.Consumer = void 0;
|
|
18
|
+
__exportStar(require("./constants"), exports);
|
|
19
|
+
var consumer_decorator_1 = require("./consumer.decorator");
|
|
20
|
+
Object.defineProperty(exports, "Consumer", { enumerable: true, get: function () { return consumer_decorator_1.Consumer; } });
|
|
21
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../lib/decorators/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AAAA,8CAA4B;AAC5B,2DAAwE;AAA/D,8GAAA,QAAQ,OAAA"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { OnModuleInit } from '@nestjs/common';
|
|
2
|
+
import { DiscoveryService, MetadataScanner, Reflector } from '@nestjs/core';
|
|
3
|
+
import { ConsumerMetadata } from '../interfaces';
|
|
4
|
+
export declare class ConsumerDiscoveryService implements OnModuleInit {
|
|
5
|
+
private readonly discoveryService;
|
|
6
|
+
private readonly reflector;
|
|
7
|
+
private readonly metadataScanner;
|
|
8
|
+
private readonly logger;
|
|
9
|
+
private discoveredConsumers;
|
|
10
|
+
constructor(discoveryService: DiscoveryService, reflector: Reflector, metadataScanner: MetadataScanner);
|
|
11
|
+
onModuleInit(): void;
|
|
12
|
+
discoverConsumers(): ConsumerMetadata[];
|
|
13
|
+
getConsumers(): ConsumerMetadata[];
|
|
14
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
var ConsumerDiscoveryService_1;
|
|
12
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
|
+
exports.ConsumerDiscoveryService = void 0;
|
|
14
|
+
const common_1 = require("@nestjs/common");
|
|
15
|
+
const core_1 = require("@nestjs/core");
|
|
16
|
+
const constants_1 = require("../decorators/constants");
|
|
17
|
+
let ConsumerDiscoveryService = ConsumerDiscoveryService_1 = class ConsumerDiscoveryService {
|
|
18
|
+
discoveryService;
|
|
19
|
+
reflector;
|
|
20
|
+
metadataScanner;
|
|
21
|
+
logger = new common_1.Logger(ConsumerDiscoveryService_1.name);
|
|
22
|
+
discoveredConsumers = [];
|
|
23
|
+
constructor(discoveryService, reflector, metadataScanner) {
|
|
24
|
+
this.discoveryService = discoveryService;
|
|
25
|
+
this.reflector = reflector;
|
|
26
|
+
this.metadataScanner = metadataScanner;
|
|
27
|
+
}
|
|
28
|
+
onModuleInit() {
|
|
29
|
+
this.discoverConsumers();
|
|
30
|
+
}
|
|
31
|
+
discoverConsumers() {
|
|
32
|
+
if (this.discoveredConsumers.length > 0) {
|
|
33
|
+
return this.discoveredConsumers;
|
|
34
|
+
}
|
|
35
|
+
const providers = this.discoveryService.getProviders();
|
|
36
|
+
for (const wrapper of providers) {
|
|
37
|
+
const { instance, metatype } = wrapper;
|
|
38
|
+
if (!instance || !metatype) {
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
const prototype = Object.getPrototypeOf(instance);
|
|
42
|
+
const methodNames = this.metadataScanner.getAllMethodNames(prototype);
|
|
43
|
+
for (const methodName of methodNames) {
|
|
44
|
+
const methodRef = prototype[methodName];
|
|
45
|
+
if (typeof methodRef !== 'function') {
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
const metadata = this.reflector.get(constants_1.KAFKA_CONSUMER_METADATA, methodRef);
|
|
49
|
+
if (!metadata) {
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
if (metadata.options.disabled) {
|
|
53
|
+
this.logger.log(`Skipping disabled consumer: ${metatype.name}.${methodName} for topic: ${metadata.topic}`);
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
const consumerMetadata = {
|
|
57
|
+
topic: metadata.topic,
|
|
58
|
+
options: {
|
|
59
|
+
...metadata.options,
|
|
60
|
+
topic: metadata.topic,
|
|
61
|
+
},
|
|
62
|
+
target: instance,
|
|
63
|
+
methodName,
|
|
64
|
+
};
|
|
65
|
+
this.discoveredConsumers.push(consumerMetadata);
|
|
66
|
+
this.logger.log(`Discovered consumer: ${metatype.name}.${methodName} for topic: ${metadata.topic}`);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return this.discoveredConsumers;
|
|
70
|
+
}
|
|
71
|
+
getConsumers() {
|
|
72
|
+
return this.discoveredConsumers;
|
|
73
|
+
}
|
|
74
|
+
};
|
|
75
|
+
exports.ConsumerDiscoveryService = ConsumerDiscoveryService;
|
|
76
|
+
exports.ConsumerDiscoveryService = ConsumerDiscoveryService = ConsumerDiscoveryService_1 = __decorate([
|
|
77
|
+
(0, common_1.Injectable)(),
|
|
78
|
+
__metadata("design:paramtypes", [core_1.DiscoveryService,
|
|
79
|
+
core_1.Reflector,
|
|
80
|
+
core_1.MetadataScanner])
|
|
81
|
+
], ConsumerDiscoveryService);
|
|
82
|
+
//# sourceMappingURL=consumer-discovery.service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"consumer-discovery.service.js","sourceRoot":"","sources":["../../lib/discovery/consumer-discovery.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAAkE;AAClE,uCAA4E;AAC5E,uDAAkE;AAK3D,IAAM,wBAAwB,gCAA9B,MAAM,wBAAwB;IAKhB;IACA;IACA;IANF,MAAM,GAAG,IAAI,eAAM,CAAC,0BAAwB,CAAC,IAAI,CAAC,CAAC;IAC5D,mBAAmB,GAAuB,EAAE,CAAC;IAErD,YACmB,gBAAkC,EAClC,SAAoB,EACpB,eAAgC;QAFhC,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,cAAS,GAAT,SAAS,CAAW;QACpB,oBAAe,GAAf,eAAe,CAAiB;IAChD,CAAC;IAEJ,YAAY;QACV,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAED,iBAAiB;QACf,IAAI,IAAI,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxC,OAAO,IAAI,CAAC,mBAAmB,CAAC;QAClC,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE,CAAC;QAEvD,KAAK,MAAM,OAAO,IAAI,SAAS,EAAE,CAAC;YAEhC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;YAEvC,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAC3B,SAAS;YACX,CAAC;YAED,MAAM,SAAS,GAAG,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAW,CAAC;YAC5D,MAAM,WAAW,GAAG,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAEtE,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACrC,MAAM,SAAS,GAAG,SAAS,CAAC,UAAoC,CAAC,CAAC;gBAElE,IAAI,OAAO,SAAS,KAAK,UAAU,EAAE,CAAC;oBACpC,SAAS;gBACX,CAAC;gBAED,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CACjC,mCAAuB,EACvB,SAAS,CACV,CAAC;gBAEF,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,SAAS;gBACX,CAAC;gBAGD,IAAI,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;oBAC9B,IAAI,CAAC,MAAM,CAAC,GAAG,CACb,+BAA+B,QAAQ,CAAC,IAAI,IAAI,UAAU,eAAe,QAAQ,CAAC,KAAK,EAAE,CAC1F,CAAC;oBACF,SAAS;gBACX,CAAC;gBAED,MAAM,gBAAgB,GAAqB;oBACzC,KAAK,EAAE,QAAQ,CAAC,KAAK;oBACrB,OAAO,EAAE;wBACP,GAAG,QAAQ,CAAC,OAAO;wBACnB,KAAK,EAAE,QAAQ,CAAC,KAAK;qBACtB;oBAED,MAAM,EAAE,QAAQ;oBAChB,UAAU;iBACX,CAAC;gBAEF,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;gBAChD,IAAI,CAAC,MAAM,CAAC,GAAG,CACb,wBAAwB,QAAQ,CAAC,IAAI,IAAI,UAAU,eAAe,QAAQ,CAAC,KAAK,EAAE,CACnF,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,mBAAmB,CAAC;IAClC,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,mBAAmB,CAAC;IAClC,CAAC;CACF,CAAA;AAhFY,4DAAwB;mCAAxB,wBAAwB;IADpC,IAAA,mBAAU,GAAE;qCAM0B,uBAAgB;QACvB,gBAAS;QACH,sBAAe;GAPxC,wBAAwB,CAgFpC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './consumer-discovery.service';
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./consumer-discovery.service"), exports);
|
|
18
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../lib/discovery/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,+DAA6C"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './kafka-health-indicator';
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./kafka-health-indicator"), exports);
|
|
18
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../lib/health/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,2DAAyC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { HealthIndicatorService, HealthIndicatorResult } from '@nestjs/terminus';
|
|
2
|
+
import { KafkaClient } from '../services/kafka-client.service';
|
|
3
|
+
import { KafkaCoreService } from '../services/kafka-core.service';
|
|
4
|
+
export declare class KafkaHealthIndicator {
|
|
5
|
+
private readonly healthIndicatorService;
|
|
6
|
+
private readonly kafkaClient;
|
|
7
|
+
private readonly kafkaCore;
|
|
8
|
+
constructor(healthIndicatorService: HealthIndicatorService, kafkaClient: KafkaClient, kafkaCore: KafkaCoreService);
|
|
9
|
+
isHealthy(key: string): HealthIndicatorResult;
|
|
10
|
+
checkBrokers(key: string): Promise<HealthIndicatorResult>;
|
|
11
|
+
checkConsumerLag(key: string, groupId: string, maxLag?: number): Promise<HealthIndicatorResult>;
|
|
12
|
+
}
|