@application-kit/kafka 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +95 -0
  3. package/dist/cjs/configs/envs/index.d.ts +2 -0
  4. package/dist/cjs/configs/envs/index.js +18 -0
  5. package/dist/cjs/configs/envs/kafka-config.environment.d.ts +9 -0
  6. package/dist/cjs/configs/envs/kafka-config.environment.js +64 -0
  7. package/dist/cjs/configs/envs/kafka-consumer.environment.d.ts +3 -0
  8. package/dist/cjs/configs/envs/kafka-consumer.environment.js +27 -0
  9. package/dist/cjs/configs/index.d.ts +4 -0
  10. package/dist/cjs/configs/index.js +20 -0
  11. package/dist/cjs/configs/kafka-consumer.config.d.ts +9 -0
  12. package/dist/cjs/configs/kafka-consumer.config.js +50 -0
  13. package/dist/cjs/configs/kafka-producer.config.d.ts +7 -0
  14. package/dist/cjs/configs/kafka-producer.config.js +41 -0
  15. package/dist/cjs/configs/kafka.config.d.ts +7 -0
  16. package/dist/cjs/configs/kafka.config.js +52 -0
  17. package/dist/cjs/constants.d.ts +3 -0
  18. package/dist/cjs/constants.js +6 -0
  19. package/dist/cjs/event-contract-version.d.ts +5 -0
  20. package/dist/cjs/event-contract-version.js +20 -0
  21. package/dist/cjs/index.d.ts +7 -0
  22. package/dist/cjs/index.js +23 -0
  23. package/dist/cjs/kafka-commit-interceptor.d.ts +13 -0
  24. package/dist/cjs/kafka-commit-interceptor.js +87 -0
  25. package/dist/cjs/kafka-error-handler.d.ts +25 -0
  26. package/dist/cjs/kafka-error-handler.js +118 -0
  27. package/dist/cjs/kafka.module.d.ts +9 -0
  28. package/dist/cjs/kafka.module.js +67 -0
  29. package/dist/cjs/libs/consumer.abstract.d.ts +6 -0
  30. package/dist/cjs/libs/consumer.abstract.js +14 -0
  31. package/dist/cjs/libs/errors/consumers-error.d.ts +4 -0
  32. package/dist/cjs/libs/errors/consumers-error.js +12 -0
  33. package/dist/cjs/libs/errors/index.d.ts +1 -0
  34. package/dist/cjs/libs/errors/index.js +17 -0
  35. package/dist/cjs/libs/heartbeat-file-writer.d.ts +6 -0
  36. package/dist/cjs/libs/heartbeat-file-writer.js +24 -0
  37. package/dist/cjs/libs/index.d.ts +9 -0
  38. package/dist/cjs/libs/index.js +25 -0
  39. package/dist/cjs/libs/kafka-consumer-heartbeat.d.ts +3 -0
  40. package/dist/cjs/libs/kafka-consumer-heartbeat.js +13 -0
  41. package/dist/cjs/libs/kafka-consumer-server.d.ts +2 -0
  42. package/dist/cjs/libs/kafka-consumer-server.js +25 -0
  43. package/dist/cjs/libs/kafka-logger.d.ts +5 -0
  44. package/dist/cjs/libs/kafka-logger.interface.d.ts +5 -0
  45. package/dist/cjs/libs/kafka-logger.interface.js +2 -0
  46. package/dist/cjs/libs/kafka-logger.js +59 -0
  47. package/dist/cjs/libs/kafka-message-context.d.ts +6 -0
  48. package/dist/cjs/libs/kafka-message-context.js +11 -0
  49. package/dist/cjs/libs/topic-suffixes.d.ts +1 -0
  50. package/dist/cjs/libs/topic-suffixes.js +10 -0
  51. package/dist/cjs/package.json +3 -0
  52. package/dist/esm/configs/envs/index.d.ts +2 -0
  53. package/dist/esm/configs/envs/index.js +2 -0
  54. package/dist/esm/configs/envs/kafka-config.environment.d.ts +9 -0
  55. package/dist/esm/configs/envs/kafka-config.environment.js +60 -0
  56. package/dist/esm/configs/envs/kafka-consumer.environment.d.ts +3 -0
  57. package/dist/esm/configs/envs/kafka-consumer.environment.js +24 -0
  58. package/dist/esm/configs/index.d.ts +4 -0
  59. package/dist/esm/configs/index.js +4 -0
  60. package/dist/esm/configs/kafka-consumer.config.d.ts +9 -0
  61. package/dist/esm/configs/kafka-consumer.config.js +47 -0
  62. package/dist/esm/configs/kafka-producer.config.d.ts +7 -0
  63. package/dist/esm/configs/kafka-producer.config.js +38 -0
  64. package/dist/esm/configs/kafka.config.d.ts +7 -0
  65. package/dist/esm/configs/kafka.config.js +49 -0
  66. package/dist/esm/constants.d.ts +3 -0
  67. package/dist/esm/constants.js +3 -0
  68. package/dist/esm/event-contract-version.d.ts +5 -0
  69. package/dist/esm/event-contract-version.js +14 -0
  70. package/dist/esm/index.d.ts +7 -0
  71. package/dist/esm/index.js +7 -0
  72. package/dist/esm/kafka-commit-interceptor.d.ts +13 -0
  73. package/dist/esm/kafka-commit-interceptor.js +83 -0
  74. package/dist/esm/kafka-error-handler.d.ts +25 -0
  75. package/dist/esm/kafka-error-handler.js +115 -0
  76. package/dist/esm/kafka.module.d.ts +9 -0
  77. package/dist/esm/kafka.module.js +64 -0
  78. package/dist/esm/libs/consumer.abstract.d.ts +6 -0
  79. package/dist/esm/libs/consumer.abstract.js +10 -0
  80. package/dist/esm/libs/errors/consumers-error.d.ts +4 -0
  81. package/dist/esm/libs/errors/consumers-error.js +8 -0
  82. package/dist/esm/libs/errors/index.d.ts +1 -0
  83. package/dist/esm/libs/errors/index.js +1 -0
  84. package/dist/esm/libs/heartbeat-file-writer.d.ts +6 -0
  85. package/dist/esm/libs/heartbeat-file-writer.js +20 -0
  86. package/dist/esm/libs/index.d.ts +9 -0
  87. package/dist/esm/libs/index.js +9 -0
  88. package/dist/esm/libs/kafka-consumer-heartbeat.d.ts +3 -0
  89. package/dist/esm/libs/kafka-consumer-heartbeat.js +9 -0
  90. package/dist/esm/libs/kafka-consumer-server.d.ts +2 -0
  91. package/dist/esm/libs/kafka-consumer-server.js +18 -0
  92. package/dist/esm/libs/kafka-logger.d.ts +5 -0
  93. package/dist/esm/libs/kafka-logger.interface.d.ts +5 -0
  94. package/dist/esm/libs/kafka-logger.interface.js +1 -0
  95. package/dist/esm/libs/kafka-logger.js +23 -0
  96. package/dist/esm/libs/kafka-message-context.d.ts +6 -0
  97. package/dist/esm/libs/kafka-message-context.js +7 -0
  98. package/dist/esm/libs/topic-suffixes.d.ts +1 -0
  99. package/dist/esm/libs/topic-suffixes.js +6 -0
  100. package/package.json +85 -0
@@ -0,0 +1,9 @@
1
+ import { KafkaOptions } from '@nestjs/microservices';
2
+ import { KafkaConsumerEnvironment } from './envs/kafka-consumer.environment.js';
3
+ import { KafkaConfig } from './kafka.config.js';
4
+ export declare class KafkaConsumerConfig {
5
+ private readonly kafkaConfig;
6
+ private readonly kafkaConsumerEnvironment;
7
+ constructor(kafkaConfig: KafkaConfig, kafkaConsumerEnvironment: KafkaConsumerEnvironment);
8
+ getOptions(): KafkaOptions;
9
+ }
@@ -0,0 +1,47 @@
1
+ var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
2
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
4
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
5
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
6
+ };
7
+ var __metadata = (this && this.__metadata) || function (k, v) {
8
+ if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
9
+ };
10
+ import { Injectable } from '@nestjs/common';
11
+ import { KafkaConsumerEnvironment } from './envs/kafka-consumer.environment.js';
12
+ import { KafkaConfig } from './kafka.config.js';
13
+ let KafkaConsumerConfig = class KafkaConsumerConfig {
14
+ kafkaConfig;
15
+ kafkaConsumerEnvironment;
16
+ constructor(kafkaConfig, kafkaConsumerEnvironment) {
17
+ this.kafkaConfig = kafkaConfig;
18
+ this.kafkaConsumerEnvironment = kafkaConsumerEnvironment;
19
+ }
20
+ getOptions() {
21
+ const options = this.kafkaConfig.getOptions();
22
+ options.options = {
23
+ ...options.options,
24
+ consumer: {
25
+ allowAutoTopicCreation: true,
26
+ groupId: this.kafkaConsumerEnvironment.KAFKA_GROUP_ID,
27
+ retry: {
28
+ initialRetryTime: 100,
29
+ retries: 0,
30
+ },
31
+ },
32
+ run: {
33
+ autoCommit: false,
34
+ },
35
+ subscribe: {
36
+ fromBeginning: true,
37
+ },
38
+ };
39
+ return options;
40
+ }
41
+ };
42
+ KafkaConsumerConfig = __decorate([
43
+ Injectable(),
44
+ __metadata("design:paramtypes", [KafkaConfig,
45
+ KafkaConsumerEnvironment])
46
+ ], KafkaConsumerConfig);
47
+ export { KafkaConsumerConfig };
@@ -0,0 +1,7 @@
1
+ import { KafkaOptions } from '@nestjs/microservices';
2
+ import { KafkaConfig } from './kafka.config.js';
3
+ export declare class KafkaProducerConfig {
4
+ private readonly kafkaConfig;
5
+ constructor(kafkaConfig: KafkaConfig);
6
+ getOptions(): KafkaOptions;
7
+ }
@@ -0,0 +1,38 @@
1
+ var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
2
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
4
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
5
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
6
+ };
7
+ var __metadata = (this && this.__metadata) || function (k, v) {
8
+ if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
9
+ };
10
+ import { Injectable } from '@nestjs/common';
11
+ import { Partitioners } from 'kafkajs';
12
+ import { KafkaConfig } from './kafka.config.js';
13
+ let KafkaProducerConfig = class KafkaProducerConfig {
14
+ kafkaConfig;
15
+ constructor(kafkaConfig) {
16
+ this.kafkaConfig = kafkaConfig;
17
+ }
18
+ getOptions() {
19
+ const options = this.kafkaConfig.getOptions();
20
+ options.options = {
21
+ ...options.options,
22
+ producer: {
23
+ allowAutoTopicCreation: true,
24
+ createPartitioner: Partitioners.DefaultPartitioner,
25
+ retry: {
26
+ retries: 1,
27
+ },
28
+ },
29
+ producerOnlyMode: true,
30
+ };
31
+ return options;
32
+ }
33
+ };
34
+ KafkaProducerConfig = __decorate([
35
+ Injectable(),
36
+ __metadata("design:paramtypes", [KafkaConfig])
37
+ ], KafkaProducerConfig);
38
+ export { KafkaProducerConfig };
@@ -0,0 +1,7 @@
1
+ import { KafkaOptions } from '@nestjs/microservices';
2
+ import { KafkaConfigEnvironment } from './envs/kafka-config.environment.js';
3
+ export declare class KafkaConfig {
4
+ protected readonly kafkaEnvironment: KafkaConfigEnvironment;
5
+ constructor(kafkaEnvironment: KafkaConfigEnvironment);
6
+ getOptions(): KafkaOptions;
7
+ }
@@ -0,0 +1,49 @@
1
+ var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
2
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
4
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
5
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
6
+ };
7
+ var __metadata = (this && this.__metadata) || function (k, v) {
8
+ if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
9
+ };
10
+ import { Injectable } from '@nestjs/common';
11
+ import { Transport } from '@nestjs/microservices';
12
+ import { logLevel } from '@nestjs/microservices/external/kafka.interface.js';
13
+ import { randomUUID } from 'node:crypto';
14
+ import { readFileSync } from 'node:fs';
15
+ import { KafkaConfigEnvironment } from './envs/kafka-config.environment.js';
16
+ let KafkaConfig = class KafkaConfig {
17
+ kafkaEnvironment;
18
+ constructor(kafkaEnvironment) {
19
+ this.kafkaEnvironment = kafkaEnvironment;
20
+ }
21
+ getOptions() {
22
+ const { KAFKA_BROKERS, KAFKA_CLIENT_ID, KAFKA_LOG_LEVEL, KAFKA_SASL_PASSWORD, KAFKA_SASL_USERNAME, KAFKA_SSL_CA_FILE, } = this.kafkaEnvironment;
23
+ let ssl;
24
+ if (KAFKA_SSL_CA_FILE !== undefined) {
25
+ ssl = { ca: [readFileSync(KAFKA_SSL_CA_FILE, 'utf8')] };
26
+ }
27
+ const sasl = KAFKA_SASL_PASSWORD === undefined || KAFKA_SASL_USERNAME === undefined
28
+ ? undefined
29
+ : { mechanism: 'scram-sha-512', password: KAFKA_SASL_PASSWORD, username: KAFKA_SASL_USERNAME };
30
+ return {
31
+ options: {
32
+ client: {
33
+ brokers: KAFKA_BROKERS.split(','),
34
+ clientId: `${KAFKA_CLIENT_ID}-${randomUUID()}`,
35
+ logLevel: logLevel[KAFKA_LOG_LEVEL],
36
+ ...(ssl !== undefined && { ssl }),
37
+ ...(sasl !== undefined && { sasl }),
38
+ },
39
+ postfixId: '',
40
+ },
41
+ transport: Transport.KAFKA,
42
+ };
43
+ }
44
+ };
45
+ KafkaConfig = __decorate([
46
+ Injectable(),
47
+ __metadata("design:paramtypes", [KafkaConfigEnvironment])
48
+ ], KafkaConfig);
49
+ export { KafkaConfig };
@@ -0,0 +1,3 @@
1
+ export declare const KAFKA_PRODUCER: unique symbol;
2
+ export declare const KAFKA_LOGGER: unique symbol;
3
+ export declare const KAFKA_ERROR_HANDLER_OPTIONS: unique symbol;
@@ -0,0 +1,3 @@
1
+ export const KAFKA_PRODUCER = Symbol('KAFKA_PRODUCER');
2
+ export const KAFKA_LOGGER = Symbol('KAFKA_LOGGER');
3
+ export const KAFKA_ERROR_HANDLER_OPTIONS = Symbol('KAFKA_ERROR_HANDLER_OPTIONS');
@@ -0,0 +1,5 @@
1
+ export declare const DEFAULT_EVENT_VERSION = "1";
2
+ export declare const EVENT_VERSION_HEADER = "x-event-version";
3
+ export declare const eventVersionOfTopic: (topic: string) => string;
4
+ export declare const eventVersionOfHeaders: (headers: Record<string, unknown> | undefined) => string;
5
+ export declare const versionedQueueName: (queueName: string, version: string) => string;
@@ -0,0 +1,14 @@
1
+ export const DEFAULT_EVENT_VERSION = '1';
2
+ export const EVENT_VERSION_HEADER = 'x-event-version';
3
+ const VERSION_SEGMENT = /^v\d+$/u;
4
+ const versionSegment = (version) => (version === DEFAULT_EVENT_VERSION ? '' : `.v${version}`);
5
+ export const eventVersionOfTopic = (topic) => {
6
+ const segment = topic.split('.').find((part) => VERSION_SEGMENT.test(part));
7
+ return segment === undefined ? DEFAULT_EVENT_VERSION : segment.slice(1);
8
+ };
9
+ export const eventVersionOfHeaders = (headers) => {
10
+ const raw = headers?.[EVENT_VERSION_HEADER];
11
+ const value = Buffer.isBuffer(raw) ? raw.toString('utf8') : raw;
12
+ return typeof value === 'string' && value !== '' ? value : DEFAULT_EVENT_VERSION;
13
+ };
14
+ export const versionedQueueName = (queueName, version) => `${queueName}${versionSegment(version)}`;
@@ -0,0 +1,7 @@
1
+ export * from './configs/index.js';
2
+ export * from './libs/index.js';
3
+ export * from './constants.js';
4
+ export * from './event-contract-version.js';
5
+ export * from './kafka-commit-interceptor.js';
6
+ export * from './kafka-error-handler.js';
7
+ export * from './kafka.module.js';
@@ -0,0 +1,7 @@
1
+ export * from './configs/index.js';
2
+ export * from './libs/index.js';
3
+ export * from './constants.js';
4
+ export * from './event-contract-version.js';
5
+ export * from './kafka-commit-interceptor.js';
6
+ export * from './kafka-error-handler.js';
7
+ export * from './kafka.module.js';
@@ -0,0 +1,13 @@
1
+ import { CallHandler, ExecutionContext, NestInterceptor } from '@nestjs/common';
2
+ import { Observable } from 'rxjs';
3
+ import type { KafkaLoggerInterface } from './libs/kafka-logger.interface.js';
4
+ import { KafkaErrorHandler } from './kafka-error-handler.js';
5
+ export declare class KafkaCommitInterceptor implements NestInterceptor {
6
+ private readonly kafkaErrorHandler;
7
+ private readonly logger;
8
+ constructor(kafkaErrorHandler: KafkaErrorHandler, logger: KafkaLoggerInterface);
9
+ intercept(context: ExecutionContext, next: CallHandler): Observable<unknown>;
10
+ private readCommitContext;
11
+ private rejectForeignContractVersion;
12
+ }
13
+ export declare const KafkaAutoCommit: () => MethodDecorator;
@@ -0,0 +1,83 @@
1
+ var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
2
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
4
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
5
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
6
+ };
7
+ var __metadata = (this && this.__metadata) || function (k, v) {
8
+ if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
9
+ };
10
+ var __param = (this && this.__param) || function (paramIndex, decorator) {
11
+ return function (target, key) { decorator(target, key, paramIndex); }
12
+ };
13
+ import { applyDecorators, BadRequestException, Inject, Injectable, UseInterceptors, } from '@nestjs/common';
14
+ import { catchError, of, tap, throwError } from 'rxjs';
15
+ import { KAFKA_LOGGER } from './constants.js';
16
+ import { eventVersionOfHeaders, eventVersionOfTopic } from './event-contract-version.js';
17
+ import { KafkaErrorHandler } from './kafka-error-handler.js';
18
+ let KafkaCommitInterceptor = class KafkaCommitInterceptor {
19
+ kafkaErrorHandler;
20
+ logger;
21
+ constructor(kafkaErrorHandler, logger) {
22
+ this.kafkaErrorHandler = kafkaErrorHandler;
23
+ this.logger = logger;
24
+ }
25
+ intercept(context, next) {
26
+ if (context.getType() !== 'rpc') {
27
+ return next.handle();
28
+ }
29
+ const kafkaContext = context.switchToRpc().getContext();
30
+ if (kafkaContext === undefined || typeof kafkaContext.getConsumer !== 'function') {
31
+ return next.handle();
32
+ }
33
+ const { consumer, logContext, nextOffset, partition, topic } = this.readCommitContext(kafkaContext);
34
+ const rejected = this.rejectForeignContractVersion(kafkaContext, topic);
35
+ return (rejected ?? next.handle()).pipe(tap(async () => {
36
+ try {
37
+ await consumer.commitOffsets([{ offset: nextOffset, partition, topic }]);
38
+ this.logger.info(logContext, 'Kafka message processed');
39
+ }
40
+ catch (commitError) {
41
+ this.logger.error({ errorContext: { ...logContext, error: commitError } }, `Failed to commit offset: ${Error.isError(commitError) ? commitError.message : 'Unknown error'}`);
42
+ }
43
+ }), catchError(async (error) => {
44
+ try {
45
+ const commitOffsets = async () => {
46
+ await consumer.commitOffsets([{ offset: nextOffset, partition, topic }]);
47
+ };
48
+ await this.kafkaErrorHandler.handleError(kafkaContext, error, commitOffsets);
49
+ this.logger.warn({ errorContext: { ...logContext, error: error.message } }, `Message moved to retry/dlq: ${error.message}`);
50
+ }
51
+ catch (catchedError) {
52
+ this.logger.error({ errorContext: { ...logContext, error: catchedError } }, `Failed to move message to retry/dlq: ${Error.isError(catchedError) ? catchedError.message : 'Unknown error'}`);
53
+ }
54
+ return of(error);
55
+ }));
56
+ }
57
+ readCommitContext(kafkaContext) {
58
+ const { headers, key, offset } = kafkaContext.getMessage();
59
+ const partition = kafkaContext.getPartition();
60
+ const topic = kafkaContext.getTopic();
61
+ return {
62
+ consumer: kafkaContext.getConsumer(),
63
+ logContext: { headers, key, offset, partition, topic },
64
+ nextOffset: (BigInt(offset) + 1n).toString(),
65
+ partition,
66
+ topic,
67
+ };
68
+ }
69
+ rejectForeignContractVersion(kafkaContext, topic) {
70
+ const expectedVersion = eventVersionOfTopic(topic);
71
+ const messageVersion = eventVersionOfHeaders(kafkaContext.getMessage().headers);
72
+ return messageVersion === expectedVersion
73
+ ? undefined
74
+ : throwError(() => new BadRequestException(`Unsupported event contract version "${messageVersion}" on topic "${topic}": expected "${expectedVersion}"`));
75
+ }
76
+ };
77
+ KafkaCommitInterceptor = __decorate([
78
+ Injectable(),
79
+ __param(1, Inject(KAFKA_LOGGER)),
80
+ __metadata("design:paramtypes", [KafkaErrorHandler, Object])
81
+ ], KafkaCommitInterceptor);
82
+ export { KafkaCommitInterceptor };
83
+ export const KafkaAutoCommit = () => applyDecorators(UseInterceptors(KafkaCommitInterceptor));
@@ -0,0 +1,25 @@
1
+ import { ClientKafka, KafkaContext } from '@nestjs/microservices';
2
+ import type { KafkaLoggerInterface } from './libs/kafka-logger.interface.js';
3
+ export interface KafkaErrorHandlerOptionsInterface {
4
+ baseDelayMs: number;
5
+ maxAttempts: number;
6
+ topicSuffix: {
7
+ dlq: string;
8
+ retry: string;
9
+ };
10
+ }
11
+ export declare class KafkaErrorHandler {
12
+ private readonly client;
13
+ private readonly logger;
14
+ private readonly options;
15
+ constructor(client: ClientKafka, logger: KafkaLoggerInterface, options: KafkaErrorHandlerOptionsInterface);
16
+ handleError(context: KafkaContext, error: Error, commit: () => Promise<void>): Promise<void>;
17
+ private buildPayload;
18
+ private headers;
19
+ private readAttempt;
20
+ private scheduleRetry;
21
+ private sendToDlq;
22
+ private stringifyKey;
23
+ private stringifyValue;
24
+ private stripRetrySuffixes;
25
+ }
@@ -0,0 +1,115 @@
1
+ var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
2
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
4
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
5
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
6
+ };
7
+ var __metadata = (this && this.__metadata) || function (k, v) {
8
+ if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
9
+ };
10
+ var __param = (this && this.__param) || function (paramIndex, decorator) {
11
+ return function (target, key) { decorator(target, key, paramIndex); }
12
+ };
13
+ import { BadRequestException, Inject, Injectable } from '@nestjs/common';
14
+ import { ClientKafka } from '@nestjs/microservices';
15
+ import { firstValueFrom } from 'rxjs';
16
+ import { KAFKA_ERROR_HANDLER_OPTIONS, KAFKA_LOGGER, KAFKA_PRODUCER, } from './constants.js';
17
+ let KafkaErrorHandler = class KafkaErrorHandler {
18
+ client;
19
+ logger;
20
+ options;
21
+ constructor(client, logger, options) {
22
+ this.client = client;
23
+ this.logger = logger;
24
+ this.options = options;
25
+ }
26
+ async handleError(context, error, commit) {
27
+ const message = context.getMessage();
28
+ const topic = context.getTopic();
29
+ const payload = this.buildPayload(message);
30
+ const attempt = this.readAttempt(message.headers ?? {});
31
+ const baseTopic = this.stripRetrySuffixes(topic);
32
+ const isRetryableError = !(error instanceof BadRequestException);
33
+ if (isRetryableError && attempt < this.options.maxAttempts) {
34
+ this.scheduleRetry({ attempt, baseTopic, commit, error, payload, topic });
35
+ return;
36
+ }
37
+ await this.sendToDlq({ baseTopic, commit, error, key: message.key, payload });
38
+ }
39
+ buildPayload(message) {
40
+ return {
41
+ headers: this.headers(message.headers ?? {}),
42
+ key: this.stringifyKey(message.key),
43
+ value: this.stringifyValue(message.value),
44
+ };
45
+ }
46
+ headers(headers) {
47
+ const flatHeaders = {};
48
+ for (const [key, value] of Object.entries(headers)) {
49
+ if (value !== undefined) {
50
+ flatHeaders[key] = typeof value === 'object' && !Buffer.isBuffer(value)
51
+ ? JSON.stringify(value)
52
+ : value;
53
+ }
54
+ }
55
+ return flatHeaders;
56
+ }
57
+ readAttempt(headers) {
58
+ const parsedAttempt = Number(headers['x-retry-attempt']?.toString());
59
+ return Number.isFinite(parsedAttempt) ? Math.trunc(parsedAttempt) : 0;
60
+ }
61
+ scheduleRetry(parameters) {
62
+ const { attempt, baseTopic, commit, error, payload, topic } = parameters;
63
+ const { baseDelayMs, maxAttempts, topicSuffix } = this.options;
64
+ const retryTopic = `${baseTopic}.${topicSuffix.retry}`;
65
+ const nextAttempt = attempt + 1;
66
+ const delayMs = baseDelayMs * 2 ** attempt;
67
+ payload.headers['x-retry-attempt'] = nextAttempt.toString();
68
+ payload.headers['x-retry-reason'] = error.message;
69
+ this.logger.warn({ errorContext: {
70
+ delayMs,
71
+ headers: payload.headers,
72
+ key: payload.key,
73
+ maxAttempts,
74
+ nextAttempt,
75
+ topic,
76
+ } }, `[Retry] Attempt ${nextAttempt}/${maxAttempts} for "${topic}". Waiting ${delayMs}ms...`);
77
+ setTimeout(async () => {
78
+ try {
79
+ await firstValueFrom(this.client.emit(retryTopic, payload));
80
+ await commit();
81
+ }
82
+ catch (retryError) {
83
+ this.logger.error({ errorContext: { error: retryError, key: payload.key, topic: retryTopic } }, `[Retry] Failed to re-emit to "${retryTopic}": ${Error.isError(retryError) ? retryError.message : 'Unknown error'}`);
84
+ }
85
+ }, delayMs);
86
+ }
87
+ async sendToDlq(parameters) {
88
+ const { baseTopic, commit, error, key, payload } = parameters;
89
+ const dlqTopic = `${baseTopic}.${this.options.topicSuffix.dlq}`;
90
+ payload.headers['x-dlq-reason'] = error.message;
91
+ this.logger.error({ errorContext: { headers: payload.headers, key, topic: dlqTopic } }, `[DLQ] Send to "${dlqTopic}"`);
92
+ await firstValueFrom(this.client.emit(dlqTopic, payload));
93
+ await commit();
94
+ }
95
+ stringifyKey(key) {
96
+ return typeof key === 'object' && !Buffer.isBuffer(key) ? JSON.stringify(key) : key;
97
+ }
98
+ stringifyValue(value) {
99
+ return typeof value === 'object' && !Buffer.isBuffer(value) ? JSON.stringify(value) : value;
100
+ }
101
+ stripRetrySuffixes(topic) {
102
+ const { topicSuffix } = this.options;
103
+ return topic
104
+ .replace(`.${topicSuffix.retry}`, '')
105
+ .replace(`.${topicSuffix.dlq}`, '');
106
+ }
107
+ };
108
+ KafkaErrorHandler = __decorate([
109
+ Injectable(),
110
+ __param(0, Inject(KAFKA_PRODUCER)),
111
+ __param(1, Inject(KAFKA_LOGGER)),
112
+ __param(2, Inject(KAFKA_ERROR_HANDLER_OPTIONS)),
113
+ __metadata("design:paramtypes", [ClientKafka, Object, Object])
114
+ ], KafkaErrorHandler);
115
+ export { KafkaErrorHandler };
@@ -0,0 +1,9 @@
1
+ import { DynamicModule, Type } from '@nestjs/common';
2
+ import { KafkaErrorHandlerOptionsInterface } from './kafka-error-handler.js';
3
+ import { KafkaLoggerInterface } from './libs/kafka-logger.interface.js';
4
+ export declare class KafkaModule {
5
+ static forRootAsync(config: {
6
+ errorHandlerOptions?: Partial<KafkaErrorHandlerOptionsInterface>;
7
+ logger: Type<KafkaLoggerInterface>;
8
+ }): DynamicModule;
9
+ }
@@ -0,0 +1,64 @@
1
+ var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
2
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
4
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
5
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
6
+ };
7
+ import { ConfigsModule } from '@libs-for-dev/nestjs-config-module';
8
+ import { Module } from '@nestjs/common';
9
+ import { ClientsModule } from '@nestjs/microservices';
10
+ import { KafkaConfigEnvironment } from './configs/envs/kafka-config.environment.js';
11
+ import { KafkaProducerConfig } from './configs/kafka-producer.config.js';
12
+ import { KafkaConfig } from './configs/kafka.config.js';
13
+ import { KAFKA_ERROR_HANDLER_OPTIONS, KAFKA_LOGGER, KAFKA_PRODUCER, } from './constants.js';
14
+ import { KafkaCommitInterceptor } from './kafka-commit-interceptor.js';
15
+ import { KafkaErrorHandler } from './kafka-error-handler.js';
16
+ let KafkaModule = class KafkaModule {
17
+ static forRootAsync(config) {
18
+ return {
19
+ exports: [KAFKA_LOGGER],
20
+ global: true,
21
+ module: this,
22
+ providers: [
23
+ {
24
+ provide: KAFKA_ERROR_HANDLER_OPTIONS,
25
+ useValue: {
26
+ baseDelayMs: 500,
27
+ maxAttempts: 5,
28
+ topicSuffix: {
29
+ dlq: 'dlq',
30
+ retry: 'retry',
31
+ },
32
+ ...config.errorHandlerOptions,
33
+ },
34
+ },
35
+ {
36
+ provide: KAFKA_LOGGER,
37
+ useClass: config.logger,
38
+ },
39
+ ],
40
+ };
41
+ }
42
+ };
43
+ KafkaModule = __decorate([
44
+ Module({
45
+ exports: [KafkaCommitInterceptor, KafkaErrorHandler],
46
+ imports: [
47
+ ClientsModule.registerAsync({
48
+ clients: [
49
+ {
50
+ imports: [ConfigsModule.forFeature([KafkaProducerConfig, KafkaConfig], [KafkaConfigEnvironment])],
51
+ inject: [KafkaProducerConfig],
52
+ name: KAFKA_PRODUCER,
53
+ useFactory: (config) => config.getOptions(),
54
+ },
55
+ ],
56
+ }),
57
+ ],
58
+ providers: [
59
+ KafkaErrorHandler,
60
+ KafkaCommitInterceptor,
61
+ ],
62
+ })
63
+ ], KafkaModule);
64
+ export { KafkaModule };
@@ -0,0 +1,6 @@
1
+ export interface HandlerInterface<TPayload = unknown> {
2
+ handle(payload: TPayload): Promise<void>;
3
+ }
4
+ export declare abstract class ConsumerAbstract {
5
+ protected handleAll<TPayload>(payload: TPayload, handlers: HandlerInterface<TPayload>[]): Promise<void>;
6
+ }
@@ -0,0 +1,10 @@
1
+ import { ConsumersError } from './errors/consumers-error.js';
2
+ export class ConsumerAbstract {
3
+ async handleAll(payload, handlers) {
4
+ const results = await Promise.allSettled(handlers.map(async (handler) => handler.handle(payload)));
5
+ const rejectedResults = results.filter((result) => result.status === 'rejected');
6
+ if (rejectedResults.length > 0) {
7
+ throw new ConsumersError(rejectedResults.map((result) => result.reason));
8
+ }
9
+ }
10
+ }
@@ -0,0 +1,4 @@
1
+ export declare class ConsumersError extends Error {
2
+ readonly errors: Error[];
3
+ constructor(errors: Error[], options?: ErrorOptions);
4
+ }
@@ -0,0 +1,8 @@
1
+ export class ConsumersError extends Error {
2
+ errors;
3
+ constructor(errors, options) {
4
+ super(errors.map((error) => `${error.name}: "${error.message}"`).join('; '), options);
5
+ this.errors = errors;
6
+ this.name = 'ConsumersError';
7
+ }
8
+ }
@@ -0,0 +1 @@
1
+ export * from './consumers-error.js';
@@ -0,0 +1 @@
1
+ export * from './consumers-error.js';
@@ -0,0 +1,6 @@
1
+ export declare class HeartbeatFileWriter {
2
+ private readonly filePath?;
3
+ constructor(filePath?: string | undefined);
4
+ static fromEnvironment(): HeartbeatFileWriter;
5
+ touch(): Promise<void>;
6
+ }
@@ -0,0 +1,20 @@
1
+ import { writeFile } from 'node:fs/promises';
2
+ export class HeartbeatFileWriter {
3
+ filePath;
4
+ constructor(filePath) {
5
+ this.filePath = filePath;
6
+ }
7
+ static fromEnvironment() {
8
+ return new HeartbeatFileWriter(process.env.HEARTBEAT_FILE);
9
+ }
10
+ async touch() {
11
+ if (this.filePath === undefined || this.filePath === '') {
12
+ return;
13
+ }
14
+ try {
15
+ await writeFile(this.filePath, String(Date.now()));
16
+ }
17
+ catch {
18
+ }
19
+ }
20
+ }
@@ -0,0 +1,9 @@
1
+ export * from './errors/index.js';
2
+ export * from './consumer.abstract.js';
3
+ export * from './heartbeat-file-writer.js';
4
+ export * from './kafka-consumer-heartbeat.js';
5
+ export * from './kafka-consumer-server.js';
6
+ export * from './kafka-logger.js';
7
+ export * from './kafka-logger.interface.js';
8
+ export * from './kafka-message-context.js';
9
+ export * from './topic-suffixes.js';
@@ -0,0 +1,9 @@
1
+ export * from './errors/index.js';
2
+ export * from './consumer.abstract.js';
3
+ export * from './heartbeat-file-writer.js';
4
+ export * from './kafka-consumer-heartbeat.js';
5
+ export * from './kafka-consumer-server.js';
6
+ export * from './kafka-logger.js';
7
+ export * from './kafka-logger.interface.js';
8
+ export * from './kafka-message-context.js';
9
+ export * from './topic-suffixes.js';
@@ -0,0 +1,3 @@
1
+ import type { Consumer } from 'kafkajs';
2
+ import type { HeartbeatFileWriter } from './heartbeat-file-writer.js';
3
+ export declare const wireKafkaConsumerHeartbeat: (consumer: Consumer, heartbeat: HeartbeatFileWriter) => void;
@@ -0,0 +1,9 @@
1
+ export const wireKafkaConsumerHeartbeat = (consumer, heartbeat) => {
2
+ void heartbeat.touch();
3
+ consumer.on(consumer.events.HEARTBEAT, () => {
4
+ void heartbeat.touch();
5
+ });
6
+ consumer.on(consumer.events.GROUP_JOIN, () => {
7
+ void heartbeat.touch();
8
+ });
9
+ };
@@ -0,0 +1,2 @@
1
+ import type { DynamicModule, Type } from '@nestjs/common';
2
+ export declare const kafkaConsumerServer: (module: DynamicModule | Type, loggerModule: Type) => Promise<void>;