@rsdk/kafka.common 4.0.0-next.6 → 4.0.0-next.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/dist/client/constants.d.ts +2 -0
- package/dist/client/constants.js +6 -0
- package/dist/client/constants.js.map +1 -0
- package/dist/client/decorators/inject-kafka.decorator.d.ts +1 -0
- package/dist/client/decorators/inject-kafka.decorator.js +8 -0
- package/dist/client/decorators/inject-kafka.decorator.js.map +1 -0
- package/dist/client/decorators/inject-producer.decorator.d.ts +1 -0
- package/dist/client/decorators/inject-producer.decorator.js +8 -0
- package/dist/client/decorators/inject-producer.decorator.js.map +1 -0
- package/dist/client/kafka-client.module.js +5 -5
- package/dist/client/kafka-client.module.js.map +1 -1
- package/dist/client/kafka.base-indicator.d.ts +10 -0
- package/dist/client/{kafka.healthcheck.js → kafka.base-indicator.js} +18 -18
- package/dist/client/kafka.base-indicator.js.map +1 -0
- package/dist/{metadata → client}/kafka.metadata.d.ts +9 -5
- package/dist/client/kafka.metadata.js +10 -0
- package/dist/client/kafka.metadata.js.map +1 -0
- package/dist/client/providers/kafka-producer.provider.d.ts +2 -0
- package/dist/client/providers/kafka-producer.provider.js +19 -0
- package/dist/client/providers/kafka-producer.provider.js.map +1 -0
- package/dist/client/providers/kafka.provider.d.ts +2 -0
- package/dist/client/providers/kafka.provider.js +24 -0
- package/dist/client/providers/kafka.provider.js.map +1 -0
- package/dist/client/providers/topics.service.d.ts +8 -0
- package/dist/{exceptions/payload-format-validator.exception.js → client/providers/topics.service.js} +22 -18
- package/dist/client/providers/topics.service.js.map +1 -0
- package/dist/event.type.d.ts +13 -0
- package/dist/event.type.js +6 -0
- package/dist/event.type.js.map +1 -1
- package/dist/exceptions/missing-topics.exception.d.ts +12 -0
- package/dist/exceptions/missing-topics.exception.js +21 -0
- package/dist/exceptions/missing-topics.exception.js.map +1 -0
- package/dist/exceptions/payload-format.exception.d.ts +5 -0
- package/dist/exceptions/payload-format.exception.js +11 -0
- package/dist/exceptions/payload-format.exception.js.map +1 -0
- package/dist/exceptions/unknown-kafka.exception.d.ts +4 -0
- package/dist/exceptions/unknown-kafka.exception.js +17 -0
- package/dist/exceptions/unknown-kafka.exception.js.map +1 -0
- package/dist/index.d.ts +14 -10
- package/dist/index.js +24 -19
- package/dist/index.js.map +1 -1
- package/dist/payload-format.d.ts +21 -0
- package/dist/payload-format.js +32 -1
- package/dist/payload-format.js.map +1 -1
- package/dist/{create-logger-adapter.fn.d.ts → utils/create-logger-adapter.fn.d.ts} +3 -0
- package/dist/{create-logger-adapter.fn.js → utils/create-logger-adapter.fn.js} +3 -0
- package/dist/utils/create-logger-adapter.fn.js.map +1 -0
- package/dist/utils/event.codec.d.ts +22 -0
- package/dist/utils/event.codec.js +32 -0
- package/dist/utils/event.codec.js.map +1 -0
- package/dist/utils/handle-kafka.error.fn.d.ts +1 -0
- package/dist/utils/handle-kafka.error.fn.js +15 -0
- package/dist/utils/handle-kafka.error.fn.js.map +1 -0
- package/dist/utils/retry-policy.infinite.d.ts +2 -0
- package/dist/{retry-policy.js → utils/retry-policy.infinite.js} +3 -3
- package/dist/utils/retry-policy.infinite.js.map +1 -0
- package/package.json +4 -4
- package/src/client/constants.ts +2 -0
- package/src/client/decorators/inject-kafka.decorator.ts +6 -0
- package/src/client/decorators/inject-producer.decorator.ts +6 -0
- package/src/client/kafka-client.module.ts +5 -6
- package/src/client/{kafka.healthcheck.ts → kafka.base-indicator.ts} +17 -18
- package/src/{metadata → client}/kafka.metadata.ts +12 -5
- package/src/client/providers/kafka-producer.provider.ts +23 -0
- package/src/client/providers/kafka.provider.ts +24 -0
- package/src/client/providers/topics.service.ts +23 -0
- package/src/event.type.ts +16 -0
- package/src/exceptions/missing-topics.exception.ts +19 -0
- package/src/exceptions/payload-format.exception.ts +10 -0
- package/src/exceptions/unknown-kafka.exception.ts +13 -0
- package/src/index.ts +20 -12
- package/src/payload-format.ts +38 -0
- package/src/{create-logger-adapter.fn.ts → utils/create-logger-adapter.fn.ts} +3 -0
- package/src/utils/event.codec.ts +34 -0
- package/src/utils/handle-kafka.error.fn.ts +17 -0
- package/src/utils/retry-policy.infinite.ts +8 -0
- package/tsconfig.json +0 -1
- package/dist/client/kafka.client.d.ts +0 -7
- package/dist/client/kafka.client.js +0 -43
- package/dist/client/kafka.client.js.map +0 -1
- package/dist/client/kafka.healthcheck.d.ts +0 -9
- package/dist/client/kafka.healthcheck.js.map +0 -1
- package/dist/create-logger-adapter.fn.js.map +0 -1
- package/dist/exception.d.ts +0 -32
- package/dist/exception.js +0 -61
- package/dist/exception.js.map +0 -1
- package/dist/exceptions/payload-format-validator.exception.d.ts +0 -4
- package/dist/exceptions/payload-format-validator.exception.js.map +0 -1
- package/dist/metadata/kafka-metadata.module.d.ts +0 -2
- package/dist/metadata/kafka-metadata.module.js +0 -21
- package/dist/metadata/kafka-metadata.module.js.map +0 -1
- package/dist/metadata/kafka-metadata.provider.d.ts +0 -8
- package/dist/metadata/kafka-metadata.provider.js +0 -42
- package/dist/metadata/kafka-metadata.provider.js.map +0 -1
- package/dist/metadata/kafka.metadata.js +0 -10
- package/dist/metadata/kafka.metadata.js.map +0 -1
- package/dist/retry-policy.d.ts +0 -6
- package/dist/retry-policy.js.map +0 -1
- package/dist/validators/payload-format.validator.d.ts +0 -3
- package/dist/validators/payload-format.validator.js +0 -14
- package/dist/validators/payload-format.validator.js.map +0 -1
- package/src/client/kafka.client.ts +0 -49
- package/src/exception.ts +0 -67
- package/src/exceptions/payload-format-validator.exception.ts +0 -17
- package/src/metadata/kafka-metadata.module.ts +0 -9
- package/src/metadata/kafka-metadata.provider.ts +0 -36
- package/src/retry-policy.ts +0 -6
- package/src/validators/payload-format.validator.ts +0 -18
package/dist/payload-format.js
CHANGED
|
@@ -1,11 +1,42 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.DEFAULT_PAYLOAD_FORMAT = exports.X_FORMAT_HEADER = exports.PayloadFormat = void 0;
|
|
3
|
+
exports.DEFAULT_PAYLOAD_FORMAT = exports.X_FORMAT_HEADER = exports.assertIsPayloadFormat = exports.isPayloadFormat = exports.PayloadFormat = void 0;
|
|
4
|
+
const common_1 = require("@rsdk/common");
|
|
5
|
+
const payload_format_exception_1 = require("./exceptions/payload-format.exception");
|
|
6
|
+
/**
|
|
7
|
+
* Способ сериализации данных события
|
|
8
|
+
*/
|
|
4
9
|
var PayloadFormat;
|
|
5
10
|
(function (PayloadFormat) {
|
|
6
11
|
PayloadFormat["PROTOBUF"] = "protobuf";
|
|
7
12
|
PayloadFormat["JSON"] = "json";
|
|
8
13
|
})(PayloadFormat || (exports.PayloadFormat = PayloadFormat = {}));
|
|
14
|
+
const allowed = common_1.Enum.values(PayloadFormat);
|
|
15
|
+
/**
|
|
16
|
+
* Проверяет, является ли переданная строка - одним
|
|
17
|
+
* из значений перечисления PayloadFormat
|
|
18
|
+
*/
|
|
19
|
+
const isPayloadFormat = (value) => allowed.includes(value);
|
|
20
|
+
exports.isPayloadFormat = isPayloadFormat;
|
|
21
|
+
/**
|
|
22
|
+
* Проверяет, является ли переданная строка - одним
|
|
23
|
+
* из значений перечисления PayloadFormat. В случае,
|
|
24
|
+
* если передано неверное значение - бросает ошибку
|
|
25
|
+
*/
|
|
26
|
+
const assertIsPayloadFormat = (value) => {
|
|
27
|
+
if ((0, exports.isPayloadFormat)(value)) {
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
throw new payload_format_exception_1.PayloadFormatException(value, allowed);
|
|
31
|
+
};
|
|
32
|
+
exports.assertIsPayloadFormat = assertIsPayloadFormat;
|
|
33
|
+
/**
|
|
34
|
+
* Заголовок в kafka, через который передаётся способ
|
|
35
|
+
* сериализации
|
|
36
|
+
*/
|
|
9
37
|
exports.X_FORMAT_HEADER = 'X-Format';
|
|
38
|
+
/**
|
|
39
|
+
* Сериализация по умолчанию (protobuf)
|
|
40
|
+
*/
|
|
10
41
|
exports.DEFAULT_PAYLOAD_FORMAT = PayloadFormat.PROTOBUF;
|
|
11
42
|
//# sourceMappingURL=payload-format.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"payload-format.js","sourceRoot":"","sources":["../src/payload-format.ts"],"names":[],"mappings":";;;AAAA,IAAY,aAGX;AAHD,WAAY,aAAa;IACvB,sCAAqB,CAAA;IACrB,8BAAa,CAAA;AACf,CAAC,EAHW,aAAa,6BAAb,aAAa,QAGxB;
|
|
1
|
+
{"version":3,"file":"payload-format.js","sourceRoot":"","sources":["../src/payload-format.ts"],"names":[],"mappings":";;;AAAA,yCAAoC;AAEpC,oFAA+E;AAE/E;;GAEG;AACH,IAAY,aAGX;AAHD,WAAY,aAAa;IACvB,sCAAqB,CAAA;IACrB,8BAAa,CAAA;AACf,CAAC,EAHW,aAAa,6BAAb,aAAa,QAGxB;AAED,MAAM,OAAO,GAAG,aAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;AAE3C;;;GAGG;AACI,MAAM,eAAe,GAAG,CAAC,KAAa,EAA0B,EAAE,CACvE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;AADb,QAAA,eAAe,mBACF;AAE1B;;;;GAIG;AACI,MAAM,qBAAqB,GAAG,CACnC,KAAa,EACmB,EAAE;IAClC,IAAI,IAAA,uBAAe,EAAC,KAAK,CAAC,EAAE;QAC1B,OAAO;KACR;IAED,MAAM,IAAI,iDAAsB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AACnD,CAAC,CAAC;AARW,QAAA,qBAAqB,yBAQhC;AAEF;;;GAGG;AACU,QAAA,eAAe,GAAG,UAAU,CAAC;AAE1C;;GAEG;AACU,QAAA,sBAAsB,GAAG,aAAa,CAAC,QAAQ,CAAC"}
|
|
@@ -3,6 +3,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.createLoggerAdapter = void 0;
|
|
4
4
|
const common_1 = require("@rsdk/common");
|
|
5
5
|
const kafkajs_1 = require("kafkajs");
|
|
6
|
+
/**
|
|
7
|
+
* Creates adapter for injecting rsdk logger to kafkajs
|
|
8
|
+
*/
|
|
6
9
|
const createLoggerAdapter = (logger) => (_logLevel) => (entry) => {
|
|
7
10
|
const { message, ...log } = entry.log;
|
|
8
11
|
const params = {
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"create-logger-adapter.fn.js","sourceRoot":"","sources":["../../src/utils/create-logger-adapter.fn.ts"],"names":[],"mappings":";;;AAAA,yCAA2C;AAG3C,qCAAmC;AAEnC;;GAEG;AACI,MAAM,mBAAmB,GAC9B,CAAC,MAAe,EAAc,EAAE,CAChC,CAAC,SAAmB,EAAE,EAAE,CACxB,CAAC,KAAe,EAAQ,EAAE;IACxB,MAAM,EAAE,OAAO,EAAE,GAAG,GAAG,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC;IACtC,MAAM,MAAM,GAAG;QACb,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,GAAG,GAAG;KACP,CAAC;IAEF,QAAQ,KAAK,CAAC,KAAK,EAAE;QACnB,KAAK,kBAAQ,CAAC,KAAK;YACjB,sFAAsF;YACtF,OAAO,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACvC,KAAK,kBAAQ,CAAC,KAAK;YACjB,OAAO,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACvC,KAAK,kBAAQ,CAAC,IAAI;YAChB,OAAO,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACtC,KAAK,kBAAQ,CAAC,IAAI;YAChB,OAAO,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACtC,KAAK,kBAAQ,CAAC,OAAO;YACnB,OAAO;QACT;YACE,IAAA,oBAAW,EAAC,KAAK,CAAC,KAAK,CAAC,CAAC;KAC5B;AACH,CAAC,CAAC;AAzBS,QAAA,mBAAmB,uBAyB5B"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import type { EventPayload, EventType } from '../event.type';
|
|
3
|
+
export interface EventCodec<E extends EventType> {
|
|
4
|
+
decode<P extends EventPayload<E> = EventPayload<E>>(buffer: Buffer): P;
|
|
5
|
+
encode<P extends EventPayload<E> = EventPayload<E>>(payload: P): Buffer;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Protobuf-encoding strategy
|
|
9
|
+
*/
|
|
10
|
+
export declare class ProtoEventCodec<E extends EventType> implements EventCodec<E> {
|
|
11
|
+
readonly eventType: E;
|
|
12
|
+
constructor(eventType: E);
|
|
13
|
+
decode<P extends EventPayload<E>>(buffer: Buffer): P;
|
|
14
|
+
encode<P extends EventPayload<E>>(payload: P): Buffer;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* JSON-encoding strategy
|
|
18
|
+
*/
|
|
19
|
+
export declare class JSONEventCodec<E extends EventType> implements EventCodec<E> {
|
|
20
|
+
decode<P extends EventPayload<E>>(buffer: Buffer): P;
|
|
21
|
+
encode<P extends EventPayload<E>>(payload: P): Buffer;
|
|
22
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.JSONEventCodec = exports.ProtoEventCodec = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* Protobuf-encoding strategy
|
|
6
|
+
*/
|
|
7
|
+
class ProtoEventCodec {
|
|
8
|
+
eventType;
|
|
9
|
+
constructor(eventType) {
|
|
10
|
+
this.eventType = eventType;
|
|
11
|
+
}
|
|
12
|
+
decode(buffer) {
|
|
13
|
+
return this.eventType.decode(buffer);
|
|
14
|
+
}
|
|
15
|
+
encode(payload) {
|
|
16
|
+
return Buffer.from(this.eventType.encode(payload).finish());
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
exports.ProtoEventCodec = ProtoEventCodec;
|
|
20
|
+
/**
|
|
21
|
+
* JSON-encoding strategy
|
|
22
|
+
*/
|
|
23
|
+
class JSONEventCodec {
|
|
24
|
+
decode(buffer) {
|
|
25
|
+
return JSON.parse(buffer.toString());
|
|
26
|
+
}
|
|
27
|
+
encode(payload) {
|
|
28
|
+
return Buffer.from(JSON.stringify(payload));
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
exports.JSONEventCodec = JSONEventCodec;
|
|
32
|
+
//# sourceMappingURL=event.codec.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"event.codec.js","sourceRoot":"","sources":["../../src/utils/event.codec.ts"],"names":[],"mappings":";;;AAOA;;GAEG;AACH,MAAa,eAAe;IACL;IAArB,YAAqB,SAAY;QAAZ,cAAS,GAAT,SAAS,CAAG;IAAG,CAAC;IAErC,MAAM,CAA4B,MAAc;QAC9C,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACvC,CAAC;IAED,MAAM,CAA4B,OAAU;QAC1C,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9D,CAAC;CACF;AAVD,0CAUC;AAED;;GAEG;AACH,MAAa,cAAc;IACzB,MAAM,CAA4B,MAAc;QAC9C,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IACvC,CAAC;IAED,MAAM,CAA4B,OAAU;QAC1C,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9C,CAAC;CACF;AARD,wCAQC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const handleKafkaError: (error: unknown, pattern: string, details?: unknown) => never;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.handleKafkaError = void 0;
|
|
4
|
+
const kafkajs_1 = require("kafkajs");
|
|
5
|
+
const missing_topics_exception_1 = require("../exceptions/missing-topics.exception");
|
|
6
|
+
const unknown_kafka_exception_1 = require("../exceptions/unknown-kafka.exception");
|
|
7
|
+
const handleKafkaError = (error, pattern, details) => {
|
|
8
|
+
// https://github.com/tulios/kafkajs/blob/master/src/protocol/error.js#L26
|
|
9
|
+
if (!(error instanceof kafkajs_1.KafkaJSProtocolError) || error.code !== 3) {
|
|
10
|
+
throw new unknown_kafka_exception_1.UnknownKafkaException(error, pattern, details);
|
|
11
|
+
}
|
|
12
|
+
throw new missing_topics_exception_1.MissingTopicsException([pattern], { cause: error, details });
|
|
13
|
+
};
|
|
14
|
+
exports.handleKafkaError = handleKafkaError;
|
|
15
|
+
//# sourceMappingURL=handle-kafka.error.fn.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"handle-kafka.error.fn.js","sourceRoot":"","sources":["../../src/utils/handle-kafka.error.fn.ts"],"names":[],"mappings":";;;AAAA,qCAA+C;AAE/C,qFAAgF;AAChF,mFAA8E;AAEvE,MAAM,gBAAgB,GAAG,CAC9B,KAAc,EACd,OAAe,EACf,OAAiB,EACV,EAAE;IACT,0EAA0E;IAC1E,IAAI,CAAC,CAAC,KAAK,YAAY,8BAAoB,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC,EAAE;QAChE,MAAM,IAAI,+CAAqB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;KAC1D;IAED,MAAM,IAAI,iDAAsB,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;AACzE,CAAC,CAAC;AAXW,QAAA,gBAAgB,oBAW3B"}
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
4
|
-
exports.
|
|
3
|
+
exports.INFINITE_RETRIES = void 0;
|
|
4
|
+
exports.INFINITE_RETRIES = {
|
|
5
5
|
maxRetryTime: 1e3,
|
|
6
6
|
retries: Number.POSITIVE_INFINITY,
|
|
7
7
|
factor: 0,
|
|
8
8
|
multiplier: 10,
|
|
9
9
|
};
|
|
10
|
-
//# sourceMappingURL=retry-policy.js.map
|
|
10
|
+
//# sourceMappingURL=retry-policy.infinite.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"retry-policy.infinite.js","sourceRoot":"","sources":["../../src/utils/retry-policy.infinite.ts"],"names":[],"mappings":";;;AAEa,QAAA,gBAAgB,GAAiB;IAC5C,YAAY,EAAE,GAAG;IACjB,OAAO,EAAE,MAAM,CAAC,iBAAiB;IACjC,MAAM,EAAE,CAAC;IACT,UAAU,EAAE,EAAE;CACf,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rsdk/kafka.common",
|
|
3
|
-
"version": "4.0.0-next.
|
|
3
|
+
"version": "4.0.0-next.8",
|
|
4
4
|
"description": "Common functionality for kafka consumers and producers",
|
|
5
5
|
"license": "Apache License 2.0",
|
|
6
6
|
"publishConfig": {
|
|
@@ -15,9 +15,9 @@
|
|
|
15
15
|
"@nestjs/core": "^10.0.0",
|
|
16
16
|
"@nestjs/microservices": "^10.0.0",
|
|
17
17
|
"@rsdk/common": "^4.0.0-next.6",
|
|
18
|
-
"@rsdk/core": "^4.0.0-next.
|
|
18
|
+
"@rsdk/core": "^4.0.0-next.8",
|
|
19
19
|
"@rsdk/logging": "^4.0.0-next.6",
|
|
20
|
-
"@rsdk/metadata": "^4.0.0-next.
|
|
20
|
+
"@rsdk/metadata": "^4.0.0-next.8",
|
|
21
21
|
"kafkajs": "^2.2.4",
|
|
22
22
|
"lodash": "^4.17.21",
|
|
23
23
|
"reflect-metadata": "^0.1.13",
|
|
@@ -26,5 +26,5 @@
|
|
|
26
26
|
"dependencies": {
|
|
27
27
|
"protobufjs": "^7.2.3"
|
|
28
28
|
},
|
|
29
|
-
"gitHead": "
|
|
29
|
+
"gitHead": "bae7c74fc8be068b8bd66b403d19e2b32cf092ec"
|
|
30
30
|
}
|
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
import { Global, Module } from '@nestjs/common';
|
|
2
2
|
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
import {
|
|
3
|
+
import { kafkaProvider } from './providers/kafka.provider';
|
|
4
|
+
import { kafkaProducerProvider } from './providers/kafka-producer.provider';
|
|
5
|
+
import { TopicsService } from './providers/topics.service';
|
|
6
6
|
|
|
7
7
|
@Global()
|
|
8
8
|
@Module({
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
exports: [KafkaClient, KafkaProducer, KafkaMetadataModule],
|
|
9
|
+
providers: [kafkaProducerProvider, kafkaProvider, TopicsService],
|
|
10
|
+
exports: [kafkaProducerProvider, kafkaProvider, TopicsService],
|
|
12
11
|
})
|
|
13
12
|
export class KafkaClientModule {}
|
|
@@ -1,27 +1,13 @@
|
|
|
1
|
+
import type { HealthIndicator } from '@rsdk/core';
|
|
1
2
|
import { CheckResult } from '@rsdk/core';
|
|
2
3
|
import type { Kafka } from 'kafkajs';
|
|
3
4
|
|
|
4
|
-
export class
|
|
5
|
+
export class KafkaBaseIndicator implements HealthIndicator {
|
|
5
6
|
constructor(
|
|
6
|
-
readonly
|
|
7
|
-
readonly
|
|
7
|
+
readonly client: Kafka,
|
|
8
|
+
readonly topics: string[],
|
|
8
9
|
) {}
|
|
9
10
|
|
|
10
|
-
async getMissingTopics(): Promise<string[]> {
|
|
11
|
-
const admin = this.kafkaClient.admin({
|
|
12
|
-
retry: { retries: 1 },
|
|
13
|
-
});
|
|
14
|
-
|
|
15
|
-
await admin.connect();
|
|
16
|
-
|
|
17
|
-
const topics = await admin.listTopics();
|
|
18
|
-
|
|
19
|
-
await admin.disconnect();
|
|
20
|
-
return this.neededTopics.filter(
|
|
21
|
-
(neededTopic) => !topics.includes(neededTopic),
|
|
22
|
-
);
|
|
23
|
-
}
|
|
24
|
-
|
|
25
11
|
async check(): Promise<CheckResult> {
|
|
26
12
|
try {
|
|
27
13
|
const missingTopics = await this.getMissingTopics();
|
|
@@ -37,4 +23,17 @@ export class KafkaHealthcheck {
|
|
|
37
23
|
});
|
|
38
24
|
}
|
|
39
25
|
}
|
|
26
|
+
|
|
27
|
+
private async getMissingTopics(): Promise<string[]> {
|
|
28
|
+
const admin = this.client.admin({
|
|
29
|
+
retry: { retries: 1 },
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
await admin.connect();
|
|
33
|
+
|
|
34
|
+
const topics = await admin.listTopics();
|
|
35
|
+
|
|
36
|
+
await admin.disconnect();
|
|
37
|
+
return this.topics.filter((neededTopic) => !topics.includes(neededTopic));
|
|
38
|
+
}
|
|
40
39
|
}
|
|
@@ -1,21 +1,28 @@
|
|
|
1
1
|
import type { EventType } from '../event.type';
|
|
2
2
|
|
|
3
|
-
export enum
|
|
4
|
-
|
|
5
|
-
|
|
3
|
+
export enum TopicType {
|
|
4
|
+
CONSUME,
|
|
5
|
+
PRODUCE,
|
|
6
6
|
}
|
|
7
7
|
|
|
8
|
+
/**
|
|
9
|
+
* TODO: Здесь надо разбираться с неймингом. Потому что хрен
|
|
10
|
+
* поймёшь сейчас, что это за типы
|
|
11
|
+
*/
|
|
12
|
+
|
|
8
13
|
export type KafkaTopicMetadataByTopic = {
|
|
9
|
-
type:
|
|
14
|
+
type: TopicType;
|
|
10
15
|
topicName: string;
|
|
11
16
|
};
|
|
17
|
+
|
|
12
18
|
export type KafkaTopicMetadataByEventType = {
|
|
13
|
-
type:
|
|
19
|
+
type: TopicType;
|
|
14
20
|
topicName: string;
|
|
15
21
|
group: string;
|
|
16
22
|
partitionKeyField: string | null;
|
|
17
23
|
eventType?: Pick<EventType, '$type' | '$group' | '$partitionKeyField'>;
|
|
18
24
|
};
|
|
25
|
+
|
|
19
26
|
export type KafkaTopicMetadata =
|
|
20
27
|
| KafkaTopicMetadataByEventType
|
|
21
28
|
| KafkaTopicMetadataByTopic;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { FactoryProvider, OnModuleDestroy } from '@nestjs/common';
|
|
2
|
+
import type { Kafka, Producer } from 'kafkajs';
|
|
3
|
+
import { Partitioners } from 'kafkajs';
|
|
4
|
+
|
|
5
|
+
import {
|
|
6
|
+
KAFKA_CLIENT_INJECTION_TOKEN,
|
|
7
|
+
KAFKA_PRODUCER_INJECTION_TOKEN,
|
|
8
|
+
} from '../constants';
|
|
9
|
+
|
|
10
|
+
export const kafkaProducerProvider: FactoryProvider = {
|
|
11
|
+
inject: [KAFKA_CLIENT_INJECTION_TOKEN],
|
|
12
|
+
provide: KAFKA_PRODUCER_INJECTION_TOKEN,
|
|
13
|
+
async useFactory(kafka: Kafka): Promise<Producer & OnModuleDestroy> {
|
|
14
|
+
const producer = kafka.producer({
|
|
15
|
+
allowAutoTopicCreation: false,
|
|
16
|
+
createPartitioner: Partitioners.DefaultPartitioner,
|
|
17
|
+
retry: { retries: 1 },
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
await producer.connect();
|
|
21
|
+
return { ...producer, onModuleDestroy: () => producer.disconnect() };
|
|
22
|
+
},
|
|
23
|
+
};
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { FactoryProvider } from '@nestjs/common';
|
|
2
|
+
import { APP_NAME } from '@rsdk/core';
|
|
3
|
+
import { LoggerFactory } from '@rsdk/logging';
|
|
4
|
+
import { Kafka } from 'kafkajs';
|
|
5
|
+
|
|
6
|
+
import { createLoggerAdapter } from '../../utils/create-logger-adapter.fn';
|
|
7
|
+
import { INFINITE_RETRIES } from '../../utils/retry-policy.infinite';
|
|
8
|
+
import { KAFKA_CLIENT_INJECTION_TOKEN } from '../constants';
|
|
9
|
+
import { KafkaConfig } from '../kafka.config';
|
|
10
|
+
|
|
11
|
+
export const kafkaProvider: FactoryProvider = {
|
|
12
|
+
inject: [KafkaConfig, APP_NAME],
|
|
13
|
+
provide: KAFKA_CLIENT_INJECTION_TOKEN,
|
|
14
|
+
useFactory(config: KafkaConfig, name: string): Kafka {
|
|
15
|
+
const logger = LoggerFactory.create('NativeKafkaProducer');
|
|
16
|
+
|
|
17
|
+
return new Kafka({
|
|
18
|
+
clientId: `${name}-${process.pid}`,
|
|
19
|
+
brokers: config.brokers,
|
|
20
|
+
logCreator: createLoggerAdapter(logger),
|
|
21
|
+
retry: INFINITE_RETRIES,
|
|
22
|
+
});
|
|
23
|
+
},
|
|
24
|
+
};
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { Injectable } from '@nestjs/common';
|
|
2
|
+
import type { Resource } from '@rsdk/metadata';
|
|
3
|
+
import { RsdkMetadataProvider } from '@rsdk/metadata';
|
|
4
|
+
|
|
5
|
+
import type { KafkaTopicMetadata, TopicType } from '../kafka.metadata';
|
|
6
|
+
import { KAFKA_TOPIC_RSDK_METADATA_SCOPE } from '../kafka.metadata';
|
|
7
|
+
|
|
8
|
+
@Injectable()
|
|
9
|
+
export class TopicsService {
|
|
10
|
+
constructor(private readonly provider: RsdkMetadataProvider) {}
|
|
11
|
+
|
|
12
|
+
getTopics(type: TopicType): string[] {
|
|
13
|
+
return this.getResources()
|
|
14
|
+
.filter(({ value }) => value.type === type)
|
|
15
|
+
.map(({ value }) => value.topicName);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
private getResources(): Resource<KafkaTopicMetadata>[] {
|
|
19
|
+
return this.provider.get<KafkaTopicMetadata>(
|
|
20
|
+
KAFKA_TOPIC_RSDK_METADATA_SCOPE,
|
|
21
|
+
);
|
|
22
|
+
}
|
|
23
|
+
}
|
package/src/event.type.ts
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import type { Reader, Writer } from 'protobufjs/minimal';
|
|
2
2
|
|
|
3
|
+
/**
|
|
4
|
+
* Генерируется при компиляции proto-файлов
|
|
5
|
+
*/
|
|
3
6
|
export interface EventType<T = any> {
|
|
4
7
|
$group: string;
|
|
5
8
|
$partitionKeyField: string | null;
|
|
@@ -8,11 +11,24 @@ export interface EventType<T = any> {
|
|
|
8
11
|
decode(data: Uint8Array | Reader): T;
|
|
9
12
|
}
|
|
10
13
|
|
|
14
|
+
/**
|
|
15
|
+
* Выводит тип payload из EventType
|
|
16
|
+
*/
|
|
17
|
+
export type EventPayload<E extends EventType> = E extends EventType<infer P>
|
|
18
|
+
? P
|
|
19
|
+
: never;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Проверяет, является ли переданный тип EventType
|
|
23
|
+
*/
|
|
11
24
|
export const isEventType = <T = any>(x: any): x is EventType<T> =>
|
|
12
25
|
typeof x === 'object' &&
|
|
13
26
|
typeof x.$type === 'string' &&
|
|
14
27
|
typeof x.$group === 'string';
|
|
15
28
|
|
|
29
|
+
/**
|
|
30
|
+
* Генерирует имя топика из EventType
|
|
31
|
+
*/
|
|
16
32
|
export const getTopicName = (eventType: EventType): string => {
|
|
17
33
|
return `events.${eventType.$group}`;
|
|
18
34
|
};
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { InternalException } from '@rsdk/core';
|
|
2
|
+
|
|
3
|
+
export class MissingTopicsException extends InternalException {
|
|
4
|
+
constructor(
|
|
5
|
+
readonly topics: string[],
|
|
6
|
+
readonly props?: {
|
|
7
|
+
details?: unknown;
|
|
8
|
+
cause?: unknown;
|
|
9
|
+
},
|
|
10
|
+
) {
|
|
11
|
+
super('Expected topics or partitions are missing!', {
|
|
12
|
+
details: {
|
|
13
|
+
topics,
|
|
14
|
+
details: props?.details,
|
|
15
|
+
},
|
|
16
|
+
cause: props?.cause,
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { EnumValue } from '@rsdk/common';
|
|
2
|
+
import { BootstrapException } from '@rsdk/core';
|
|
3
|
+
|
|
4
|
+
export class PayloadFormatException extends BootstrapException {
|
|
5
|
+
constructor(value: string, allowed: EnumValue[]) {
|
|
6
|
+
super(
|
|
7
|
+
`PayloadFormat has wrong value: ${value}, allowed: ${allowed.join(',')}`,
|
|
8
|
+
);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { InternalException } from '@rsdk/core';
|
|
2
|
+
|
|
3
|
+
export class UnknownKafkaException extends InternalException {
|
|
4
|
+
constructor(cause: unknown, pattern: string, details?: unknown) {
|
|
5
|
+
super('Unknown exception from ServerKafka!', {
|
|
6
|
+
details: {
|
|
7
|
+
pattern,
|
|
8
|
+
details: details,
|
|
9
|
+
},
|
|
10
|
+
cause,
|
|
11
|
+
});
|
|
12
|
+
}
|
|
13
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -1,31 +1,39 @@
|
|
|
1
|
-
export {
|
|
1
|
+
export { KafkaTopicMetadataByEventType } from './client/kafka.metadata';
|
|
2
2
|
|
|
3
|
+
export { InjectKafka } from './client/decorators/inject-kafka.decorator';
|
|
4
|
+
export { InjectKafkaProducer } from './client/decorators/inject-producer.decorator';
|
|
3
5
|
export {
|
|
4
6
|
KAFKA_PRODUCER_INJECTION_TOKEN,
|
|
5
7
|
KAFKA_CLIENT_INJECTION_TOKEN,
|
|
6
|
-
} from './client/
|
|
8
|
+
} from './client/constants';
|
|
9
|
+
export { INFINITE_RETRIES } from './utils/retry-policy.infinite';
|
|
10
|
+
export { EventPayload } from './event.type';
|
|
11
|
+
|
|
12
|
+
export {
|
|
13
|
+
ProtoEventCodec,
|
|
14
|
+
EventCodec,
|
|
15
|
+
JSONEventCodec,
|
|
16
|
+
} from './utils/event.codec';
|
|
7
17
|
|
|
8
18
|
export { KafkaClientModule } from './client/kafka-client.module';
|
|
9
19
|
|
|
10
20
|
export { KafkaConfig } from './client/kafka.config';
|
|
11
21
|
|
|
12
|
-
export {
|
|
13
|
-
|
|
14
|
-
export { KafkaMetadataProvider } from './metadata/kafka-metadata.provider';
|
|
22
|
+
export { TopicsService } from './client/providers/topics.service';
|
|
15
23
|
export {
|
|
16
24
|
KAFKA_TOPIC_RSDK_METADATA_SCOPE,
|
|
17
25
|
KafkaTopicMetadata,
|
|
18
|
-
|
|
19
|
-
} from './
|
|
26
|
+
TopicType,
|
|
27
|
+
} from './client/kafka.metadata';
|
|
20
28
|
|
|
21
|
-
export {
|
|
29
|
+
export { handleKafkaError } from './utils/handle-kafka.error.fn';
|
|
22
30
|
|
|
23
|
-
export { createLoggerAdapter } from './create-logger-adapter.fn';
|
|
31
|
+
export { createLoggerAdapter } from './utils/create-logger-adapter.fn';
|
|
24
32
|
|
|
25
33
|
export { getTopicName, isEventType, EventType } from './event.type';
|
|
26
34
|
|
|
27
|
-
export {
|
|
35
|
+
export { KafkaBaseIndicator } from './client/kafka.base-indicator';
|
|
28
36
|
|
|
29
37
|
export * from './payload-format';
|
|
30
|
-
export * from './exceptions/payload-format
|
|
31
|
-
export * from './
|
|
38
|
+
export * from './exceptions/payload-format.exception';
|
|
39
|
+
export * from './exceptions/missing-topics.exception';
|
package/src/payload-format.ts
CHANGED
|
@@ -1,8 +1,46 @@
|
|
|
1
|
+
import { Enum } from '@rsdk/common';
|
|
2
|
+
|
|
3
|
+
import { PayloadFormatException } from './exceptions/payload-format.exception';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Способ сериализации данных события
|
|
7
|
+
*/
|
|
1
8
|
export enum PayloadFormat {
|
|
2
9
|
PROTOBUF = 'protobuf',
|
|
3
10
|
JSON = 'json',
|
|
4
11
|
}
|
|
5
12
|
|
|
13
|
+
const allowed = Enum.values(PayloadFormat);
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Проверяет, является ли переданная строка - одним
|
|
17
|
+
* из значений перечисления PayloadFormat
|
|
18
|
+
*/
|
|
19
|
+
export const isPayloadFormat = (value: string): value is PayloadFormat =>
|
|
20
|
+
allowed.includes(value);
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Проверяет, является ли переданная строка - одним
|
|
24
|
+
* из значений перечисления PayloadFormat. В случае,
|
|
25
|
+
* если передано неверное значение - бросает ошибку
|
|
26
|
+
*/
|
|
27
|
+
export const assertIsPayloadFormat = (
|
|
28
|
+
value: string,
|
|
29
|
+
): asserts value is PayloadFormat => {
|
|
30
|
+
if (isPayloadFormat(value)) {
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
throw new PayloadFormatException(value, allowed);
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Заголовок в kafka, через который передаётся способ
|
|
39
|
+
* сериализации
|
|
40
|
+
*/
|
|
6
41
|
export const X_FORMAT_HEADER = 'X-Format';
|
|
7
42
|
|
|
43
|
+
/**
|
|
44
|
+
* Сериализация по умолчанию (protobuf)
|
|
45
|
+
*/
|
|
8
46
|
export const DEFAULT_PAYLOAD_FORMAT = PayloadFormat.PROTOBUF;
|
|
@@ -3,6 +3,9 @@ import type { ILogger } from '@rsdk/logging';
|
|
|
3
3
|
import type { logCreator, LogEntry } from 'kafkajs';
|
|
4
4
|
import { logLevel } from 'kafkajs';
|
|
5
5
|
|
|
6
|
+
/**
|
|
7
|
+
* Creates adapter for injecting rsdk logger to kafkajs
|
|
8
|
+
*/
|
|
6
9
|
export const createLoggerAdapter =
|
|
7
10
|
(logger: ILogger): logCreator =>
|
|
8
11
|
(_logLevel: logLevel) =>
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { EventPayload, EventType } from '../event.type';
|
|
2
|
+
|
|
3
|
+
export interface EventCodec<E extends EventType> {
|
|
4
|
+
decode<P extends EventPayload<E> = EventPayload<E>>(buffer: Buffer): P;
|
|
5
|
+
encode<P extends EventPayload<E> = EventPayload<E>>(payload: P): Buffer;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Protobuf-encoding strategy
|
|
10
|
+
*/
|
|
11
|
+
export class ProtoEventCodec<E extends EventType> implements EventCodec<E> {
|
|
12
|
+
constructor(readonly eventType: E) {}
|
|
13
|
+
|
|
14
|
+
decode<P extends EventPayload<E>>(buffer: Buffer): P {
|
|
15
|
+
return this.eventType.decode(buffer);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
encode<P extends EventPayload<E>>(payload: P): Buffer {
|
|
19
|
+
return Buffer.from(this.eventType.encode(payload).finish());
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* JSON-encoding strategy
|
|
25
|
+
*/
|
|
26
|
+
export class JSONEventCodec<E extends EventType> implements EventCodec<E> {
|
|
27
|
+
decode<P extends EventPayload<E>>(buffer: Buffer): P {
|
|
28
|
+
return JSON.parse(buffer.toString());
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
encode<P extends EventPayload<E>>(payload: P): Buffer {
|
|
32
|
+
return Buffer.from(JSON.stringify(payload));
|
|
33
|
+
}
|
|
34
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { KafkaJSProtocolError } from 'kafkajs';
|
|
2
|
+
|
|
3
|
+
import { MissingTopicsException } from '../exceptions/missing-topics.exception';
|
|
4
|
+
import { UnknownKafkaException } from '../exceptions/unknown-kafka.exception';
|
|
5
|
+
|
|
6
|
+
export const handleKafkaError = (
|
|
7
|
+
error: unknown,
|
|
8
|
+
pattern: string,
|
|
9
|
+
details?: unknown,
|
|
10
|
+
): never => {
|
|
11
|
+
// https://github.com/tulios/kafkajs/blob/master/src/protocol/error.js#L26
|
|
12
|
+
if (!(error instanceof KafkaJSProtocolError) || error.code !== 3) {
|
|
13
|
+
throw new UnknownKafkaException(error, pattern, details);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
throw new MissingTopicsException([pattern], { cause: error, details });
|
|
17
|
+
};
|