@rsdk/kafka.producer 4.0.0-next.7 → 4.0.0-next.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/constants.d.ts +1 -0
- package/dist/constants.js +5 -0
- package/dist/constants.js.map +1 -0
- package/dist/events-factory.abstract.d.ts +17 -0
- package/dist/events-factory.abstract.js +63 -0
- package/dist/events-factory.abstract.js.map +1 -0
- package/dist/index.d.ts +4 -4
- package/dist/index.js +7 -7
- package/dist/index.js.map +1 -1
- package/dist/interfaces.d.ts +27 -0
- package/dist/{types.js → interfaces.js} +1 -1
- package/dist/interfaces.js.map +1 -0
- package/dist/producer.config.d.ts +6 -0
- package/dist/{kafka-producer-plugin.config.js → producer.config.js} +7 -7
- package/dist/producer.config.js.map +1 -0
- package/package.json +4 -4
- package/src/constants.ts +1 -0
- package/src/events-factory.abstract.ts +90 -0
- package/src/index.ts +13 -4
- package/src/interfaces.ts +42 -0
- package/src/{kafka-producer-plugin.config.ts → producer.config.ts} +3 -1
- package/tsconfig.json +1 -2
- package/dist/base.producer.d.ts +0 -14
- package/dist/base.producer.js +0 -52
- package/dist/base.producer.js.map +0 -1
- package/dist/inject-producer.decorator.d.ts +0 -3
- package/dist/inject-producer.decorator.js +0 -7
- package/dist/inject-producer.decorator.js.map +0 -1
- package/dist/kafka-producer-plugin.config.d.ts +0 -5
- package/dist/kafka-producer-plugin.config.js.map +0 -1
- package/dist/types.d.ts +0 -17
- package/dist/types.js.map +0 -1
- package/src/base.producer.ts +0 -79
- package/src/inject-producer.decorator.ts +0 -6
- package/src/types.ts +0 -22
package/CHANGELOG.md
CHANGED
|
@@ -3,6 +3,12 @@
|
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
|
4
4
|
See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
|
|
5
5
|
|
|
6
|
+
## [4.0.0-next.8](https://github.com/R-Vision/rsdk/compare/v4.0.0-next.7...v4.0.0-next.8) (2023-11-29)
|
|
7
|
+
|
|
8
|
+
### Features
|
|
9
|
+
|
|
10
|
+
* Kafka batch producer ([#150](https://github.com/R-Vision/rsdk/issues/150)) ([2faa2f4](https://github.com/R-Vision/rsdk/commit/2faa2f41ca0dbb6d8a92cbf0ab20cf5cefeaf4e9))
|
|
11
|
+
|
|
6
12
|
## [4.0.0-next.7](https://github.com/R-Vision/rsdk/compare/v4.0.0-next.6...v4.0.0-next.7) (2023-11-21)
|
|
7
13
|
|
|
8
14
|
**Note:** Version bump only for package @rsdk/kafka.producer
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const EVENT_TYPE_TOKEN: unique symbol;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":";;;AAAa,QAAA,gBAAgB,GAAG,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import type { EventPayload, EventType } from '@rsdk/kafka.common';
|
|
3
|
+
import type { EventMetadata, ProducerOptions, PublishOptions } from './interfaces';
|
|
4
|
+
/**
|
|
5
|
+
* Абстрактный класс, содержащий полезные методы для подготовки
|
|
6
|
+
* данных, как для непосредственной отправки в kafka посредством
|
|
7
|
+
* kafkajs, так и для сохранения в outbox
|
|
8
|
+
*/
|
|
9
|
+
export declare abstract class EventsFactory<T extends EventType, K extends object> {
|
|
10
|
+
protected readonly eventType: T;
|
|
11
|
+
private readonly payloadFormat;
|
|
12
|
+
constructor(eventType: T, config: ProducerOptions);
|
|
13
|
+
protected getMetadata(metadata: EventMetadata | undefined | null): Record<string, string>;
|
|
14
|
+
protected serializePayload<P extends EventPayload<T>>(payload: P): Buffer;
|
|
15
|
+
protected getPartitionKey<P extends Record<string, any>>(payload: P, options?: PublishOptions): string;
|
|
16
|
+
abstract create(payload: EventPayload<T>, options?: PublishOptions): K;
|
|
17
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.EventsFactory = void 0;
|
|
4
|
+
const kafka_common_1 = require("@rsdk/kafka.common");
|
|
5
|
+
const lodash_1 = require("lodash");
|
|
6
|
+
const node_crypto_1 = require("node:crypto");
|
|
7
|
+
/**
|
|
8
|
+
* Абстрактный класс, содержащий полезные методы для подготовки
|
|
9
|
+
* данных, как для непосредственной отправки в kafka посредством
|
|
10
|
+
* kafkajs, так и для сохранения в outbox
|
|
11
|
+
*/
|
|
12
|
+
class EventsFactory {
|
|
13
|
+
eventType;
|
|
14
|
+
payloadFormat;
|
|
15
|
+
constructor(eventType, config) {
|
|
16
|
+
this.eventType = eventType;
|
|
17
|
+
this.payloadFormat = config.payloadFormat ?? kafka_common_1.DEFAULT_PAYLOAD_FORMAT;
|
|
18
|
+
}
|
|
19
|
+
getMetadata(metadata) {
|
|
20
|
+
return {
|
|
21
|
+
...metadata,
|
|
22
|
+
...(this.payloadFormat &&
|
|
23
|
+
this.payloadFormat !== kafka_common_1.DEFAULT_PAYLOAD_FORMAT && {
|
|
24
|
+
[kafka_common_1.X_FORMAT_HEADER]: this.payloadFormat,
|
|
25
|
+
}),
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
serializePayload(payload) {
|
|
29
|
+
const codec = this.payloadFormat === kafka_common_1.PayloadFormat.JSON
|
|
30
|
+
? new kafka_common_1.JSONEventCodec()
|
|
31
|
+
: new kafka_common_1.ProtoEventCodec(this.eventType);
|
|
32
|
+
return codec.encode(payload);
|
|
33
|
+
}
|
|
34
|
+
getPartitionKey(payload, options) {
|
|
35
|
+
/**
|
|
36
|
+
* Если ключ партиционирования передаётся
|
|
37
|
+
* в опциях явно - берём его
|
|
38
|
+
*/
|
|
39
|
+
if (options?.partitionKey) {
|
|
40
|
+
return String(options.partitionKey);
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Если в типе события определено имя ключевого поля,
|
|
44
|
+
* извлекаем его значение и используем
|
|
45
|
+
*/
|
|
46
|
+
const { $partitionKeyField } = this.eventType;
|
|
47
|
+
if ($partitionKeyField && typeof $partitionKeyField === 'string') {
|
|
48
|
+
// TODO: Может быть лучше ошибку кидать?
|
|
49
|
+
return (0, lodash_1.get)(payload, $partitionKeyField) ?? (0, node_crypto_1.randomUUID)();
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Ищем последовательно поля id, uuid, guid,
|
|
53
|
+
* если ничего не найдено, используем генерируем
|
|
54
|
+
* ключ случайным образом
|
|
55
|
+
*/
|
|
56
|
+
return ((0, lodash_1.get)(payload, 'id') ??
|
|
57
|
+
(0, lodash_1.get)(payload, 'uuid') ??
|
|
58
|
+
(0, lodash_1.get)(payload, 'guid') ??
|
|
59
|
+
(0, node_crypto_1.randomUUID)());
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
exports.EventsFactory = EventsFactory;
|
|
63
|
+
//# sourceMappingURL=events-factory.abstract.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"events-factory.abstract.js","sourceRoot":"","sources":["../src/events-factory.abstract.ts"],"names":[],"mappings":";;;AACA,qDAM4B;AAC5B,mCAA6B;AAC7B,6CAAyC;AAQzC;;;;GAIG;AACH,MAAsB,aAAa;IAIZ;IAHJ,aAAa,CAAgB;IAE9C,YACqB,SAAY,EAC/B,MAAuB;QADJ,cAAS,GAAT,SAAS,CAAG;QAG/B,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,qCAAsB,CAAC;IACtE,CAAC;IAES,WAAW,CACnB,QAA0C;QAE1C,OAAO;YACL,GAAG,QAAQ;YACX,GAAG,CAAC,IAAI,CAAC,aAAa;gBACpB,IAAI,CAAC,aAAa,KAAK,qCAAsB,IAAI;gBAC/C,CAAC,8BAAe,CAAC,EAAE,IAAI,CAAC,aAAa;aACtC,CAAC;SACL,CAAC;IACJ,CAAC;IAES,gBAAgB,CAA4B,OAAU;QAC9D,MAAM,KAAK,GACT,IAAI,CAAC,aAAa,KAAK,4BAAa,CAAC,IAAI;YACvC,CAAC,CAAC,IAAI,6BAAc,EAAK;YACzB,CAAC,CAAC,IAAI,8BAAe,CAAI,IAAI,CAAC,SAAS,CAAC,CAAC;QAE7C,OAAO,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAES,eAAe,CACvB,OAAU,EACV,OAAwB;QAExB;;;WAGG;QACH,IAAI,OAAO,EAAE,YAAY,EAAE;YACzB,OAAO,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;SACrC;QAED;;;WAGG;QACH,MAAM,EAAE,kBAAkB,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC;QAC9C,IAAI,kBAAkB,IAAI,OAAO,kBAAkB,KAAK,QAAQ,EAAE;YAChE,wCAAwC;YACxC,OAAO,IAAA,YAAG,EAAC,OAAO,EAAE,kBAAkB,CAAC,IAAI,IAAA,wBAAU,GAAE,CAAC;SACzD;QAED;;;;WAIG;QACH,OAAO,CACL,IAAA,YAAG,EAAC,OAAO,EAAE,IAAI,CAAC;YAClB,IAAA,YAAG,EAAC,OAAO,EAAE,MAAM,CAAC;YACpB,IAAA,YAAG,EAAC,OAAO,EAAE,MAAM,CAAC;YACpB,IAAA,wBAAU,GAAE,CACb,CAAC;IACJ,CAAC;CAGF;AAnED,sCAmEC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export {
|
|
2
|
-
export {
|
|
3
|
-
export {
|
|
4
|
-
export {
|
|
1
|
+
export { EventsFactory } from './events-factory.abstract';
|
|
2
|
+
export { EVENT_TYPE_TOKEN } from './constants';
|
|
3
|
+
export { KafkaProducer, BatchProducerItem, KafkaBatchProducer, PublishOptions, ProducerOptions, EventMetadata, } from './interfaces';
|
|
4
|
+
export { ProducerConfig } from './producer.config';
|
package/dist/index.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
4
|
-
var
|
|
5
|
-
Object.defineProperty(exports, "
|
|
6
|
-
var
|
|
7
|
-
Object.defineProperty(exports, "
|
|
8
|
-
var
|
|
9
|
-
Object.defineProperty(exports, "
|
|
3
|
+
exports.ProducerConfig = exports.EVENT_TYPE_TOKEN = exports.EventsFactory = void 0;
|
|
4
|
+
var events_factory_abstract_1 = require("./events-factory.abstract");
|
|
5
|
+
Object.defineProperty(exports, "EventsFactory", { enumerable: true, get: function () { return events_factory_abstract_1.EventsFactory; } });
|
|
6
|
+
var constants_1 = require("./constants");
|
|
7
|
+
Object.defineProperty(exports, "EVENT_TYPE_TOKEN", { enumerable: true, get: function () { return constants_1.EVENT_TYPE_TOKEN; } });
|
|
8
|
+
var producer_config_1 = require("./producer.config");
|
|
9
|
+
Object.defineProperty(exports, "ProducerConfig", { enumerable: true, get: function () { return producer_config_1.ProducerConfig; } });
|
|
10
10
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,qEAA0D;AAAjD,wHAAA,aAAa,OAAA;AAEtB,yCAA+C;AAAtC,6GAAA,gBAAgB,OAAA;AAUzB,qDAAmD;AAA1C,iHAAA,cAAc,OAAA"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { MaybeReadonlyArray } from '@rsdk/common.node';
|
|
2
|
+
import type { EventPayload, EventType, PayloadFormat } from '@rsdk/kafka.common';
|
|
3
|
+
import type { TupleToUnion } from 'type-fest';
|
|
4
|
+
export interface ProducerOptions {
|
|
5
|
+
payloadFormat?: PayloadFormat;
|
|
6
|
+
}
|
|
7
|
+
export type EventMetadata = Record<string, unknown>;
|
|
8
|
+
export interface PublishOptions {
|
|
9
|
+
/** Metadata must be a plain object. Object will be stringified */
|
|
10
|
+
metadata?: EventMetadata;
|
|
11
|
+
/** Override partition key */
|
|
12
|
+
partitionKey?: string | number;
|
|
13
|
+
}
|
|
14
|
+
export interface BatchProducerItem<E extends EventType> {
|
|
15
|
+
eventType: E;
|
|
16
|
+
payload: EventPayload<E>;
|
|
17
|
+
options?: PublishOptions;
|
|
18
|
+
}
|
|
19
|
+
export interface RawKafkaBatchProducer {
|
|
20
|
+
publish<E extends EventType>(items: BatchProducerItem<E>[]): Promise<void>;
|
|
21
|
+
}
|
|
22
|
+
export interface KafkaBatchProducer<E extends MaybeReadonlyArray<EventType>> {
|
|
23
|
+
publish(items: BatchProducerItem<TupleToUnion<E>>[]): Promise<void>;
|
|
24
|
+
}
|
|
25
|
+
export interface KafkaProducer<E extends EventType> {
|
|
26
|
+
publish<P extends EventPayload<E>>(payload: P, options?: PublishOptions): Promise<void>;
|
|
27
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"interfaces.js","sourceRoot":"","sources":["../src/interfaces.ts"],"names":[],"mappings":""}
|
|
@@ -9,24 +9,24 @@ var __metadata = (this && this.__metadata) || function (k, v) {
|
|
|
9
9
|
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
10
|
};
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
exports.
|
|
12
|
+
exports.ProducerConfig = void 0;
|
|
13
13
|
const core_1 = require("@rsdk/core");
|
|
14
14
|
const kafka_common_1 = require("@rsdk/kafka.common");
|
|
15
|
-
let
|
|
15
|
+
let ProducerConfig = class ProducerConfig extends core_1.Config {
|
|
16
16
|
payloadFormat;
|
|
17
17
|
};
|
|
18
|
-
exports.
|
|
18
|
+
exports.ProducerConfig = ProducerConfig;
|
|
19
19
|
__decorate([
|
|
20
20
|
(0, core_1.Property)('KAFKA_PAYLOAD_FORMAT', new core_1.EnumParser(kafka_common_1.PayloadFormat), {
|
|
21
21
|
description: 'Option for producer payload serializer',
|
|
22
22
|
defaultValue: kafka_common_1.DEFAULT_PAYLOAD_FORMAT,
|
|
23
23
|
}),
|
|
24
24
|
__metadata("design:type", String)
|
|
25
|
-
],
|
|
26
|
-
exports.
|
|
25
|
+
], ProducerConfig.prototype, "payloadFormat", void 0);
|
|
26
|
+
exports.ProducerConfig = ProducerConfig = __decorate([
|
|
27
27
|
(0, core_1.ConfigSection)({
|
|
28
28
|
name: 'kafka producer config',
|
|
29
29
|
tags: [core_1.ConfigTag.infrastructure, core_1.ConfigTag.storage],
|
|
30
30
|
})
|
|
31
|
-
],
|
|
32
|
-
//# sourceMappingURL=
|
|
31
|
+
], ProducerConfig);
|
|
32
|
+
//# sourceMappingURL=producer.config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"producer.config.js","sourceRoot":"","sources":["../src/producer.config.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,qCAMoB;AACpB,qDAA2E;AAQpE,IAAM,cAAc,GAApB,MAAM,cAAe,SAAQ,aAAM;IAKxC,aAAa,CAAiB;CAC/B,CAAA;AANY,wCAAc;AAKzB;IAJC,IAAA,eAAQ,EAAC,sBAAsB,EAAE,IAAI,iBAAU,CAAC,4BAAa,CAAC,EAAE;QAC/D,WAAW,EAAE,wCAAwC;QACrD,YAAY,EAAE,qCAAsB;KACrC,CAAC;;qDAC4B;yBALnB,cAAc;IAJ1B,IAAA,oBAAa,EAAC;QACb,IAAI,EAAE,uBAAuB;QAC7B,IAAI,EAAE,CAAC,gBAAS,CAAC,cAAc,EAAE,gBAAS,CAAC,OAAO,CAAC;KACpD,CAAC;GACW,cAAc,CAM1B"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rsdk/kafka.producer",
|
|
3
|
-
"version": "4.0.0-next.
|
|
3
|
+
"version": "4.0.0-next.8",
|
|
4
4
|
"description": "Publishers for kafka (different direct and outbox strategies)",
|
|
5
5
|
"license": "Apache License 2.0",
|
|
6
6
|
"publishConfig": {
|
|
@@ -15,12 +15,12 @@
|
|
|
15
15
|
"@nestjs/core": "^10.0.0",
|
|
16
16
|
"@nestjs/microservices": "^10.0.0",
|
|
17
17
|
"@rsdk/common": "^4.0.0-next.6",
|
|
18
|
-
"@rsdk/core": "^4.0.0-next.
|
|
19
|
-
"@rsdk/kafka.common": "^4.0.0-next.
|
|
18
|
+
"@rsdk/core": "^4.0.0-next.8",
|
|
19
|
+
"@rsdk/kafka.common": "^4.0.0-next.8",
|
|
20
20
|
"@rsdk/logging": "^4.0.0-next.6",
|
|
21
21
|
"lodash": "^4.17.21",
|
|
22
22
|
"reflect-metadata": "^0.1.13",
|
|
23
23
|
"rxjs": "^7.0.0"
|
|
24
24
|
},
|
|
25
|
-
"gitHead": "
|
|
25
|
+
"gitHead": "bae7c74fc8be068b8bd66b403d19e2b32cf092ec"
|
|
26
26
|
}
|
package/src/constants.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export const EVENT_TYPE_TOKEN = Symbol.for('EVENT_TYPE_TOKEN');
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import type { EventPayload, EventType } from '@rsdk/kafka.common';
|
|
2
|
+
import {
|
|
3
|
+
DEFAULT_PAYLOAD_FORMAT,
|
|
4
|
+
JSONEventCodec,
|
|
5
|
+
PayloadFormat,
|
|
6
|
+
ProtoEventCodec,
|
|
7
|
+
X_FORMAT_HEADER,
|
|
8
|
+
} from '@rsdk/kafka.common';
|
|
9
|
+
import { get } from 'lodash';
|
|
10
|
+
import { randomUUID } from 'node:crypto';
|
|
11
|
+
|
|
12
|
+
import type {
|
|
13
|
+
EventMetadata,
|
|
14
|
+
ProducerOptions,
|
|
15
|
+
PublishOptions,
|
|
16
|
+
} from './interfaces';
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Абстрактный класс, содержащий полезные методы для подготовки
|
|
20
|
+
* данных, как для непосредственной отправки в kafka посредством
|
|
21
|
+
* kafkajs, так и для сохранения в outbox
|
|
22
|
+
*/
|
|
23
|
+
export abstract class EventsFactory<T extends EventType, K extends object> {
|
|
24
|
+
private readonly payloadFormat: PayloadFormat;
|
|
25
|
+
|
|
26
|
+
constructor(
|
|
27
|
+
protected readonly eventType: T,
|
|
28
|
+
config: ProducerOptions,
|
|
29
|
+
) {
|
|
30
|
+
this.payloadFormat = config.payloadFormat ?? DEFAULT_PAYLOAD_FORMAT;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
protected getMetadata(
|
|
34
|
+
metadata: EventMetadata | undefined | null,
|
|
35
|
+
): Record<string, string> {
|
|
36
|
+
return {
|
|
37
|
+
...metadata,
|
|
38
|
+
...(this.payloadFormat &&
|
|
39
|
+
this.payloadFormat !== DEFAULT_PAYLOAD_FORMAT && {
|
|
40
|
+
[X_FORMAT_HEADER]: this.payloadFormat,
|
|
41
|
+
}),
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
protected serializePayload<P extends EventPayload<T>>(payload: P): Buffer {
|
|
46
|
+
const codec =
|
|
47
|
+
this.payloadFormat === PayloadFormat.JSON
|
|
48
|
+
? new JSONEventCodec<T>()
|
|
49
|
+
: new ProtoEventCodec<T>(this.eventType);
|
|
50
|
+
|
|
51
|
+
return codec.encode(payload);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
protected getPartitionKey<P extends Record<string, any>>(
|
|
55
|
+
payload: P,
|
|
56
|
+
options?: PublishOptions,
|
|
57
|
+
): string {
|
|
58
|
+
/**
|
|
59
|
+
* Если ключ партиционирования передаётся
|
|
60
|
+
* в опциях явно - берём его
|
|
61
|
+
*/
|
|
62
|
+
if (options?.partitionKey) {
|
|
63
|
+
return String(options.partitionKey);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Если в типе события определено имя ключевого поля,
|
|
68
|
+
* извлекаем его значение и используем
|
|
69
|
+
*/
|
|
70
|
+
const { $partitionKeyField } = this.eventType;
|
|
71
|
+
if ($partitionKeyField && typeof $partitionKeyField === 'string') {
|
|
72
|
+
// TODO: Может быть лучше ошибку кидать?
|
|
73
|
+
return get(payload, $partitionKeyField) ?? randomUUID();
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Ищем последовательно поля id, uuid, guid,
|
|
78
|
+
* если ничего не найдено, используем генерируем
|
|
79
|
+
* ключ случайным образом
|
|
80
|
+
*/
|
|
81
|
+
return (
|
|
82
|
+
get(payload, 'id') ??
|
|
83
|
+
get(payload, 'uuid') ??
|
|
84
|
+
get(payload, 'guid') ??
|
|
85
|
+
randomUUID()
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
abstract create(payload: EventPayload<T>, options?: PublishOptions): K;
|
|
90
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -1,4 +1,13 @@
|
|
|
1
|
-
export {
|
|
2
|
-
|
|
3
|
-
export {
|
|
4
|
-
|
|
1
|
+
export { EventsFactory } from './events-factory.abstract';
|
|
2
|
+
|
|
3
|
+
export { EVENT_TYPE_TOKEN } from './constants';
|
|
4
|
+
|
|
5
|
+
export {
|
|
6
|
+
KafkaProducer,
|
|
7
|
+
BatchProducerItem,
|
|
8
|
+
KafkaBatchProducer,
|
|
9
|
+
PublishOptions,
|
|
10
|
+
ProducerOptions,
|
|
11
|
+
EventMetadata,
|
|
12
|
+
} from './interfaces';
|
|
13
|
+
export { ProducerConfig } from './producer.config';
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { MaybeReadonlyArray } from '@rsdk/common.node';
|
|
2
|
+
import type {
|
|
3
|
+
EventPayload,
|
|
4
|
+
EventType,
|
|
5
|
+
PayloadFormat,
|
|
6
|
+
} from '@rsdk/kafka.common';
|
|
7
|
+
import type { TupleToUnion } from 'type-fest';
|
|
8
|
+
|
|
9
|
+
export interface ProducerOptions {
|
|
10
|
+
payloadFormat?: PayloadFormat;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export type EventMetadata = Record<string, unknown>;
|
|
14
|
+
|
|
15
|
+
export interface PublishOptions {
|
|
16
|
+
/** Metadata must be a plain object. Object will be stringified */
|
|
17
|
+
metadata?: EventMetadata;
|
|
18
|
+
|
|
19
|
+
/** Override partition key */
|
|
20
|
+
partitionKey?: string | number;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface BatchProducerItem<E extends EventType> {
|
|
24
|
+
eventType: E;
|
|
25
|
+
payload: EventPayload<E>;
|
|
26
|
+
options?: PublishOptions;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface RawKafkaBatchProducer {
|
|
30
|
+
publish<E extends EventType>(items: BatchProducerItem<E>[]): Promise<void>;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface KafkaBatchProducer<E extends MaybeReadonlyArray<EventType>> {
|
|
34
|
+
publish(items: BatchProducerItem<TupleToUnion<E>>[]): Promise<void>;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface KafkaProducer<E extends EventType> {
|
|
38
|
+
publish<P extends EventPayload<E>>(
|
|
39
|
+
payload: P,
|
|
40
|
+
options?: PublishOptions,
|
|
41
|
+
): Promise<void>;
|
|
42
|
+
}
|
|
@@ -7,11 +7,13 @@ import {
|
|
|
7
7
|
} from '@rsdk/core';
|
|
8
8
|
import { DEFAULT_PAYLOAD_FORMAT, PayloadFormat } from '@rsdk/kafka.common';
|
|
9
9
|
|
|
10
|
+
import type { ProducerOptions } from './interfaces';
|
|
11
|
+
|
|
10
12
|
@ConfigSection({
|
|
11
13
|
name: 'kafka producer config',
|
|
12
14
|
tags: [ConfigTag.infrastructure, ConfigTag.storage],
|
|
13
15
|
})
|
|
14
|
-
export class
|
|
16
|
+
export class ProducerConfig extends Config implements ProducerOptions {
|
|
15
17
|
@Property('KAFKA_PAYLOAD_FORMAT', new EnumParser(PayloadFormat), {
|
|
16
18
|
description: 'Option for producer payload serializer',
|
|
17
19
|
defaultValue: DEFAULT_PAYLOAD_FORMAT,
|
package/tsconfig.json
CHANGED
|
@@ -4,6 +4,5 @@
|
|
|
4
4
|
"declaration": true,
|
|
5
5
|
"outDir": "dist"
|
|
6
6
|
},
|
|
7
|
-
"
|
|
8
|
-
"exclude": ["node_modules", "dist", "test", "**/*.spec.ts", "**/*.test.ts", "**/*.spec.e2e.ts", "**/*.test.e2e.ts"]
|
|
7
|
+
"exclude": ["node_modules", "dist", "test", "**/*.spec.ts", "**/*.test.ts", "**/*.spec.e2e.ts", "**/*.test.e2e.ts"]
|
|
9
8
|
}
|
package/dist/base.producer.d.ts
DELETED
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
|
-
import { type EventType } from '@rsdk/kafka.common';
|
|
3
|
-
import type { KafkaProducerConfig } from './kafka-producer-plugin.config';
|
|
4
|
-
import type { KafkaProducer, OutboxTable, PublishOptions, PublishOptionsMetadata } from './types';
|
|
5
|
-
export declare abstract class BaseProducer<T extends EventType, P = T extends EventType<infer X> ? X : never> implements KafkaProducer<P> {
|
|
6
|
-
readonly eventType: T;
|
|
7
|
-
readonly kafkaProducerConfig: KafkaProducerConfig;
|
|
8
|
-
constructor(eventType: T, kafkaProducerConfig: KafkaProducerConfig);
|
|
9
|
-
protected getPartitionKey(payload: P, partitionKeyField: string | null): string;
|
|
10
|
-
protected prepareDataForOutbox(payload: P, options?: PublishOptions): Partial<OutboxTable>;
|
|
11
|
-
protected serializePayload(payload: P): Buffer;
|
|
12
|
-
protected addSerializeFormatToMetadata(metadata: PublishOptionsMetadata | undefined | null): PublishOptionsMetadata | null;
|
|
13
|
-
abstract publish(payload: P, options?: PublishOptions | undefined): Promise<void>;
|
|
14
|
-
}
|
package/dist/base.producer.js
DELETED
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.BaseProducer = void 0;
|
|
7
|
-
const kafka_common_1 = require("@rsdk/kafka.common");
|
|
8
|
-
const get_1 = __importDefault(require("lodash/get"));
|
|
9
|
-
const node_crypto_1 = require("node:crypto");
|
|
10
|
-
class BaseProducer {
|
|
11
|
-
eventType;
|
|
12
|
-
kafkaProducerConfig;
|
|
13
|
-
constructor(eventType, kafkaProducerConfig) {
|
|
14
|
-
this.eventType = eventType;
|
|
15
|
-
this.kafkaProducerConfig = kafkaProducerConfig;
|
|
16
|
-
}
|
|
17
|
-
getPartitionKey(payload, partitionKeyField) {
|
|
18
|
-
const partitionKey = typeof partitionKeyField === 'string' && partitionKeyField.length > 0
|
|
19
|
-
? (0, get_1.default)(payload, partitionKeyField, (0, node_crypto_1.randomUUID)())
|
|
20
|
-
: (0, get_1.default)(payload, 'id') ??
|
|
21
|
-
(0, get_1.default)(payload, 'uuid') ??
|
|
22
|
-
(0, get_1.default)(payload, 'guid', (0, node_crypto_1.randomUUID)());
|
|
23
|
-
return String(partitionKey);
|
|
24
|
-
}
|
|
25
|
-
prepareDataForOutbox(payload, options) {
|
|
26
|
-
return {
|
|
27
|
-
group: this.eventType.$group,
|
|
28
|
-
metadata: this.addSerializeFormatToMetadata(options?.metadata || {}),
|
|
29
|
-
partition_key: options?.partitionKey
|
|
30
|
-
? String(options.partitionKey)
|
|
31
|
-
: this.getPartitionKey(payload, this.eventType.$partitionKeyField),
|
|
32
|
-
payload: this.serializePayload(payload),
|
|
33
|
-
type: this.eventType.$type,
|
|
34
|
-
};
|
|
35
|
-
}
|
|
36
|
-
serializePayload(payload) {
|
|
37
|
-
if (this.kafkaProducerConfig.payloadFormat === kafka_common_1.PayloadFormat.JSON) {
|
|
38
|
-
return Buffer.from(JSON.stringify(payload));
|
|
39
|
-
}
|
|
40
|
-
return Buffer.from(this.eventType.encode(payload).finish());
|
|
41
|
-
}
|
|
42
|
-
addSerializeFormatToMetadata(metadata) {
|
|
43
|
-
return {
|
|
44
|
-
...metadata,
|
|
45
|
-
...(this.kafkaProducerConfig.payloadFormat !== kafka_common_1.DEFAULT_PAYLOAD_FORMAT && {
|
|
46
|
-
[kafka_common_1.X_FORMAT_HEADER]: this.kafkaProducerConfig.payloadFormat,
|
|
47
|
-
}),
|
|
48
|
-
};
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
exports.BaseProducer = BaseProducer;
|
|
52
|
-
//# sourceMappingURL=base.producer.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"base.producer.js","sourceRoot":"","sources":["../src/base.producer.ts"],"names":[],"mappings":";;;;;;AAAA,qDAK4B;AAC5B,qDAA6B;AAC7B,6CAAyC;AAUzC,MAAsB,YAAY;IAMrB;IACA;IAFX,YACW,SAAY,EACZ,mBAAwC;QADxC,cAAS,GAAT,SAAS,CAAG;QACZ,wBAAmB,GAAnB,mBAAmB,CAAqB;IAChD,CAAC;IAEM,eAAe,CACvB,OAAU,EACV,iBAAgC;QAEhC,MAAM,YAAY,GAChB,OAAO,iBAAiB,KAAK,QAAQ,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC;YACnE,CAAC,CAAC,IAAA,aAAG,EAAC,OAAO,EAAE,iBAAiB,EAAE,IAAA,wBAAU,GAAE,CAAC;YAC/C,CAAC,CAAC,IAAA,aAAG,EAAC,OAAO,EAAE,IAAI,CAAC;gBAClB,IAAA,aAAG,EAAC,OAAO,EAAE,MAAM,CAAC;gBACpB,IAAA,aAAG,EAAC,OAAO,EAAE,MAAM,EAAE,IAAA,wBAAU,GAAE,CAAC,CAAC;QAEzC,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC;IAC9B,CAAC;IAES,oBAAoB,CAC5B,OAAU,EACV,OAAwB;QAExB,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM;YAC5B,QAAQ,EAAE,IAAI,CAAC,4BAA4B,CAAC,OAAO,EAAE,QAAQ,IAAI,EAAE,CAAC;YACpE,aAAa,EAAE,OAAO,EAAE,YAAY;gBAClC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC;gBAC9B,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC;YACpE,OAAO,EAAE,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC;YACvC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK;SAC3B,CAAC;IACJ,CAAC;IAES,gBAAgB,CAAC,OAAU;QACnC,IAAI,IAAI,CAAC,mBAAmB,CAAC,aAAa,KAAK,4BAAa,CAAC,IAAI,EAAE;YACjE,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;SAC7C;QACD,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9D,CAAC;IAES,4BAA4B,CACpC,QAAmD;QAEnD,OAAO;YACL,GAAG,QAAQ;YACX,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,aAAa,KAAK,qCAAsB,IAAI;gBACvE,CAAC,8BAAe,CAAC,EAAE,IAAI,CAAC,mBAAmB,CAAC,aAAa;aAC1D,CAAC;SACH,CAAC;IACJ,CAAC;CAMF;AA7DD,oCA6DC"}
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.InjectProducer = void 0;
|
|
4
|
-
const common_1 = require("@nestjs/common");
|
|
5
|
-
const InjectProducer = (eventType) => (0, common_1.Inject)(eventType.$type);
|
|
6
|
-
exports.InjectProducer = InjectProducer;
|
|
7
|
-
//# sourceMappingURL=inject-producer.decorator.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"inject-producer.decorator.js","sourceRoot":"","sources":["../src/inject-producer.decorator.ts"],"names":[],"mappings":";;;AAAA,2CAAwC;AAGjC,MAAM,cAAc,GAAG,CAC5B,SAAuB,EACI,EAAE,CAAC,IAAA,eAAM,EAAC,SAAS,CAAC,KAAK,CAAC,CAAC;AAF3C,QAAA,cAAc,kBAE6B"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"kafka-producer-plugin.config.js","sourceRoot":"","sources":["../src/kafka-producer-plugin.config.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,qCAMoB;AACpB,qDAA2E;AAMpE,IAAM,mBAAmB,GAAzB,MAAM,mBAAoB,SAAQ,aAAM;IAK7C,aAAa,CAAiB;CAC/B,CAAA;AANY,kDAAmB;AAK9B;IAJC,IAAA,eAAQ,EAAC,sBAAsB,EAAE,IAAI,iBAAU,CAAC,4BAAa,CAAC,EAAE;QAC/D,WAAW,EAAE,wCAAwC;QACrD,YAAY,EAAE,qCAAsB;KACrC,CAAC;;0DAC4B;8BALnB,mBAAmB;IAJ/B,IAAA,oBAAa,EAAC;QACb,IAAI,EAAE,uBAAuB;QAC7B,IAAI,EAAE,CAAC,gBAAS,CAAC,cAAc,EAAE,gBAAS,CAAC,OAAO,CAAC;KACpD,CAAC;GACW,mBAAmB,CAM/B"}
|
package/dist/types.d.ts
DELETED
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
|
-
export type PublishOptionsMetadata = Record<string, any>;
|
|
3
|
-
export interface PublishOptions {
|
|
4
|
-
metadata?: PublishOptionsMetadata;
|
|
5
|
-
partitionKey?: string | number;
|
|
6
|
-
}
|
|
7
|
-
export interface OutboxTable {
|
|
8
|
-
id: string;
|
|
9
|
-
type: string;
|
|
10
|
-
group: string;
|
|
11
|
-
partition_key: string;
|
|
12
|
-
metadata: PublishOptionsMetadata | null;
|
|
13
|
-
payload: Buffer;
|
|
14
|
-
}
|
|
15
|
-
export interface KafkaProducer<P> {
|
|
16
|
-
publish(payload: P, options?: PublishOptions | undefined): Promise<void>;
|
|
17
|
-
}
|
package/dist/types.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}
|
package/src/base.producer.ts
DELETED
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
DEFAULT_PAYLOAD_FORMAT,
|
|
3
|
-
type EventType,
|
|
4
|
-
PayloadFormat,
|
|
5
|
-
X_FORMAT_HEADER,
|
|
6
|
-
} from '@rsdk/kafka.common';
|
|
7
|
-
import get from 'lodash/get';
|
|
8
|
-
import { randomUUID } from 'node:crypto';
|
|
9
|
-
|
|
10
|
-
import type { KafkaProducerConfig } from './kafka-producer-plugin.config';
|
|
11
|
-
import type {
|
|
12
|
-
KafkaProducer,
|
|
13
|
-
OutboxTable,
|
|
14
|
-
PublishOptions,
|
|
15
|
-
PublishOptionsMetadata,
|
|
16
|
-
} from './types';
|
|
17
|
-
|
|
18
|
-
export abstract class BaseProducer<
|
|
19
|
-
T extends EventType,
|
|
20
|
-
P = T extends EventType<infer X> ? X : never,
|
|
21
|
-
> implements KafkaProducer<P>
|
|
22
|
-
{
|
|
23
|
-
constructor(
|
|
24
|
-
readonly eventType: T,
|
|
25
|
-
readonly kafkaProducerConfig: KafkaProducerConfig,
|
|
26
|
-
) {}
|
|
27
|
-
|
|
28
|
-
protected getPartitionKey(
|
|
29
|
-
payload: P,
|
|
30
|
-
partitionKeyField: string | null,
|
|
31
|
-
): string {
|
|
32
|
-
const partitionKey =
|
|
33
|
-
typeof partitionKeyField === 'string' && partitionKeyField.length > 0
|
|
34
|
-
? get(payload, partitionKeyField, randomUUID())
|
|
35
|
-
: get(payload, 'id') ??
|
|
36
|
-
get(payload, 'uuid') ??
|
|
37
|
-
get(payload, 'guid', randomUUID());
|
|
38
|
-
|
|
39
|
-
return String(partitionKey);
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
protected prepareDataForOutbox(
|
|
43
|
-
payload: P,
|
|
44
|
-
options?: PublishOptions,
|
|
45
|
-
): Partial<OutboxTable> {
|
|
46
|
-
return {
|
|
47
|
-
group: this.eventType.$group,
|
|
48
|
-
metadata: this.addSerializeFormatToMetadata(options?.metadata || {}),
|
|
49
|
-
partition_key: options?.partitionKey
|
|
50
|
-
? String(options.partitionKey)
|
|
51
|
-
: this.getPartitionKey(payload, this.eventType.$partitionKeyField),
|
|
52
|
-
payload: this.serializePayload(payload),
|
|
53
|
-
type: this.eventType.$type,
|
|
54
|
-
};
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
protected serializePayload(payload: P): Buffer {
|
|
58
|
-
if (this.kafkaProducerConfig.payloadFormat === PayloadFormat.JSON) {
|
|
59
|
-
return Buffer.from(JSON.stringify(payload));
|
|
60
|
-
}
|
|
61
|
-
return Buffer.from(this.eventType.encode(payload).finish());
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
protected addSerializeFormatToMetadata(
|
|
65
|
-
metadata: PublishOptionsMetadata | undefined | null,
|
|
66
|
-
): PublishOptionsMetadata | null {
|
|
67
|
-
return {
|
|
68
|
-
...metadata,
|
|
69
|
-
...(this.kafkaProducerConfig.payloadFormat !== DEFAULT_PAYLOAD_FORMAT && {
|
|
70
|
-
[X_FORMAT_HEADER]: this.kafkaProducerConfig.payloadFormat,
|
|
71
|
-
}),
|
|
72
|
-
};
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
abstract publish(
|
|
76
|
-
payload: P,
|
|
77
|
-
options?: PublishOptions | undefined,
|
|
78
|
-
): Promise<void>;
|
|
79
|
-
}
|
package/src/types.ts
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
export type PublishOptionsMetadata = Record<string, any>;
|
|
2
|
-
|
|
3
|
-
export interface PublishOptions {
|
|
4
|
-
// Metadata must be a plain object. Object will be stringified.
|
|
5
|
-
metadata?: PublishOptionsMetadata;
|
|
6
|
-
partitionKey?: string | number;
|
|
7
|
-
}
|
|
8
|
-
|
|
9
|
-
export interface OutboxTable {
|
|
10
|
-
id: string;
|
|
11
|
-
type: string;
|
|
12
|
-
group: string;
|
|
13
|
-
partition_key: string;
|
|
14
|
-
metadata: PublishOptionsMetadata | null;
|
|
15
|
-
payload: Buffer;
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
// TODO: Добавить типизацию T extends EventType. Но с текущим генерируемым кодом это не работает
|
|
19
|
-
// т. к. интерфейс события по факту не экстендит EventType
|
|
20
|
-
export interface KafkaProducer<P> {
|
|
21
|
-
publish(payload: P, options?: PublishOptions | undefined): Promise<void>;
|
|
22
|
-
}
|