@rsdk/kafka.producer 3.6.0-next.2 → 3.6.0-next.3

package/CHANGELOG.md

@@ -3,6 +3,12 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [3.6.0-next.3](https://github.com/R-Vision/rsdk/compare/v3.6.0-next.2...v3.6.0-next.3) (2023-10-11)
+
+### Features
+
+* **kafka.producer:** pfm-461-add-kafka-producer-plugin-with-producer-serializer ([3faeaa3](https://github.com/R-Vision/rsdk/commit/3faeaa30682eed824abf16200f684e5cb6f2e596))
+
 ## [3.6.0-next.2](https://github.com/R-Vision/rsdk/compare/v3.6.0-next.1...v3.6.0-next.2) (2023-10-10)
 
 **Note:** Version bump only for package @rsdk/kafka.producer

package/dist/base.producer.d.ts

@@ -1,9 +1,14 @@
-import type { EventType } from '@rsdk/kafka.common';
-import type { KafkaProducer, OutboxTable, PublishOptions } from './types';
+/// <reference types="node" />
+import { type EventType } from '@rsdk/kafka.common';
+import type { KafkaProducerConfig } from './kafka-producer-plugin.config';
+import type { KafkaProducer, OutboxTable, PublishOptions, PublishOptionsMetadata } from './types';
 export declare abstract class BaseProducer<T extends EventType, P = T extends EventType<infer X> ? X : never> implements KafkaProducer<P> {
     readonly eventType: T;
-    constructor(eventType: T);
+    readonly kafkaProducerConfig: KafkaProducerConfig;
+    constructor(eventType: T, kafkaProducerConfig: KafkaProducerConfig);
     protected getPartitionKey(payload: P, partitionKeyField: string | null): string;
-    protected parsePayload(payload: P, options?: PublishOptions): Partial<OutboxTable>;
+    protected prepareDataForOutbox(payload: P, options?: PublishOptions): Partial<OutboxTable>;
+    protected serializePayload(payload: P): Buffer;
+    protected addSerializeFormatToMetadata(metadata: PublishOptionsMetadata | undefined | null): PublishOptionsMetadata | null;
     abstract publish(payload: P, options?: PublishOptions | undefined): Promise<void>;
 }

package/dist/base.producer.js

@@ -4,12 +4,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.BaseProducer = void 0;
+const kafka_common_1 = require("@rsdk/kafka.common");
 const get_1 = __importDefault(require("lodash/get"));
 const node_crypto_1 = require("node:crypto");
 class BaseProducer {
     eventType;
-    constructor(eventType) {
+    kafkaProducerConfig;
+    constructor(eventType, kafkaProducerConfig) {
         this.eventType = eventType;
+        this.kafkaProducerConfig = kafkaProducerConfig;
     }
     getPartitionKey(payload, partitionKeyField) {
         const partitionKey = typeof partitionKeyField === 'string' && partitionKeyField.length > 0
@@ -19,17 +22,31 @@ class BaseProducer {
                 (0, get_1.default)(payload, 'guid', (0, node_crypto_1.randomUUID)());
         return String(partitionKey);
     }
-    parsePayload(payload, options) {
+    prepareDataForOutbox(payload, options) {
         return {
             group: this.eventType.$group,
-            metadata: options?.metadata ?? null,
+            metadata: this.addSerializeFormatToMetadata(options?.metadata || {}),
             partition_key: options?.partitionKey
                 ? String(options.partitionKey)
                 : this.getPartitionKey(payload, this.eventType.$partitionKeyField),
-            payload: Buffer.from(this.eventType.encode(payload).finish()),
+            payload: this.serializePayload(payload),
             type: this.eventType.$type,
         };
     }
+    serializePayload(payload) {
+        if (this.kafkaProducerConfig.payloadFormat === kafka_common_1.PayloadFormat.JSON) {
+            return Buffer.from(JSON.stringify(payload));
+        }
+        return Buffer.from(this.eventType.encode(payload).finish());
+    }
+    addSerializeFormatToMetadata(metadata) {
+        return {
+            ...metadata,
+            ...(this.kafkaProducerConfig.payloadFormat !== kafka_common_1.DEFAULT_PAYLOAD_FORMAT && {
+                [kafka_common_1.X_FORMAT_HEADER]: this.kafkaProducerConfig.payloadFormat,
+            }),
+        };
+    }
 }
 exports.BaseProducer = BaseProducer;
 //# sourceMappingURL=base.producer.js.map

package/dist/base.producer.js.map

@@ -1 +1 @@
-{"version":3,"file":"base.producer.js","sourceRoot":"","sources":["../src/base.producer.ts"],"names":[],"mappings":";;;;;;AACA,qDAA6B;AAC7B,6CAAyC;AAIzC,MAAsB,YAAY;IAKX;IAArB,YAAqB,SAAY;QAAZ,cAAS,GAAT,SAAS,CAAG;IAAG,CAAC;IAE3B,eAAe,CACvB,OAAU,EACV,iBAAgC;QAEhC,MAAM,YAAY,GAChB,OAAO,iBAAiB,KAAK,QAAQ,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC;YACnE,CAAC,CAAC,IAAA,aAAG,EAAC,OAAO,EAAE,iBAAiB,EAAE,IAAA,wBAAU,GAAE,CAAC;YAC/C,CAAC,CAAC,IAAA,aAAG,EAAC,OAAO,EAAE,IAAI,CAAC;gBAClB,IAAA,aAAG,EAAC,OAAO,EAAE,MAAM,CAAC;gBACpB,IAAA,aAAG,EAAC,OAAO,EAAE,MAAM,EAAE,IAAA,wBAAU,GAAE,CAAC,CAAC;QAEzC,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC;IAC9B,CAAC;IAES,YAAY,CACpB,OAAU,EACV,OAAwB;QAExB,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM;YAC5B,QAAQ,EAAE,OAAO,EAAE,QAAQ,IAAI,IAAI;YACnC,aAAa,EAAE,OAAO,EAAE,YAAY;gBAClC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC;gBAC9B,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC;YACpE,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;YAC7D,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK;SAC3B,CAAC;IACJ,CAAC;CAMF;AAxCD,oCAwCC"}
+{"version":3,"file":"base.producer.js","sourceRoot":"","sources":["../src/base.producer.ts"],"names":[],"mappings":";;;;;;AAAA,qDAK4B;AAC5B,qDAA6B;AAC7B,6CAAyC;AAUzC,MAAsB,YAAY;IAMrB;IACA;IAFX,YACW,SAAY,EACZ,mBAAwC;QADxC,cAAS,GAAT,SAAS,CAAG;QACZ,wBAAmB,GAAnB,mBAAmB,CAAqB;IAChD,CAAC;IAEM,eAAe,CACvB,OAAU,EACV,iBAAgC;QAEhC,MAAM,YAAY,GAChB,OAAO,iBAAiB,KAAK,QAAQ,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC;YACnE,CAAC,CAAC,IAAA,aAAG,EAAC,OAAO,EAAE,iBAAiB,EAAE,IAAA,wBAAU,GAAE,CAAC;YAC/C,CAAC,CAAC,IAAA,aAAG,EAAC,OAAO,EAAE,IAAI,CAAC;gBAClB,IAAA,aAAG,EAAC,OAAO,EAAE,MAAM,CAAC;gBACpB,IAAA,aAAG,EAAC,OAAO,EAAE,MAAM,EAAE,IAAA,wBAAU,GAAE,CAAC,CAAC;QAEzC,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC;IAC9B,CAAC;IAES,oBAAoB,CAC5B,OAAU,EACV,OAAwB;QAExB,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM;YAC5B,QAAQ,EAAE,IAAI,CAAC,4BAA4B,CAAC,OAAO,EAAE,QAAQ,IAAI,EAAE,CAAC;YACpE,aAAa,EAAE,OAAO,EAAE,YAAY;gBAClC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC;gBAC9B,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC;YACpE,OAAO,EAAE,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC;YACvC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK;SAC3B,CAAC;IACJ,CAAC;IAES,gBAAgB,CAAC,OAAU;QACnC,IAAI,IAAI,CAAC,mBAAmB,CAAC,aAAa,KAAK,4BAAa,CAAC,IAAI,EAAE;YACjE,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;SAC7C;QACD,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9D,CAAC;IAES,4BAA4B,CACpC,QAAmD;QAEnD,OAAO;YACL,GAAG,QAAQ;YACX,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,aAAa,KAAK,qCAAsB,IAAI;gBACvE,CAAC,8BAAe,CAAC,EAAE,IAAI,CAAC,mBAAmB,CAAC,aAAa;aAC1D,CAAC;SACH,CAAC;IACJ,CAAC;CAMF;AA7DD,oCA6DC"}

package/dist/index.d.ts

@@ -1,3 +1,4 @@
 export { BaseProducer } from './base.producer';
 export { InjectProducer } from './inject-producer.decorator';
+export { KafkaProducerConfig } from './kafka-producer-plugin.config';
 export { KafkaProducer, OutboxTable, PublishOptions } from './types';

package/dist/index.js

@@ -1,8 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.InjectProducer = exports.BaseProducer = void 0;
+exports.KafkaProducerConfig = exports.InjectProducer = exports.BaseProducer = void 0;
 var base_producer_1 = require("./base.producer");
 Object.defineProperty(exports, "BaseProducer", { enumerable: true, get: function () { return base_producer_1.BaseProducer; } });
 var inject_producer_decorator_1 = require("./inject-producer.decorator");
 Object.defineProperty(exports, "InjectProducer", { enumerable: true, get: function () { return inject_producer_decorator_1.InjectProducer; } });
+var kafka_producer_plugin_config_1 = require("./kafka-producer-plugin.config");
+Object.defineProperty(exports, "KafkaProducerConfig", { enumerable: true, get: function () { return kafka_producer_plugin_config_1.KafkaProducerConfig; } });
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,iDAA+C;AAAtC,6GAAA,YAAY,OAAA;AACrB,yEAA6D;AAApD,2HAAA,cAAc,OAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,iDAA+C;AAAtC,6GAAA,YAAY,OAAA;AACrB,yEAA6D;AAApD,2HAAA,cAAc,OAAA;AACvB,+EAAqE;AAA5D,mIAAA,mBAAmB,OAAA"}

package/dist/kafka-producer-plugin.config.d.ts

@@ -0,0 +1,5 @@
+import { Config } from '@rsdk/core';
+import { PayloadFormat } from '@rsdk/kafka.common';
+export declare class KafkaProducerConfig extends Config {
+    payloadFormat: PayloadFormat;
+}

package/dist/kafka-producer-plugin.config.js

@@ -0,0 +1,31 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KafkaProducerConfig = void 0;
+const core_1 = require("@rsdk/core");
+const kafka_common_1 = require("@rsdk/kafka.common");
+let KafkaProducerConfig = exports.KafkaProducerConfig = class KafkaProducerConfig extends core_1.Config {
+    payloadFormat;
+};
+__decorate([
+    (0, core_1.Property)('KAFKA_PAYLOAD_FORMAT', new core_1.EnumParser(kafka_common_1.PayloadFormat), {
+        description: 'Option for producer payload serializer',
+        defaultValue: kafka_common_1.DEFAULT_PAYLOAD_FORMAT,
+    }),
+    __metadata("design:type", String)
+], KafkaProducerConfig.prototype, "payloadFormat", void 0);
+exports.KafkaProducerConfig = KafkaProducerConfig = __decorate([
+    (0, core_1.ConfigSection)({
+        name: 'kafka producer config',
+        tags: [core_1.ConfigTag.infrastructure, core_1.ConfigTag.storage],
+    })
+], KafkaProducerConfig);
+//# sourceMappingURL=kafka-producer-plugin.config.js.map

package/dist/kafka-producer-plugin.config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"kafka-producer-plugin.config.js","sourceRoot":"","sources":["../src/kafka-producer-plugin.config.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,qCAMoB;AACpB,qDAA2E;AAMpE,IAAM,mBAAmB,iCAAzB,MAAM,mBAAoB,SAAQ,aAAM;IAK7C,aAAa,CAAiB;CAC/B,CAAA;AADC;IAJC,IAAA,eAAQ,EAAC,sBAAsB,EAAE,IAAI,iBAAU,CAAC,4BAAa,CAAC,EAAE;QAC/D,WAAW,EAAE,wCAAwC;QACrD,YAAY,EAAE,qCAAsB;KACrC,CAAC;;0DAC4B;8BALnB,mBAAmB;IAJ/B,IAAA,oBAAa,EAAC;QACb,IAAI,EAAE,uBAAuB;QAC7B,IAAI,EAAE,CAAC,gBAAS,CAAC,cAAc,EAAE,gBAAS,CAAC,OAAO,CAAC;KACpD,CAAC;GACW,mBAAmB,CAM/B"}

package/dist/types.d.ts

@@ -1,6 +1,7 @@
 /// <reference types="node" />
+export type PublishOptionsMetadata = Record<string, any>;
 export interface PublishOptions {
-    metadata?: Record<string, any>;
+    metadata?: PublishOptionsMetadata;
     partitionKey?: string | number;
 }
 export interface OutboxTable {
@@ -8,7 +9,7 @@ export interface OutboxTable {
     type: string;
     group: string;
     partition_key: string;
-    metadata: Record<string, any> | null;
+    metadata: PublishOptionsMetadata | null;
     payload: Buffer;
 }
 export interface KafkaProducer<P> {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rsdk/kafka.producer",
-  "version": "3.6.0-next.2",
+  "version": "3.6.0-next.3",
   "description": "Publishers for kafka (different direct and outbox strategies)",
   "license": "Apache License 2.0",
   "publishConfig": {
@@ -16,9 +16,9 @@
     "@nestjs/typeorm": "^10.0.0",
     "@rsdk/common": "^3.5.0",
     "@rsdk/core": "^3.6.0-next.2",
-    "@rsdk/kafka.common": "^3.6.0-next.2",
+    "@rsdk/kafka.common": "^3.6.0-next.3",
     "@rsdk/logging": "^3.5.0",
     "lodash": "^4.17.21"
   },
-  "gitHead": "9852255e942109015c5e47acaf73ae3c00c703cd"
+  "gitHead": "9568fb812dd4f326c98f14cbc6f2b6902a0968f0"
 }

package/src/base.producer.ts

@@ -1,15 +1,29 @@
-import type { EventType } from '@rsdk/kafka.common';
+import {
+  DEFAULT_PAYLOAD_FORMAT,
+  type EventType,
+  PayloadFormat,
+  X_FORMAT_HEADER,
+} from '@rsdk/kafka.common';
 import get from 'lodash/get';
 import { randomUUID } from 'node:crypto';
 
-import type { KafkaProducer, OutboxTable, PublishOptions } from './types';
+import type { KafkaProducerConfig } from './kafka-producer-plugin.config';
+import type {
+  KafkaProducer,
+  OutboxTable,
+  PublishOptions,
+  PublishOptionsMetadata,
+} from './types';
 
 export abstract class BaseProducer<
   T extends EventType,
   P = T extends EventType<infer X> ? X : never,
 > implements KafkaProducer<P>
 {
-  constructor(readonly eventType: T) {}
+  constructor(
+    readonly eventType: T,
+    readonly kafkaProducerConfig: KafkaProducerConfig,
+  ) {}
 
   protected getPartitionKey(
     payload: P,
@@ -25,21 +39,39 @@ export abstract class BaseProducer<
     return String(partitionKey);
   }
 
-  protected parsePayload(
+  protected prepareDataForOutbox(
     payload: P,
     options?: PublishOptions,
   ): Partial<OutboxTable> {
     return {
       group: this.eventType.$group,
-      metadata: options?.metadata ?? null,
+      metadata: this.addSerializeFormatToMetadata(options?.metadata || {}),
       partition_key: options?.partitionKey
         ? String(options.partitionKey)
         : this.getPartitionKey(payload, this.eventType.$partitionKeyField),
-      payload: Buffer.from(this.eventType.encode(payload).finish()),
+      payload: this.serializePayload(payload),
       type: this.eventType.$type,
     };
   }
 
+  protected serializePayload(payload: P): Buffer {
+    if (this.kafkaProducerConfig.payloadFormat === PayloadFormat.JSON) {
+      return Buffer.from(JSON.stringify(payload));
+    }
+    return Buffer.from(this.eventType.encode(payload).finish());
+  }
+
+  protected addSerializeFormatToMetadata(
+    metadata: PublishOptionsMetadata | undefined | null,
+  ): PublishOptionsMetadata | null {
+    return {
+      ...metadata,
+      ...(this.kafkaProducerConfig.payloadFormat !== DEFAULT_PAYLOAD_FORMAT && {
+        [X_FORMAT_HEADER]: this.kafkaProducerConfig.payloadFormat,
+      }),
+    };
+  }
+
   abstract publish(
     payload: P,
     options?: PublishOptions | undefined,

package/src/index.ts

@@ -1,3 +1,4 @@
 export { BaseProducer } from './base.producer';
 export { InjectProducer } from './inject-producer.decorator';
+export { KafkaProducerConfig } from './kafka-producer-plugin.config';
 export { KafkaProducer, OutboxTable, PublishOptions } from './types';

package/src/kafka-producer-plugin.config.ts

@@ -0,0 +1,20 @@
+import {
+  Config,
+  ConfigSection,
+  ConfigTag,
+  EnumParser,
+  Property,
+} from '@rsdk/core';
+import { DEFAULT_PAYLOAD_FORMAT, PayloadFormat } from '@rsdk/kafka.common';
+
+@ConfigSection({
+  name: 'kafka producer config',
+  tags: [ConfigTag.infrastructure, ConfigTag.storage],
+})
+export class KafkaProducerConfig extends Config {
+  @Property('KAFKA_PAYLOAD_FORMAT', new EnumParser(PayloadFormat), {
+    description: 'Option for producer payload serializer',
+    defaultValue: DEFAULT_PAYLOAD_FORMAT,
+  })
+  payloadFormat!: PayloadFormat;
+}

package/src/types.ts

@@ -1,6 +1,8 @@
+export type PublishOptionsMetadata = Record<string, any>;
+
 export interface PublishOptions {
   // Metadata must be a plain object. Object will be stringified.
-  metadata?: Record<string, any>;
+  metadata?: PublishOptionsMetadata;
   partitionKey?: string | number;
 }
 
@@ -9,7 +11,7 @@ export interface OutboxTable {
   type: string;
   group: string;
   partition_key: string;
-  metadata: Record<string, any> | null;
+  metadata: PublishOptionsMetadata | null;
   payload: Buffer;
 }