@drarzter/kafka-client 0.1.1 → 0.1.2

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Rostislav Chapega
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,5 +1,9 @@
 # @drarzter/kafka-client
 
+[![npm version](https://img.shields.io/npm/v/@drarzter/kafka-client)](https://www.npmjs.com/package/@drarzter/kafka-client)
+[![CI](https://github.com/drarzter/kafka-client/actions/workflows/publish.yml/badge.svg)](https://github.com/drarzter/kafka-client/actions/workflows/publish.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+
 Type-safe Kafka client wrapper for NestJS. Built on top of [kafkajs](https://kafka.js.org/).
 
 ## What is this?
@@ -8,10 +12,18 @@ An opinionated wrapper around kafkajs that integrates with NestJS as a DynamicMo
 
 ## Why?
 
-- **Typed topics** — you define a map of topic → message shape, and the compiler won't let you send wrong data to wrong topic
+- **Typed topics** — you define a map of topic -> message shape, and the compiler won't let you send wrong data to wrong topic
 - **NestJS-native** — `register()` / `registerAsync()`, DI injection, lifecycle hooks out of the box
 - **Idempotent producer** — `acks: -1`, `idempotent: true` by default
-- **Minimal** — ~150 lines of code, no magic
+- **Retry + DLQ** — configurable retries with backoff, dead letter queue for failed messages
+- **Batch sending** — send multiple messages in a single request
+- **Partition key support** — route related messages to the same partition
+- **Custom headers** — attach metadata headers to messages
+- **Transactions** — exactly-once semantics with `producer.transaction()`
+- **Consumer interceptors** — before/after/onError hooks for message processing
+- **Health check** — built-in health indicator for monitoring
+- **Multiple consumer groups** — named clients for different bounded contexts
+- **Declarative & imperative** — use `@SubscribeTo()` decorator or `startConsumer()` directly
 
 ## Installation
 
@@ -21,7 +33,63 @@ npm install @drarzter/kafka-client
 pnpm add @drarzter/kafka-client
 ```
 
-Peer dependencies: `@nestjs/common`, `reflect-metadata`, `rxjs`
+Peer dependencies: `@nestjs/common`, `@nestjs/core`, `reflect-metadata`, `rxjs`
+
+## Quick start
+
+Send and receive a message in 3 files:
+
+```typescript
+// types.ts
+import { TTopicMessageMap } from '@drarzter/kafka-client';
+
+export interface MyTopics extends TTopicMessageMap {
+  'hello': { text: string };
+}
+```
+
+```typescript
+// app.module.ts
+import { Module } from '@nestjs/common';
+import { KafkaModule } from '@drarzter/kafka-client';
+import { MyTopics } from './types';
+import { AppService } from './app.service';
+
+@Module({
+  imports: [
+    KafkaModule.register<MyTopics>({
+      clientId: 'my-app',
+      groupId: 'my-group',
+      brokers: ['localhost:9092'],
+    }),
+  ],
+  providers: [AppService],
+})
+export class AppModule {}
+```
+
+```typescript
+// app.service.ts
+import { Injectable } from '@nestjs/common';
+import { InjectKafkaClient, KafkaClient, SubscribeTo } from '@drarzter/kafka-client';
+import { MyTopics } from './types';
+
+@Injectable()
+export class AppService {
+  constructor(
+    @InjectKafkaClient() private readonly kafka: KafkaClient<MyTopics>,
+  ) {}
+
+  async send() {
+    await this.kafka.sendMessage('hello', { text: 'Hello, Kafka!' });
+  }
+
+  @SubscribeTo('hello')
+  async onHello(message: MyTopics['hello']) {
+    console.log('Received:', message.text);
+  }
+}
+```
 
 ## Usage
 
@@ -78,53 +146,269 @@ KafkaModule.registerAsync<OrdersTopicMap>({
 ### 3. Inject and use
 
 ```typescript
-import { Injectable, Inject } from '@nestjs/common';
-import { KafkaClient, KAFKA_CLIENT } from '@drarzter/kafka-client';
+import { Injectable } from '@nestjs/common';
+import { InjectKafkaClient, KafkaClient } from '@drarzter/kafka-client';
 import { OrdersTopicMap } from './orders.types';
 
 @Injectable()
 export class OrdersService {
   constructor(
-    @Inject(KAFKA_CLIENT)
+    @InjectKafkaClient()
     private readonly kafka: KafkaClient<OrdersTopicMap>,
   ) {}
 
   async createOrder() {
-    // ✅ type-safe — compiler knows the shape
     await this.kafka.sendMessage('order.created', {
       orderId: '123',
       userId: '456',
       amount: 100,
     });
+  }
+}
+```
 
-    // ❌ won't compile — wrong payload for this topic
-    // await this.kafka.sendMessage('order.created', { wrong: 'data' });
+## Consuming messages
+
+Two ways — choose what fits your style.
+
+### Declarative: @SubscribeTo()
+
+```typescript
+import { Injectable } from '@nestjs/common';
+import { SubscribeTo } from '@drarzter/kafka-client';
+
+@Injectable()
+export class OrdersHandler {
+  @SubscribeTo('order.created')
+  async handleOrderCreated(message: OrdersTopicMap['order.created'], topic: string) {
+    console.log('New order:', message.orderId);
+  }
+
+  @SubscribeTo('order.completed', { retry: { maxRetries: 3 }, dlq: true })
+  async handleOrderCompleted(message: OrdersTopicMap['order.completed'], topic: string) {
+    console.log('Order completed:', message.orderId);
   }
+}
+```
+
+The module auto-discovers `@SubscribeTo()` methods on startup and subscribes them.
+
+### Imperative: startConsumer()
+
+```typescript
+@Injectable()
+export class OrdersService implements OnModuleInit {
+  constructor(
+    @InjectKafkaClient()
+    private readonly kafka: KafkaClient<OrdersTopicMap>,
+  ) {}
 
-  async listen() {
+  async onModuleInit() {
     await this.kafka.startConsumer(
-      ['order.created'],
+      ['order.created', 'order.completed'],
       async (message, topic) => {
-        console.log(message.orderId); // typed
+        console.log(`${topic}:`, message);
+      },
+      {
+        retry: { maxRetries: 3, backoffMs: 1000 },
+        dlq: true,
       },
     );
   }
 }
 ```
 
-### Consumer options
+## Multiple consumer groups
+
+Register multiple named clients for different bounded contexts:
 
 ```typescript
-await this.kafka.startConsumer(
-  ['order.created'],
-  handler,
+@Module({
+  imports: [
+    KafkaModule.register<OrdersTopicMap>({
+      name: 'orders',
+      clientId: 'orders-service',
+      groupId: 'orders-consumer',
+      brokers: ['localhost:9092'],
+    }),
+    KafkaModule.register<PaymentsTopicMap>({
+      name: 'payments',
+      clientId: 'payments-service',
+      groupId: 'payments-consumer',
+      brokers: ['localhost:9092'],
+    }),
+  ],
+})
+export class AppModule {}
+```
+
+Inject by name — the string in `@InjectKafkaClient()` must match the `name` from `register()`:
+
+```typescript
+@Injectable()
+export class OrdersService {
+  constructor(
+    @InjectKafkaClient('orders')    // ← matches name: 'orders' above
+    private readonly kafka: KafkaClient<OrdersTopicMap>,
+  ) {}
+}
+```
+
+Same with `@SubscribeTo()` — use `clientName` to target a specific named client:
+
+```typescript
+@SubscribeTo('payment.received', { clientName: 'payments' })  // ← matches name: 'payments'
+async handlePayment(message: PaymentsTopicMap['payment.received']) {
+  // ...
+}
+```
+
+## Partition key
+
+Route all events for the same order to the same partition:
+
+```typescript
+await this.kafka.sendMessage(
+  'order.created',
+  { orderId: '123', userId: '456', amount: 100 },
+  { key: '123' },
+);
+```
+
+## Message headers
+
+Attach metadata to messages:
+
+```typescript
+await this.kafka.sendMessage(
+  'order.created',
+  { orderId: '123', userId: '456', amount: 100 },
   {
-    fromBeginning: false, // default: false
-    autoCommit: true,     // default: true
+    key: '123',
+    headers: { 'x-correlation-id': 'abc-def', 'x-source': 'api-gateway' },
   },
 );
 ```
 
+Headers work with batch sending too:
+
+```typescript
+await this.kafka.sendBatch('order.created', [
+  {
+    value: { orderId: '1', userId: '10', amount: 50 },
+    key: '1',
+    headers: { 'x-correlation-id': 'req-1' },
+  },
+]);
+```
+
+## Batch sending
+
+```typescript
+await this.kafka.sendBatch('order.created', [
+  { value: { orderId: '1', userId: '10', amount: 50 }, key: '1' },
+  { value: { orderId: '2', userId: '20', amount: 75 }, key: '2' },
+  { value: { orderId: '3', userId: '30', amount: 100 }, key: '3' },
+]);
+```
+
+## Transactions
+
+Send multiple messages atomically with exactly-once semantics:
+
+```typescript
+await this.kafka.transaction(async (tx) => {
+  await tx.send('order.created', {
+    orderId: '123',
+    userId: '456',
+    amount: 100,
+  });
+  await tx.send('order.completed', {
+    orderId: '123',
+    completedAt: new Date().toISOString(),
+  });
+  // if anything throws, all messages are rolled back
+});
+```
+
+`tx.sendBatch()` is also available inside transactions.
+
+## Consumer interceptors
+
+Add before/after/onError hooks to message processing:
+
+```typescript
+import { ConsumerInterceptor } from '@drarzter/kafka-client';
+
+const loggingInterceptor: ConsumerInterceptor<OrdersTopicMap> = {
+  before: (message, topic) => {
+    console.log(`Processing ${topic}`, message);
+  },
+  after: (message, topic) => {
+    console.log(`Done ${topic}`);
+  },
+  onError: (message, topic, error) => {
+    console.error(`Failed ${topic}:`, error.message);
+  },
+};
+
+await this.kafka.startConsumer(['order.created'], handler, {
+  interceptors: [loggingInterceptor],
+});
+```
+
+Multiple interceptors run in order. All hooks are optional.
+
+## Consumer options
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| `fromBeginning` | `false` | Read from the beginning of the topic |
+| `autoCommit` | `true` | Auto-commit offsets |
+| `retry.maxRetries` | — | Number of retry attempts |
+| `retry.backoffMs` | `1000` | Base delay between retries (multiplied by attempt number) |
+| `dlq` | `false` | Send to `{topic}.dlq` after all retries exhausted |
+| `interceptors` | `[]` | Array of before/after/onError hooks |
+
+## Health check
+
+Monitor Kafka connectivity with the built-in health indicator:
+
+```typescript
+import { Injectable } from '@nestjs/common';
+import { InjectKafkaClient, KafkaClient, KafkaHealthIndicator } from '@drarzter/kafka-client';
+import { OrdersTopicMap } from './orders.types';
+
+@Injectable()
+export class HealthService {
+  private readonly health = new KafkaHealthIndicator();
+
+  constructor(
+    @InjectKafkaClient()
+    private readonly kafka: KafkaClient<OrdersTopicMap>,
+  ) {}
+
+  async checkKafka() {
+    return this.health.check(this.kafka);
+    // { status: 'up', clientId: 'my-service', topics: ['order.created', ...] }
+    // or { status: 'down', clientId: 'my-service', error: 'Connection refused' }
+  }
+}
+```
+
+## Project structure
+
+```
+src/
+├── client/         # KafkaClient, types, interfaces
+├── module/         # KafkaModule, KafkaExplorer, DI constants
+├── decorators/     # @InjectKafkaClient(), @SubscribeTo()
+├── health/         # KafkaHealthIndicator
+└── index.ts        # Public API re-exports
+```
+
+All exported types and methods have JSDoc comments — your IDE will show inline docs and autocomplete.
+
 ## License
 
-MIT
+[MIT](LICENSE)

package/dist/index.d.mts

@@ -1,61 +1,200 @@
-import { OnModuleDestroy, DynamicModule } from '@nestjs/common';
+import { DynamicModule, OnModuleInit } from '@nestjs/common';
+import { DiscoveryService, ModuleRef } from '@nestjs/core';
 
+/**
+ * Mapping of topic names to their message types.
+ * Define this interface to get type-safe publish/subscribe across your app.
+ *
+ * @example
+ * ```ts
+ * interface MyTopics extends TTopicMessageMap {
+ *   "orders.created": { orderId: string; amount: number };
+ *   "users.updated": { userId: string; name: string };
+ * }
+ * ```
+ */
 type TTopicMessageMap = {
     [topic: string]: Record<string, any>;
 };
 type ClientId = string;
 type GroupId = string;
-interface ConsumerOptions {
+type MessageHeaders = Record<string, string>;
+/** Options for sending a single message. */
+interface SendOptions {
+    /** Partition key for message routing. */
+    key?: string;
+    /** Custom headers attached to the message. */
+    headers?: MessageHeaders;
+}
+/** Options for configuring a Kafka consumer. */
+interface ConsumerOptions<T extends TTopicMessageMap = TTopicMessageMap> {
+    /** Start reading from earliest offset. Default: `false`. */
     fromBeginning?: boolean;
+    /** Automatically commit offsets. Default: `true`. */
     autoCommit?: boolean;
+    /** Retry policy for failed message processing. */
+    retry?: RetryOptions;
+    /** Send failed messages to a Dead Letter Queue (`<topic>.dlq`). */
+    dlq?: boolean;
+    /** Interceptors called before/after each message. */
+    interceptors?: ConsumerInterceptor<T>[];
+}
+/** Configuration for consumer retry behavior. */
+interface RetryOptions {
+    /** Maximum number of retry attempts before giving up. */
+    maxRetries: number;
+    /** Base delay between retries in ms (multiplied by attempt number). Default: `1000`. */
+    backoffMs?: number;
+}
+/**
+ * Interceptor hooks for consumer message processing.
+ * All methods are optional — implement only what you need.
+ */
+interface ConsumerInterceptor<T extends TTopicMessageMap = TTopicMessageMap> {
+    /** Called before the message handler. */
+    before?(message: T[keyof T], topic: string): Promise<void> | void;
+    /** Called after the message handler succeeds. */
+    after?(message: T[keyof T], topic: string): Promise<void> | void;
+    /** Called when the message handler throws. */
+    onError?(message: T[keyof T], topic: string, error: Error): Promise<void> | void;
+}
+/** Context passed to the `transaction()` callback with type-safe send methods. */
+interface TransactionContext<T extends TTopicMessageMap> {
+    send<K extends keyof T>(topic: K, message: T[K], options?: SendOptions): Promise<void>;
+    sendBatch<K extends keyof T>(topic: K, messages: Array<{
+        value: T[K];
+        key?: string;
+        headers?: MessageHeaders;
+    }>): Promise<void>;
 }
+/** Interface describing all public methods of the Kafka client. */
 interface IKafkaClient<T extends TTopicMessageMap> {
     checkStatus(): Promise<{
         topics: string[];
     }>;
-    startConsumer<K extends Array<keyof T>>(topics: K, handleMessage: (message: T[K[number]], topic: K[number]) => Promise<void>, options?: ConsumerOptions): Promise<void>;
+    startConsumer<K extends Array<keyof T>>(topics: K, handleMessage: (message: T[K[number]], topic: K[number]) => Promise<void>, options?: ConsumerOptions<T>): Promise<void>;
     stopConsumer(): Promise<void>;
-    sendMessage<K extends keyof T>(topic: K, message: T[K]): Promise<void>;
+    sendMessage<K extends keyof T>(topic: K, message: T[K], options?: SendOptions): Promise<void>;
+    sendBatch<K extends keyof T>(topic: K, messages: Array<{
+        value: T[K];
+        key?: string;
+        headers?: MessageHeaders;
+    }>): Promise<void>;
+    transaction(fn: (ctx: TransactionContext<T>) => Promise<void>): Promise<void>;
     getClientId: () => ClientId;
+    disconnect(): Promise<void>;
 }
+/**
+ * Type-safe Kafka client for NestJS.
+ * Wraps kafkajs with JSON serialization, retries, DLQ, transactions, and interceptors.
+ *
+ * @typeParam T - Topic-to-message type mapping for compile-time safety.
+ */
 declare class KafkaClient<T extends TTopicMessageMap> implements IKafkaClient<T> {
     private readonly kafka;
     private readonly producer;
     private readonly consumer;
     private readonly admin;
     private readonly logger;
+    private isConsumerRunning;
     readonly clientId: ClientId;
     constructor(clientId: ClientId, groupId: GroupId, brokers: string[]);
-    sendMessage<K extends keyof T>(topic: K, message: T[K]): Promise<void>;
+    /** Send a single typed message to a topic. */
+    sendMessage<K extends keyof T>(topic: K, message: T[K], options?: SendOptions): Promise<void>;
+    /** Send multiple typed messages to a topic in one call. */
+    sendBatch<K extends keyof T>(topic: K, messages: Array<{
+        value: T[K];
+        key?: string;
+        headers?: MessageHeaders;
+    }>): Promise<void>;
+    /** Execute multiple sends atomically. Commits on success, aborts on error. */
+    transaction(fn: (ctx: TransactionContext<T>) => Promise<void>): Promise<void>;
+    /** Connect the idempotent producer. Called automatically by `KafkaModule.register()`. */
     connectProducer(): Promise<void>;
     disconnectProducer(): Promise<void>;
-    startConsumer<K extends Array<keyof T>>(topics: K, handleMessage: (message: T[K[number]], topic: K[number]) => Promise<void>, options?: ConsumerOptions): Promise<void>;
+    /** Subscribe to topics and start consuming messages with the given handler. */
+    startConsumer<K extends Array<keyof T>>(topics: K, handleMessage: (message: T[K[number]], topic: K[number]) => Promise<void>, options?: ConsumerOptions<T>): Promise<void>;
     stopConsumer(): Promise<void>;
+    /** Check broker connectivity and return available topics. */
     checkStatus(): Promise<{
         topics: string[];
     }>;
     getClientId(): ClientId;
+    /** Gracefully disconnect producer and consumer. */
     disconnect(): Promise<void>;
+    private sendToDlq;
+    private sleep;
 }
 
+/** Synchronous configuration for `KafkaModule.register()`. */
 interface KafkaModuleOptions {
+    /** Optional name for multi-client setups. Must match `@InjectKafkaClient(name)`. */
+    name?: string;
+    /** Unique Kafka client identifier. */
     clientId: ClientId;
+    /** Consumer group identifier. */
     groupId: GroupId;
+    /** List of Kafka broker addresses. */
     brokers: string[];
 }
+/** Async configuration for `KafkaModule.registerAsync()` with dependency injection. */
 interface KafkaModuleAsyncOptions {
+    name?: string;
     imports?: any[];
     useFactory: (...args: any[]) => KafkaModuleOptions | Promise<KafkaModuleOptions>;
     inject?: any[];
 }
-declare class KafkaModule implements OnModuleDestroy {
-    private readonly client?;
-    constructor(client?: KafkaClient<any> | undefined);
-    onModuleDestroy(): Promise<void>;
+/**
+ * NestJS dynamic module for registering type-safe Kafka clients.
+ * Use `register()` for static config or `registerAsync()` for DI-based config.
+ */
+declare class KafkaModule {
+    /** Register a Kafka client with static options. */
     static register<T extends TTopicMessageMap>(options: KafkaModuleOptions): DynamicModule;
+    /** Register a Kafka client with async/factory-based options. */
     static registerAsync<T extends TTopicMessageMap>(asyncOptions: KafkaModuleAsyncOptions): DynamicModule;
 }
 
+/** Default DI token for the Kafka client. */
 declare const KAFKA_CLIENT = "KAFKA_CLIENT";
+/** Returns the DI token for a named (or default) Kafka client instance. */
+declare const getKafkaClientToken: (name?: string) => string;
+
+declare const KAFKA_SUBSCRIBER_METADATA = "KAFKA_SUBSCRIBER_METADATA";
+interface KafkaSubscriberMetadata {
+    topics: string[];
+    options?: ConsumerOptions;
+    clientName?: string;
+}
+/** Inject a `KafkaClient` instance. Pass a name to target a specific named client. */
+declare const InjectKafkaClient: (name?: string) => ParameterDecorator;
+/**
+ * Decorator that auto-subscribes a method to Kafka topics on module init.
+ * The decorated method receives `(message, topic)` for each consumed message.
+ */
+declare const SubscribeTo: (topics: string | string[], options?: ConsumerOptions & {
+    clientName?: string;
+}) => MethodDecorator;
+
+/** Discovers `@SubscribeTo()` decorators and wires them to their Kafka clients on startup. */
+declare class KafkaExplorer implements OnModuleInit {
+    private readonly discoveryService;
+    private readonly moduleRef;
+    private readonly logger;
+    constructor(discoveryService: DiscoveryService, moduleRef: ModuleRef);
+    onModuleInit(): Promise<void>;
+}
+
+/** Result returned by `KafkaHealthIndicator.check()`. */
+interface KafkaHealthResult {
+    status: "up" | "down";
+    clientId: string;
+    topics?: string[];
+    error?: string;
+}
+/** Health check service. Call `check(client)` to verify broker connectivity. */
+declare class KafkaHealthIndicator {
+    check<T extends TTopicMessageMap>(client: KafkaClient<T>): Promise<KafkaHealthResult>;
+}
 
-export { type ClientId, type ConsumerOptions, type GroupId, type IKafkaClient, KAFKA_CLIENT, KafkaClient, KafkaModule, type KafkaModuleAsyncOptions, type KafkaModuleOptions, type TTopicMessageMap };
+export { type ClientId, type ConsumerInterceptor, type ConsumerOptions, type GroupId, type IKafkaClient, InjectKafkaClient, KAFKA_CLIENT, KAFKA_SUBSCRIBER_METADATA, KafkaClient, KafkaExplorer, KafkaHealthIndicator, type KafkaHealthResult, KafkaModule, type KafkaModuleAsyncOptions, type KafkaModuleOptions, type KafkaSubscriberMetadata, type MessageHeaders, type RetryOptions, type SendOptions, SubscribeTo, type TTopicMessageMap, type TransactionContext, getKafkaClientToken };