@drarzter/kafka-client 0.1.9 → 0.2.1

package/README.md

@@ -18,6 +18,7 @@ An opinionated wrapper around kafkajs that integrates with NestJS as a DynamicMo
 - **Idempotent producer** — `acks: -1`, `idempotent: true` by default
 - **Retry + DLQ** — configurable retries with backoff, dead letter queue for failed messages
 - **Batch sending** — send multiple messages in a single request
+- **Batch consuming** — `startBatchConsumer()` for high-throughput `eachBatch` processing
 - **Partition key support** — route related messages to the same partition
 - **Custom headers** — attach metadata headers to messages
 - **Transactions** — exactly-once semantics with `producer.transaction()`
@@ -314,6 +315,31 @@ export class OrdersService implements OnModuleInit {
 
 ## Multiple consumer groups
 
+### Per-consumer groupId
+
+Override the default consumer group for specific consumers. Each unique `groupId` creates a separate kafkajs Consumer internally:
+
+```typescript
+// Default group from constructor
+await kafka.startConsumer(['orders'], handler);
+
+// Custom group — receives its own copy of messages
+await kafka.startConsumer(['orders'], auditHandler, { groupId: 'orders-audit' });
+
+// Works with @SubscribeTo too
+@SubscribeTo('orders', { groupId: 'orders-audit' })
+async auditOrders(message) { ... }
+```
+
+**Important:** You cannot mix `eachMessage` and `eachBatch` consumers on the same `groupId`. The library throws a clear error if you try:
+
+```text
+Cannot use eachBatch on consumer group "my-group" — it is already running with eachMessage.
+Use a different groupId for this consumer.
+```
+
+### Named clients
+
 Register multiple named clients for different bounded contexts:
 
 ```typescript
@@ -406,6 +432,38 @@ await this.kafka.sendBatch('order.created', [
 ]);
 ```
 
+## Batch consuming
+
+Process messages in batches for higher throughput. The handler receives an array of parsed messages and a `BatchMeta` object with offset management controls:
+
+```typescript
+await this.kafka.startBatchConsumer(
+  ['order.created'],
+  async (messages, topic, meta) => {
+    // messages: OrdersTopicMap['order.created'][]
+    for (const msg of messages) {
+      await processOrder(msg);
+      meta.resolveOffset(/* ... */);
+    }
+    await meta.commitOffsetsIfNecessary();
+  },
+  { retry: { maxRetries: 3 }, dlq: true },
+);
+```
+
+With `@SubscribeTo()`:
+
+```typescript
+@SubscribeTo('order.created', { batch: true })
+async handleOrders(messages: OrdersTopicMap['order.created'][], topic: string) {
+  // messages is an array
+}
+```
+
+Schema validation runs per-message — invalid messages are skipped (DLQ'd if enabled), valid ones are passed to the handler. Retry applies to the whole batch.
+
+`BatchMeta` exposes: `partition`, `highWatermark`, `heartbeat()`, `resolveOffset(offset)`, `commitOffsetsIfNecessary()`.
+
 ## Transactions
 
 Send multiple messages atomically with exactly-once semantics:
@@ -453,27 +511,86 @@ await this.kafka.startConsumer(['order.created'], handler, {
 
 Multiple interceptors run in order. All hooks are optional.
 
-## Consumer options
+## Options reference
+
+### Send options
+
+Options for `sendMessage()` — the third argument:
+
+| Option    | Default | Description                                      |
+|-----------|---------|--------------------------------------------------|
+| `key`     | —       | Partition key for message routing                 |
+| `headers` | —       | Custom metadata headers (`Record<string, string>`) |
+
+`sendBatch()` accepts `key` and `headers` per message inside the array items.
+
+### Consumer options
 
 | Option | Default | Description |
 |--------|---------|-------------|
+| `groupId` | constructor value | Override consumer group for this subscription |
 | `fromBeginning` | `false` | Read from the beginning of the topic |
 | `autoCommit` | `true` | Auto-commit offsets |
 | `retry.maxRetries` | — | Number of retry attempts |
 | `retry.backoffMs` | `1000` | Base delay between retries (multiplied by attempt number) |
 | `dlq` | `false` | Send to `{topic}.dlq` after all retries exhausted |
 | `interceptors` | `[]` | Array of before/after/onError hooks |
+| `batch` | `false` | (decorator only) Use `startBatchConsumer` instead of `startConsumer` |
+| `subscribeRetry.retries` | `5` | Max attempts for `consumer.subscribe()` when topic doesn't exist yet |
+| `subscribeRetry.backoffMs` | `5000` | Delay between subscribe retry attempts (ms) |
 
 ### Module options
 
+Passed to `KafkaModule.register()` or returned from `registerAsync()` factory:
+
 | Option | Default | Description |
 |--------|---------|-------------|
-| `clientId` | — | Kafka client identifier |
-| `groupId` | — | Consumer group ID |
-| `brokers` | — | Array of broker addresses |
-| `name` | — | Named client for multi-group setups |
-| `isGlobal` | `false` | Make the client available in all modules |
-| `autoCreateTopics` | `false` | Auto-create topics on first send/consume |
+| `clientId` | — | Kafka client identifier (required) |
+| `groupId` | — | Default consumer group ID (required) |
+| `brokers` | — | Array of broker addresses (required) |
+| `name` | — | Named client identifier for multi-client setups |
+| `isGlobal` | `false` | Make the client available in all modules without re-importing |
+| `autoCreateTopics` | `false` | Auto-create topics on first send (dev only) |
+| `strictSchemas` | `true` | Validate string topic keys against schemas registered via TopicDescriptor |
+
+**Module-scoped** (default) — import `KafkaModule` in each module that needs it:
+
+```typescript
+// orders.module.ts
+@Module({
+  imports: [
+    KafkaModule.register<OrdersTopicMap>({
+      clientId: 'orders',
+      groupId: 'orders-group',
+      brokers: ['localhost:9092'],
+    }),
+  ],
+})
+export class OrdersModule {}
+```
+
+**App-wide** — register once in `AppModule` with `isGlobal: true`, inject anywhere:
+
+```typescript
+// app.module.ts
+@Module({
+  imports: [
+    KafkaModule.register<MyTopics>({
+      clientId: 'my-app',
+      groupId: 'my-group',
+      brokers: ['localhost:9092'],
+      isGlobal: true,
+    }),
+  ],
+})
+export class AppModule {}
+
+// any module — no KafkaModule import needed
+@Injectable()
+export class PaymentService {
+  constructor(@InjectKafkaClient() private readonly kafka: KafkaClient<MyTopics>) {}
+}
+```
 
 ## Error classes
 
@@ -508,6 +625,99 @@ const interceptor: ConsumerInterceptor<MyTopics> = {
 
 When `retry.maxRetries` is set and all attempts fail, `KafkaRetryExhaustedError` is passed to `onError` interceptors automatically.
 
+**`KafkaValidationError`** — thrown when schema validation fails on the consumer side. Has `topic`, `originalMessage`, and `cause`:
+
+```typescript
+import { KafkaValidationError } from '@drarzter/kafka-client';
+
+const interceptor: ConsumerInterceptor<MyTopics> = {
+  onError: (message, topic, error) => {
+    if (error instanceof KafkaValidationError) {
+      console.log(`Bad message on ${error.topic}:`, error.cause?.message);
+    }
+  },
+};
+```
+
+## Schema validation
+
+Add runtime message validation using any library with a `.parse()` method — Zod, Valibot, ArkType, or a custom validator. No extra dependency required.
+
+### Defining topics with schemas
+
+```typescript
+import { topic, TopicsFrom } from '@drarzter/kafka-client';
+import { z } from 'zod';  // or valibot, arktype, etc.
+
+// Schema-validated — type inferred from schema, no generic needed
+export const OrderCreated = topic('order.created').schema(z.object({
+  orderId: z.string(),
+  userId: z.string(),
+  amount: z.number().positive(),
+}));
+
+// Without schema — explicit generic (still works)
+export const OrderAudit = topic('order.audit')<{ orderId: string; action: string }>();
+
+export type MyTopics = TopicsFrom<typeof OrderCreated | typeof OrderAudit>;
+```
+
+### How it works
+
+**On send** — `sendMessage`, `sendBatch`, and `transaction` call `schema.parse(message)` before serializing. Invalid messages throw immediately (the schema library's error, e.g. `ZodError`):
+
+```typescript
+// This throws ZodError — amount must be positive
+await kafka.sendMessage(OrderCreated, { orderId: '1', userId: '2', amount: -5 });
+```
+
+**On consume** — after `JSON.parse`, the consumer validates each message against the schema. Invalid messages are:
+
+1. Logged as errors
+2. Sent to DLQ if `dlq: true`
+3. Passed to `onError` interceptors as `KafkaValidationError`
+4. Skipped (handler is NOT called)
+
+```typescript
+@SubscribeTo(OrderCreated, { dlq: true })
+async handleOrder(message) {
+  // `message` is guaranteed to match the schema
+  console.log(message.orderId); // string — validated at runtime
+}
+```
+
+### Strict schema mode
+
+By default (`strictSchemas: true`), once a schema is registered via a TopicDescriptor, string topic keys are also validated against it:
+
+```typescript
+// First call registers the schema in the internal registry
+await kafka.sendMessage(OrderCreated, { orderId: '1', userId: '2', amount: 100 });
+
+// Now this is ALSO validated — throws if data doesn't match OrderCreated's schema
+await kafka.sendMessage('order.created', { orderId: 123, userId: null, amount: -5 });
+```
+
+Disable with `strictSchemas: false` in `KafkaModule.register()` options if you want the old behavior (string topics bypass validation).
+
+### Bring your own validator
+
+Any object with `parse(data: unknown): T` works:
+
+```typescript
+import { SchemaLike } from '@drarzter/kafka-client';
+
+const customValidator: SchemaLike<{ id: string }> = {
+  parse(data: unknown) {
+    const d = data as any;
+    if (typeof d?.id !== 'string') throw new Error('id must be a string');
+    return { id: d.id };
+  },
+};
+
+const MyTopic = topic('my.topic').schema(customValidator);
+```
+
 ## Health check
 
 Monitor Kafka connectivity with the built-in health indicator:

package/dist/index.d.mts

@@ -1,6 +1,21 @@
 import { DynamicModule, OnModuleInit } from '@nestjs/common';
 import { DiscoveryService, ModuleRef } from '@nestjs/core';
 
+/**
+ * Any validation library with a `.parse()` method.
+ * Works with Zod, Valibot, ArkType, or any custom validator.
+ *
+ * @example
+ * ```ts
+ * import { z } from 'zod';
+ * const schema: SchemaLike<{ id: string }> = z.object({ id: z.string() });
+ * ```
+ */
+interface SchemaLike<T = any> {
+    parse(data: unknown): T;
+}
+/** Infer the output type from a SchemaLike. */
+type InferSchema<S extends SchemaLike> = S extends SchemaLike<infer T> ? T : never;
 /**
  * A typed topic descriptor that pairs a topic name with its message type.
  * Created via the `topic()` factory function.
@@ -12,14 +27,23 @@ interface TopicDescriptor<N extends string = string, M extends Record<string, an
     readonly __topic: N;
     /** @internal Phantom type — never has a real value at runtime. */
     readonly __type: M;
+    /** Runtime schema validator. Present only when created via `topic().schema()`. */
+    readonly __schema?: SchemaLike<M>;
 }
 /**
  * Define a typed topic descriptor.
  *
  * @example
  * ```ts
+ * // Without schema — type provided explicitly:
  * const OrderCreated = topic('order.created')<{ orderId: string; amount: number }>();
  *
+ * // With schema — type inferred from schema:
+ * const OrderCreated = topic('order.created').schema(z.object({
+ *   orderId: z.string(),
+ *   amount: z.number(),
+ * }));
+ *
  * // Use with KafkaClient:
  * await kafka.sendMessage(OrderCreated, { orderId: '123', amount: 100 });
  *
@@ -28,7 +52,10 @@ interface TopicDescriptor<N extends string = string, M extends Record<string, an
  * async handleOrder(msg) { ... }
  * ```
  */
-declare function topic<N extends string>(name: N): <M extends Record<string, any>>() => TopicDescriptor<N, M>;
+declare function topic<N extends string>(name: N): {
+    <M extends Record<string, any>>(): TopicDescriptor<N, M>;
+    schema<S extends SchemaLike<Record<string, any>>>(schema: S): TopicDescriptor<N, InferSchema<S>>;
+};
 /**
  * Build a topic-message map type from a union of TopicDescriptors.
  *
@@ -83,8 +110,23 @@ interface SendOptions {
     /** Custom headers attached to the message. */
     headers?: MessageHeaders;
 }
+/** Metadata exposed to batch consumer handlers. */
+interface BatchMeta {
+    /** Partition number for this batch. */
+    partition: number;
+    /** Highest offset available on the broker for this partition. */
+    highWatermark: string;
+    /** Send a heartbeat to the broker to prevent session timeout. */
+    heartbeat(): Promise<void>;
+    /** Mark an offset as processed (for manual offset management). */
+    resolveOffset(offset: string): void;
+    /** Commit offsets if the auto-commit threshold has been reached. */
+    commitOffsetsIfNecessary(): Promise<void>;
+}
 /** Options for configuring a Kafka consumer. */
 interface ConsumerOptions<T extends TopicMapConstraint<T> = TTopicMessageMap> {
+    /** Override the default consumer group ID from the constructor. */
+    groupId?: string;
     /** Start reading from earliest offset. Default: `false`. */
     fromBeginning?: boolean;
     /** Automatically commit offsets. Default: `true`. */
@@ -95,6 +137,10 @@ interface ConsumerOptions<T extends TopicMapConstraint<T> = TTopicMessageMap> {
     dlq?: boolean;
     /** Interceptors called before/after each message. */
     interceptors?: ConsumerInterceptor<T>[];
+    /** @internal Schema map populated by @SubscribeTo when descriptors have schemas. */
+    schemas?: Map<string, SchemaLike>;
+    /** Retry config for `consumer.subscribe()` when the topic doesn't exist yet. */
+    subscribeRetry?: SubscribeRetryOptions;
 }
 /** Configuration for consumer retry behavior. */
 interface RetryOptions {
@@ -137,6 +183,8 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
     }>;
     startConsumer<K extends Array<keyof T>>(topics: K, handleMessage: (message: T[K[number]], topic: K[number]) => Promise<void>, options?: ConsumerOptions<T>): Promise<void>;
     startConsumer<D extends TopicDescriptor<string & keyof T, T[string & keyof T]>>(topics: D[], handleMessage: (message: D["__type"], topic: D["__topic"]) => Promise<void>, options?: ConsumerOptions<T>): Promise<void>;
+    startBatchConsumer<K extends Array<keyof T>>(topics: K, handleBatch: (messages: T[K[number]][], topic: K[number], meta: BatchMeta) => Promise<void>, options?: ConsumerOptions<T>): Promise<void>;
+    startBatchConsumer<D extends TopicDescriptor<string & keyof T, T[string & keyof T]>>(topics: D[], handleBatch: (messages: D["__type"][], topic: D["__topic"], meta: BatchMeta) => Promise<void>, options?: ConsumerOptions<T>): Promise<void>;
     stopConsumer(): Promise<void>;
     sendMessage<K extends keyof T>(topic: K, message: T[K], options?: SendOptions): Promise<void>;
     sendBatch<K extends keyof T>(topic: K, messages: Array<{
@@ -150,8 +198,17 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
 }
 /** Options for `KafkaClient` constructor. */
 interface KafkaClientOptions {
-    /** Auto-create topics via admin before the first `sendMessage`, `sendBatch`, `transaction`, or `startConsumer` for each topic. Useful for development — not recommended in production. */
+    /** Auto-create topics via admin before the first `sendMessage`, `sendBatch`, or `transaction` for each topic. Useful for development — not recommended in production. */
     autoCreateTopics?: boolean;
+    /** When `true`, string topic keys are validated against any schema previously registered via a TopicDescriptor. Default: `true`. */
+    strictSchemas?: boolean;
+}
+/** Options for consumer subscribe retry when topic doesn't exist yet. */
+interface SubscribeRetryOptions {
+    /** Maximum number of subscribe attempts. Default: `5`. */
+    retries?: number;
+    /** Delay between retries in ms. Default: `5000`. */
+    backoffMs?: number;
 }
 
 /**
@@ -163,16 +220,22 @@ interface KafkaClientOptions {
 declare class KafkaClient<T extends TopicMapConstraint<T>> implements IKafkaClient<T> {
     private readonly kafka;
     private readonly producer;
-    private readonly consumer;
+    private readonly consumers;
     private readonly admin;
     private readonly logger;
     private readonly autoCreateTopicsEnabled;
+    private readonly strictSchemasEnabled;
     private readonly ensuredTopics;
+    private readonly defaultGroupId;
+    private readonly schemaRegistry;
+    private readonly runningConsumers;
     private isAdminConnected;
     readonly clientId: ClientId;
     constructor(clientId: ClientId, groupId: GroupId, brokers: string[], options?: KafkaClientOptions);
+    private getOrCreateConsumer;
     private resolveTopicName;
     private ensureTopic;
+    private validateMessage;
     /** Send a single typed message. Accepts a topic key or a TopicDescriptor. */
     sendMessage<D extends TopicDescriptor<string & keyof T, T[string & keyof T]>>(descriptor: D, message: D["__type"], options?: SendOptions): Promise<void>;
     sendMessage<K extends keyof T>(topic: K, message: T[K], options?: SendOptions): Promise<void>;
@@ -195,16 +258,21 @@ declare class KafkaClient<T extends TopicMapConstraint<T>> implements IKafkaClie
     /** Subscribe to topics and start consuming messages with the given handler. */
     startConsumer<K extends Array<keyof T>>(topics: K, handleMessage: (message: T[K[number]], topic: K[number]) => Promise<void>, options?: ConsumerOptions<T>): Promise<void>;
     startConsumer<D extends TopicDescriptor<string & keyof T, T[string & keyof T]>>(topics: D[], handleMessage: (message: D["__type"], topic: D["__topic"]) => Promise<void>, options?: ConsumerOptions<T>): Promise<void>;
+    /** Subscribe to topics and consume messages in batches. */
+    startBatchConsumer<K extends Array<keyof T>>(topics: K, handleBatch: (messages: T[K[number]][], topic: K[number], meta: BatchMeta) => Promise<void>, options?: ConsumerOptions<T>): Promise<void>;
+    startBatchConsumer<D extends TopicDescriptor<string & keyof T, T[string & keyof T]>>(topics: D[], handleBatch: (messages: D["__type"][], topic: D["__topic"], meta: BatchMeta) => Promise<void>, options?: ConsumerOptions<T>): Promise<void>;
     stopConsumer(): Promise<void>;
     /** Check broker connectivity and return available topics. */
     checkStatus(): Promise<{
         topics: string[];
     }>;
     getClientId(): ClientId;
-    /** Gracefully disconnect producer, consumer, and admin. */
+    /** Gracefully disconnect producer, all consumers, and admin. */
     disconnect(): Promise<void>;
+    private buildSchemaMap;
     private processMessage;
     private sendToDlq;
+    private subscribeWithRetry;
     private sleep;
 }
 
@@ -254,6 +322,15 @@ declare class KafkaProcessingError extends Error {
         cause?: Error;
     });
 }
+/** Error thrown when schema validation fails on send or consume. */
+declare class KafkaValidationError extends Error {
+    readonly topic: string;
+    readonly originalMessage: unknown;
+    readonly cause?: Error;
+    constructor(topic: string, originalMessage: unknown, options?: {
+        cause?: Error;
+    });
+}
 /** Error thrown when all retry attempts are exhausted for a message. */
 declare class KafkaRetryExhaustedError extends KafkaProcessingError {
     readonly attempts: number;
@@ -270,8 +347,11 @@ declare const getKafkaClientToken: (name?: string) => string;
 declare const KAFKA_SUBSCRIBER_METADATA = "KAFKA_SUBSCRIBER_METADATA";
 interface KafkaSubscriberMetadata {
     topics: string[];
+    schemas?: Map<string, SchemaLike>;
     options?: ConsumerOptions;
     clientName?: string;
+    batch?: boolean;
+    methodName?: string | symbol;
 }
 /** Inject a `KafkaClient` instance. Pass a name to target a specific named client. */
 declare const InjectKafkaClient: (name?: string) => ParameterDecorator;
@@ -281,6 +361,7 @@ declare const InjectKafkaClient: (name?: string) => ParameterDecorator;
  */
 declare const SubscribeTo: (topics: string | string[] | TopicDescriptor | TopicDescriptor[] | (string | TopicDescriptor)[], options?: ConsumerOptions & {
     clientName?: string;
+    batch?: boolean;
 }) => MethodDecorator;
 
 /** Discovers `@SubscribeTo()` decorators and wires them to their Kafka clients on startup. */
@@ -304,4 +385,4 @@ declare class KafkaHealthIndicator {
     check<T extends TopicMapConstraint<T>>(client: KafkaClient<T>): Promise<KafkaHealthResult>;
 }
 
-export { type ClientId, type ConsumerInterceptor, type ConsumerOptions, type GroupId, type IKafkaClient, InjectKafkaClient, KAFKA_CLIENT, KAFKA_SUBSCRIBER_METADATA, KafkaClient, type KafkaClientOptions, KafkaExplorer, KafkaHealthIndicator, type KafkaHealthResult, KafkaModule, type KafkaModuleAsyncOptions, type KafkaModuleOptions, KafkaProcessingError, KafkaRetryExhaustedError, type KafkaSubscriberMetadata, type MessageHeaders, type RetryOptions, type SendOptions, SubscribeTo, type TTopicMessageMap, type TopicDescriptor, type TopicMapConstraint, type TopicsFrom, type TransactionContext, getKafkaClientToken, topic };
+export { type BatchMeta, type ClientId, type ConsumerInterceptor, type ConsumerOptions, type GroupId, type IKafkaClient, type InferSchema, InjectKafkaClient, KAFKA_CLIENT, KAFKA_SUBSCRIBER_METADATA, KafkaClient, type KafkaClientOptions, KafkaExplorer, KafkaHealthIndicator, type KafkaHealthResult, KafkaModule, type KafkaModuleAsyncOptions, type KafkaModuleOptions, KafkaProcessingError, KafkaRetryExhaustedError, type KafkaSubscriberMetadata, KafkaValidationError, type MessageHeaders, type RetryOptions, type SchemaLike, type SendOptions, type SubscribeRetryOptions, SubscribeTo, type TTopicMessageMap, type TopicDescriptor, type TopicMapConstraint, type TopicsFrom, type TransactionContext, getKafkaClientToken, topic };