@drarzter/kafka-client 0.2.1 → 0.3.0

package/README.md

@@ -4,17 +4,17 @@
 [![CI](https://github.com/drarzter/kafka-client/actions/workflows/publish.yml/badge.svg)](https://github.com/drarzter/kafka-client/actions/workflows/publish.yml)
 [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
 
-Type-safe Kafka client wrapper for NestJS. Built on top of [kafkajs](https://kafka.js.org/).
+Type-safe Kafka client for Node.js. Framework-agnostic core with a first-class NestJS adapter. Built on top of [kafkajs](https://kafka.js.org/).
 
 ## What is this?
 
-An opinionated wrapper around kafkajs that integrates with NestJS as a DynamicModule. Not a full-featured framework — just a clean, typed abstraction for producing and consuming Kafka messages.
+An opinionated, type-safe abstraction over kafkajs. Works standalone (Express, Fastify, raw Node) or as a NestJS DynamicModule. Not a full-featured framework — just a clean, typed layer for producing and consuming Kafka messages.
 
 ## Why?
 
 - **Typed topics** — you define a map of topic -> message shape, and the compiler won't let you send wrong data to wrong topic
 - **Topic descriptors** — `topic()` DX sugar lets you define topics as standalone typed objects instead of string keys
-- **NestJS-native** — `register()` / `registerAsync()`, DI injection, lifecycle hooks out of the box
+- **Framework-agnostic** — use standalone or with NestJS (`register()` / `registerAsync()`, DI, lifecycle hooks)
 - **Idempotent producer** — `acks: -1`, `idempotent: true` by default
 - **Retry + DLQ** — configurable retries with backoff, dead letter queue for failed messages
 - **Batch sending** — send multiple messages in a single request
@@ -29,17 +29,43 @@ An opinionated wrapper around kafkajs that integrates with NestJS as a DynamicMo
 - **Multiple consumer groups** — named clients for different bounded contexts
 - **Declarative & imperative** — use `@SubscribeTo()` decorator or `startConsumer()` directly
 
+See the [Roadmap](./ROADMAP.md) for upcoming features and version history.
+
 ## Installation
 
 ```bash
 npm install @drarzter/kafka-client
-# or
-pnpm add @drarzter/kafka-client
 ```
 
-Peer dependencies: `@nestjs/common`, `@nestjs/core`, `reflect-metadata`, `rxjs`
+For NestJS projects, install peer dependencies: `@nestjs/common`, `@nestjs/core`, `reflect-metadata`, `rxjs`.
+
+For standalone usage (Express, Fastify, raw Node), no extra dependencies needed — import from `@drarzter/kafka-client/core`.
+
+## Standalone usage (no NestJS)
+
+```typescript
+import { KafkaClient, topic } from '@drarzter/kafka-client/core';
+
+const OrderCreated = topic('order.created')<{ orderId: string; amount: number }>();
+
+const kafka = new KafkaClient('my-app', 'my-group', ['localhost:9092']);
+await kafka.connectProducer();
+
+// Send
+await kafka.sendMessage(OrderCreated, { orderId: '123', amount: 100 });
+
+// Consume
+await kafka.startConsumer([OrderCreated], async (message, topic) => {
+  console.log(`${topic}:`, message.orderId);
+});
+
+// Custom logger (winston, pino, etc.)
+const kafka2 = new KafkaClient('my-app', 'my-group', ['localhost:9092'], {
+  logger: myWinstonLogger,
+});
+```
 
-## Quick start
+## Quick start (NestJS)
 
 Send and receive a message in 3 files:
 
@@ -551,6 +577,7 @@ Passed to `KafkaModule.register()` or returned from `registerAsync()` factory:
 | `name` | — | Named client identifier for multi-client setups |
 | `isGlobal` | `false` | Make the client available in all modules without re-importing |
 | `autoCreateTopics` | `false` | Auto-create topics on first send (dev only) |
+| `numPartitions` | `1` | Number of partitions for auto-created topics |
 | `strictSchemas` | `true` | Validate string topic keys against schemas registered via TopicDescriptor |
 
 **Module-scoped** (default) — import `KafkaModule` in each module that needs it:
@@ -766,11 +793,10 @@ Both suites run in CI on every push to `main`.
 
 ```
 src/
-├── client/         # KafkaClient, types, topic(), error classes
-├── module/         # KafkaModule, KafkaExplorer, DI constants
-├── decorators/     # @InjectKafkaClient(), @SubscribeTo()
-├── health/         # KafkaHealthIndicator
-└── index.ts        # Public API re-exports
+├── client/         # Core — KafkaClient, types, topic(), error classes (0 framework deps)
+├── nest/           # NestJS adapter — Module, Explorer, decorators, health
+├── core.ts         # Standalone entrypoint (@drarzter/kafka-client/core)
+└── index.ts        # Full entrypoint — core + NestJS adapter
 ```
 
 All exported types and methods have JSDoc comments — your IDE will show inline docs and autocomplete.

package/dist/chunk-UDOHIMAZ.mjs

@@ -0,0 +1,545 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __decorateClass = (decorators, target, key, kind) => {
+  var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc(target, key) : target;
+  for (var i = decorators.length - 1, decorator; i >= 0; i--)
+    if (decorator = decorators[i])
+      result = (kind ? decorator(target, key, result) : decorator(result)) || result;
+  if (kind && result) __defProp(target, key, result);
+  return result;
+};
+var __decorateParam = (index, decorator) => (target, key) => decorator(target, key, index);
+
+// src/client/kafka.client.ts
+import { Kafka, Partitioners } from "kafkajs";
+
+// src/client/errors.ts
+var KafkaProcessingError = class extends Error {
+  constructor(message, topic2, originalMessage, options) {
+    super(message, options);
+    this.topic = topic2;
+    this.originalMessage = originalMessage;
+    this.name = "KafkaProcessingError";
+    if (options?.cause) this.cause = options.cause;
+  }
+};
+var KafkaValidationError = class extends Error {
+  constructor(topic2, originalMessage, options) {
+    super(`Schema validation failed for topic "${topic2}"`, options);
+    this.topic = topic2;
+    this.originalMessage = originalMessage;
+    this.name = "KafkaValidationError";
+    if (options?.cause) this.cause = options.cause;
+  }
+};
+var KafkaRetryExhaustedError = class extends KafkaProcessingError {
+  constructor(topic2, originalMessage, attempts, options) {
+    super(
+      `Message processing failed after ${attempts} attempts on topic "${topic2}"`,
+      topic2,
+      originalMessage,
+      options
+    );
+    this.attempts = attempts;
+    this.name = "KafkaRetryExhaustedError";
+  }
+};
+
+// src/client/kafka.client.ts
+var ACKS_ALL = -1;
+function toError(error) {
+  return error instanceof Error ? error : new Error(String(error));
+}
+var KafkaClient = class {
+  kafka;
+  producer;
+  consumers = /* @__PURE__ */ new Map();
+  admin;
+  logger;
+  autoCreateTopicsEnabled;
+  strictSchemasEnabled;
+  numPartitions;
+  ensuredTopics = /* @__PURE__ */ new Set();
+  defaultGroupId;
+  schemaRegistry = /* @__PURE__ */ new Map();
+  runningConsumers = /* @__PURE__ */ new Map();
+  isAdminConnected = false;
+  clientId;
+  constructor(clientId, groupId, brokers, options) {
+    this.clientId = clientId;
+    this.defaultGroupId = groupId;
+    this.logger = options?.logger ?? {
+      log: (msg) => console.log(`[KafkaClient:${clientId}] ${msg}`),
+      warn: (msg, ...args) => console.warn(`[KafkaClient:${clientId}] ${msg}`, ...args),
+      error: (msg, ...args) => console.error(`[KafkaClient:${clientId}] ${msg}`, ...args)
+    };
+    this.autoCreateTopicsEnabled = options?.autoCreateTopics ?? false;
+    this.strictSchemasEnabled = options?.strictSchemas ?? true;
+    this.numPartitions = options?.numPartitions ?? 1;
+    this.kafka = new Kafka({
+      clientId: this.clientId,
+      brokers
+    });
+    this.producer = this.kafka.producer({
+      createPartitioner: Partitioners.DefaultPartitioner,
+      idempotent: true,
+      transactionalId: `${clientId}-tx`,
+      maxInFlightRequests: 1
+    });
+    this.admin = this.kafka.admin();
+  }
+  async sendMessage(topicOrDesc, message, options = {}) {
+    const payload = this.buildSendPayload(topicOrDesc, [
+      { value: message, key: options.key, headers: options.headers }
+    ]);
+    await this.ensureTopic(payload.topic);
+    await this.producer.send(payload);
+  }
+  async sendBatch(topicOrDesc, messages) {
+    const payload = this.buildSendPayload(topicOrDesc, messages);
+    await this.ensureTopic(payload.topic);
+    await this.producer.send(payload);
+  }
+  /** Execute multiple sends atomically. Commits on success, aborts on error. */
+  async transaction(fn) {
+    const tx = await this.producer.transaction();
+    try {
+      const ctx = {
+        send: async (topicOrDesc, message, options = {}) => {
+          const payload = this.buildSendPayload(topicOrDesc, [
+            { value: message, key: options.key, headers: options.headers }
+          ]);
+          await this.ensureTopic(payload.topic);
+          await tx.send(payload);
+        },
+        sendBatch: async (topicOrDesc, messages) => {
+          const payload = this.buildSendPayload(topicOrDesc, messages);
+          await this.ensureTopic(payload.topic);
+          await tx.send(payload);
+        }
+      };
+      await fn(ctx);
+      await tx.commit();
+    } catch (error) {
+      try {
+        await tx.abort();
+      } catch (abortError) {
+        this.logger.error(
+          "Failed to abort transaction:",
+          toError(abortError).message
+        );
+      }
+      throw error;
+    }
+  }
+  // ── Producer lifecycle ───────────────────────────────────────────
+  /** Connect the idempotent producer. Called automatically by `KafkaModule.register()`. */
+  async connectProducer() {
+    await this.producer.connect();
+    this.logger.log("Producer connected");
+  }
+  async disconnectProducer() {
+    await this.producer.disconnect();
+    this.logger.log("Producer disconnected");
+  }
+  async startConsumer(topics, handleMessage, options = {}) {
+    const { consumer, schemaMap, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", options);
+    await consumer.run({
+      autoCommit: options.autoCommit ?? true,
+      eachMessage: async ({ topic: topic2, message }) => {
+        if (!message.value) {
+          this.logger.warn(`Received empty message from topic ${topic2}`);
+          return;
+        }
+        const raw = message.value.toString();
+        const parsed = this.parseJsonMessage(raw, topic2);
+        if (parsed === null) return;
+        const validated = await this.validateWithSchema(
+          parsed,
+          raw,
+          topic2,
+          schemaMap,
+          interceptors,
+          dlq
+        );
+        if (validated === null) return;
+        await this.executeWithRetry(
+          () => handleMessage(validated, topic2),
+          { topic: topic2, messages: validated, rawMessages: [raw], interceptors, dlq, retry }
+        );
+      }
+    });
+    this.runningConsumers.set(gid, "eachMessage");
+  }
+  async startBatchConsumer(topics, handleBatch, options = {}) {
+    const { consumer, schemaMap, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", options);
+    await consumer.run({
+      autoCommit: options.autoCommit ?? true,
+      eachBatch: async ({
+        batch,
+        heartbeat,
+        resolveOffset,
+        commitOffsetsIfNecessary
+      }) => {
+        const validMessages = [];
+        const rawMessages = [];
+        for (const message of batch.messages) {
+          if (!message.value) {
+            this.logger.warn(
+              `Received empty message from topic ${batch.topic}`
+            );
+            continue;
+          }
+          const raw = message.value.toString();
+          const parsed = this.parseJsonMessage(raw, batch.topic);
+          if (parsed === null) continue;
+          const validated = await this.validateWithSchema(
+            parsed,
+            raw,
+            batch.topic,
+            schemaMap,
+            interceptors,
+            dlq
+          );
+          if (validated === null) continue;
+          validMessages.push(validated);
+          rawMessages.push(raw);
+        }
+        if (validMessages.length === 0) return;
+        const meta = {
+          partition: batch.partition,
+          highWatermark: batch.highWatermark,
+          heartbeat,
+          resolveOffset,
+          commitOffsetsIfNecessary
+        };
+        await this.executeWithRetry(
+          () => handleBatch(validMessages, batch.topic, meta),
+          {
+            topic: batch.topic,
+            messages: validMessages,
+            rawMessages: batch.messages.filter((m) => m.value).map((m) => m.value.toString()),
+            interceptors,
+            dlq,
+            retry,
+            isBatch: true
+          }
+        );
+      }
+    });
+    this.runningConsumers.set(gid, "eachBatch");
+  }
+  // ── Consumer lifecycle ───────────────────────────────────────────
+  async stopConsumer() {
+    const tasks = [];
+    for (const consumer of this.consumers.values()) {
+      tasks.push(consumer.disconnect());
+    }
+    await Promise.allSettled(tasks);
+    this.consumers.clear();
+    this.runningConsumers.clear();
+    this.logger.log("All consumers disconnected");
+  }
+  /** Check broker connectivity and return available topics. */
+  async checkStatus() {
+    if (!this.isAdminConnected) {
+      await this.admin.connect();
+      this.isAdminConnected = true;
+    }
+    const topics = await this.admin.listTopics();
+    return { topics };
+  }
+  getClientId() {
+    return this.clientId;
+  }
+  /** Gracefully disconnect producer, all consumers, and admin. */
+  async disconnect() {
+    const tasks = [this.producer.disconnect()];
+    for (const consumer of this.consumers.values()) {
+      tasks.push(consumer.disconnect());
+    }
+    if (this.isAdminConnected) {
+      tasks.push(this.admin.disconnect());
+      this.isAdminConnected = false;
+    }
+    await Promise.allSettled(tasks);
+    this.consumers.clear();
+    this.runningConsumers.clear();
+    this.logger.log("All connections closed");
+  }
+  // ── Private helpers ──────────────────────────────────────────────
+  getOrCreateConsumer(groupId) {
+    const gid = groupId || this.defaultGroupId;
+    if (!this.consumers.has(gid)) {
+      this.consumers.set(gid, this.kafka.consumer({ groupId: gid }));
+    }
+    return this.consumers.get(gid);
+  }
+  resolveTopicName(topicOrDescriptor) {
+    if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
+    if (topicOrDescriptor && typeof topicOrDescriptor === "object" && "__topic" in topicOrDescriptor) {
+      return topicOrDescriptor.__topic;
+    }
+    return String(topicOrDescriptor);
+  }
+  async ensureTopic(topic2) {
+    if (!this.autoCreateTopicsEnabled || this.ensuredTopics.has(topic2)) return;
+    if (!this.isAdminConnected) {
+      await this.admin.connect();
+      this.isAdminConnected = true;
+    }
+    await this.admin.createTopics({
+      topics: [{ topic: topic2, numPartitions: this.numPartitions }]
+    });
+    this.ensuredTopics.add(topic2);
+  }
+  /** Register schema from descriptor into global registry (side-effect). */
+  registerSchema(topicOrDesc) {
+    if (topicOrDesc?.__schema) {
+      const topic2 = this.resolveTopicName(topicOrDesc);
+      this.schemaRegistry.set(topic2, topicOrDesc.__schema);
+    }
+  }
+  /** Validate message against schema. Pure — no side-effects on registry. */
+  validateMessage(topicOrDesc, message) {
+    if (topicOrDesc?.__schema) {
+      return topicOrDesc.__schema.parse(message);
+    }
+    if (this.strictSchemasEnabled && typeof topicOrDesc === "string") {
+      const schema = this.schemaRegistry.get(topicOrDesc);
+      if (schema) return schema.parse(message);
+    }
+    return message;
+  }
+  /**
+   * Build a kafkajs-ready send payload.
+   * Handles: topic resolution, schema registration, validation, JSON serialization.
+   */
+  buildSendPayload(topicOrDesc, messages) {
+    this.registerSchema(topicOrDesc);
+    const topic2 = this.resolveTopicName(topicOrDesc);
+    return {
+      topic: topic2,
+      messages: messages.map((m) => ({
+        value: JSON.stringify(this.validateMessage(topicOrDesc, m.value)),
+        key: m.key ?? null,
+        headers: m.headers
+      })),
+      acks: ACKS_ALL
+    };
+  }
+  /** Shared consumer setup: groupId check, schema map, connect, subscribe. */
+  async setupConsumer(topics, mode, options) {
+    const {
+      groupId: optGroupId,
+      fromBeginning = false,
+      retry,
+      dlq = false,
+      interceptors = [],
+      schemas: optionSchemas
+    } = options;
+    const gid = optGroupId || this.defaultGroupId;
+    const existingMode = this.runningConsumers.get(gid);
+    const oppositeMode = mode === "eachMessage" ? "eachBatch" : "eachMessage";
+    if (existingMode === oppositeMode) {
+      throw new Error(
+        `Cannot use ${mode} on consumer group "${gid}" \u2014 it is already running with ${oppositeMode}. Use a different groupId for this consumer.`
+      );
+    }
+    const consumer = this.getOrCreateConsumer(optGroupId);
+    const schemaMap = this.buildSchemaMap(topics, optionSchemas);
+    const topicNames = topics.map(
+      (t) => this.resolveTopicName(t)
+    );
+    await consumer.connect();
+    await this.subscribeWithRetry(consumer, topicNames, fromBeginning, options.subscribeRetry);
+    this.logger.log(
+      `${mode === "eachBatch" ? "Batch consumer" : "Consumer"} subscribed to topics: ${topicNames.join(", ")}`
+    );
+    return { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry };
+  }
+  buildSchemaMap(topics, optionSchemas) {
+    const schemaMap = /* @__PURE__ */ new Map();
+    for (const t of topics) {
+      if (t?.__schema) {
+        const name = this.resolveTopicName(t);
+        schemaMap.set(name, t.__schema);
+        this.schemaRegistry.set(name, t.__schema);
+      }
+    }
+    if (optionSchemas) {
+      for (const [k, v] of optionSchemas) {
+        schemaMap.set(k, v);
+        this.schemaRegistry.set(k, v);
+      }
+    }
+    return schemaMap;
+  }
+  /** Parse raw message as JSON. Returns null on failure (logs error). */
+  parseJsonMessage(raw, topic2) {
+    try {
+      return JSON.parse(raw);
+    } catch (error) {
+      this.logger.error(
+        `Failed to parse message from topic ${topic2}:`,
+        toError(error).stack
+      );
+      return null;
+    }
+  }
+  /**
+   * Validate a parsed message against the schema map.
+   * On failure: logs error, sends to DLQ if enabled, calls interceptor.onError.
+   * Returns validated message or null.
+   */
+  async validateWithSchema(message, raw, topic2, schemaMap, interceptors, dlq) {
+    const schema = schemaMap.get(topic2);
+    if (!schema) return message;
+    try {
+      return schema.parse(message);
+    } catch (error) {
+      const err = toError(error);
+      const validationError = new KafkaValidationError(topic2, message, {
+        cause: err
+      });
+      this.logger.error(
+        `Schema validation failed for topic ${topic2}:`,
+        err.message
+      );
+      if (dlq) await this.sendToDlq(topic2, raw);
+      for (const interceptor of interceptors) {
+        await interceptor.onError?.(message, topic2, validationError);
+      }
+      return null;
+    }
+  }
+  /**
+   * Execute a handler with retry, interceptors, and DLQ support.
+   * Used by both single-message and batch consumers.
+   */
+  async executeWithRetry(fn, ctx) {
+    const { topic: topic2, messages, rawMessages, interceptors, dlq, retry, isBatch } = ctx;
+    const maxAttempts = retry ? retry.maxRetries + 1 : 1;
+    const backoffMs = retry?.backoffMs ?? 1e3;
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+      try {
+        if (isBatch) {
+          for (const interceptor of interceptors) {
+            for (const msg of messages) {
+              await interceptor.before?.(msg, topic2);
+            }
+          }
+        } else {
+          for (const interceptor of interceptors) {
+            await interceptor.before?.(messages, topic2);
+          }
+        }
+        await fn();
+        if (isBatch) {
+          for (const interceptor of interceptors) {
+            for (const msg of messages) {
+              await interceptor.after?.(msg, topic2);
+            }
+          }
+        } else {
+          for (const interceptor of interceptors) {
+            await interceptor.after?.(messages, topic2);
+          }
+        }
+        return;
+      } catch (error) {
+        const err = toError(error);
+        const isLastAttempt = attempt === maxAttempts;
+        if (isLastAttempt && maxAttempts > 1) {
+          const exhaustedError = new KafkaRetryExhaustedError(
+            topic2,
+            messages,
+            maxAttempts,
+            { cause: err }
+          );
+          for (const interceptor of interceptors) {
+            await interceptor.onError?.(messages, topic2, exhaustedError);
+          }
+        } else {
+          for (const interceptor of interceptors) {
+            await interceptor.onError?.(messages, topic2, err);
+          }
+        }
+        this.logger.error(
+          `Error processing ${isBatch ? "batch" : "message"} from topic ${topic2} (attempt ${attempt}/${maxAttempts}):`,
+          err.stack
+        );
+        if (isLastAttempt) {
+          if (dlq) {
+            for (const raw of rawMessages) {
+              await this.sendToDlq(topic2, raw);
+            }
+          }
+        } else {
+          await this.sleep(backoffMs * attempt);
+        }
+      }
+    }
+  }
+  async sendToDlq(topic2, rawMessage) {
+    const dlqTopic = `${topic2}.dlq`;
+    try {
+      await this.producer.send({
+        topic: dlqTopic,
+        messages: [{ value: rawMessage }],
+        acks: ACKS_ALL
+      });
+      this.logger.warn(`Message sent to DLQ: ${dlqTopic}`);
+    } catch (error) {
+      this.logger.error(
+        `Failed to send message to DLQ ${dlqTopic}:`,
+        toError(error).stack
+      );
+    }
+  }
+  async subscribeWithRetry(consumer, topics, fromBeginning, retryOpts) {
+    const maxAttempts = retryOpts?.retries ?? 5;
+    const backoffMs = retryOpts?.backoffMs ?? 5e3;
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+      try {
+        await consumer.subscribe({ topics, fromBeginning });
+        return;
+      } catch (error) {
+        if (attempt === maxAttempts) throw error;
+        const msg = toError(error).message;
+        this.logger.warn(
+          `Failed to subscribe to [${topics.join(", ")}] (attempt ${attempt}/${maxAttempts}): ${msg}. Retrying in ${backoffMs}ms...`
+        );
+        await this.sleep(backoffMs);
+      }
+    }
+  }
+  sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+};
+
+// src/client/topic.ts
+function topic(name) {
+  const fn = () => ({
+    __topic: name,
+    __type: void 0
+  });
+  fn.schema = (schema) => ({
+    __topic: name,
+    __type: void 0,
+    __schema: schema
+  });
+  return fn;
+}
+
+export {
+  __decorateClass,
+  __decorateParam,
+  KafkaProcessingError,
+  KafkaValidationError,
+  KafkaRetryExhaustedError,
+  KafkaClient,
+  topic
+};
+//# sourceMappingURL=chunk-UDOHIMAZ.mjs.map