@drarzter/kafka-client 0.2.2 → 0.3.1

package/dist/index.mjs

@@ -1,525 +1,28 @@
-var __defProp = Object.defineProperty;
-var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
-var __decorateClass = (decorators, target, key, kind) => {
-  var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc(target, key) : target;
-  for (var i = decorators.length - 1, decorator; i >= 0; i--)
-    if (decorator = decorators[i])
-      result = (kind ? decorator(target, key, result) : decorator(result)) || result;
-  if (kind && result) __defProp(target, key, result);
-  return result;
-};
-var __decorateParam = (index, decorator) => (target, key) => decorator(target, key, index);
+import {
+  KafkaClient,
+  KafkaProcessingError,
+  KafkaRetryExhaustedError,
+  KafkaValidationError,
+  topic
+} from "./chunk-A56D7HXR.mjs";
+import {
+  __decorateClass,
+  __decorateParam
+} from "./chunk-EQQGB2QZ.mjs";
 
-// src/module/kafka.module.ts
-import { Module } from "@nestjs/common";
+// src/nest/kafka.module.ts
+import { Module, Logger as Logger2 } from "@nestjs/common";
 import { DiscoveryModule } from "@nestjs/core";
 
-// src/client/kafka.client.ts
-import { Kafka, Partitioners } from "kafkajs";
-import { Logger } from "@nestjs/common";
-
-// src/client/errors.ts
-var KafkaProcessingError = class extends Error {
-  constructor(message, topic2, originalMessage, options) {
-    super(message, options);
-    this.topic = topic2;
-    this.originalMessage = originalMessage;
-    this.name = "KafkaProcessingError";
-    if (options?.cause) this.cause = options.cause;
-  }
-};
-var KafkaValidationError = class extends Error {
-  constructor(topic2, originalMessage, options) {
-    super(`Schema validation failed for topic "${topic2}"`, options);
-    this.topic = topic2;
-    this.originalMessage = originalMessage;
-    this.name = "KafkaValidationError";
-    if (options?.cause) this.cause = options.cause;
-  }
-};
-var KafkaRetryExhaustedError = class extends KafkaProcessingError {
-  constructor(topic2, originalMessage, attempts, options) {
-    super(
-      `Message processing failed after ${attempts} attempts on topic "${topic2}"`,
-      topic2,
-      originalMessage,
-      options
-    );
-    this.attempts = attempts;
-    this.name = "KafkaRetryExhaustedError";
-  }
-};
-
-// src/client/kafka.client.ts
-var ACKS_ALL = -1;
-function toError(error) {
-  return error instanceof Error ? error : new Error(String(error));
-}
-var KafkaClient = class {
-  kafka;
-  producer;
-  consumers = /* @__PURE__ */ new Map();
-  admin;
-  logger;
-  autoCreateTopicsEnabled;
-  strictSchemasEnabled;
-  ensuredTopics = /* @__PURE__ */ new Set();
-  defaultGroupId;
-  schemaRegistry = /* @__PURE__ */ new Map();
-  runningConsumers = /* @__PURE__ */ new Map();
-  isAdminConnected = false;
-  clientId;
-  constructor(clientId, groupId, brokers, options) {
-    this.clientId = clientId;
-    this.defaultGroupId = groupId;
-    this.logger = new Logger(`KafkaClient:${clientId}`);
-    this.autoCreateTopicsEnabled = options?.autoCreateTopics ?? false;
-    this.strictSchemasEnabled = options?.strictSchemas ?? true;
-    this.kafka = new Kafka({
-      clientId: this.clientId,
-      brokers
-    });
-    this.producer = this.kafka.producer({
-      createPartitioner: Partitioners.DefaultPartitioner,
-      idempotent: true,
-      transactionalId: `${clientId}-tx`,
-      maxInFlightRequests: 1
-    });
-    this.admin = this.kafka.admin();
-  }
-  async sendMessage(topicOrDesc, message, options = {}) {
-    const payload = this.buildSendPayload(topicOrDesc, [
-      { value: message, key: options.key, headers: options.headers }
-    ]);
-    await this.ensureTopic(payload.topic);
-    await this.producer.send(payload);
-  }
-  async sendBatch(topicOrDesc, messages) {
-    const payload = this.buildSendPayload(topicOrDesc, messages);
-    await this.ensureTopic(payload.topic);
-    await this.producer.send(payload);
-  }
-  /** Execute multiple sends atomically. Commits on success, aborts on error. */
-  async transaction(fn) {
-    const tx = await this.producer.transaction();
-    try {
-      const ctx = {
-        send: async (topicOrDesc, message, options = {}) => {
-          const payload = this.buildSendPayload(topicOrDesc, [
-            { value: message, key: options.key, headers: options.headers }
-          ]);
-          await this.ensureTopic(payload.topic);
-          await tx.send(payload);
-        },
-        sendBatch: async (topicOrDesc, messages) => {
-          const payload = this.buildSendPayload(topicOrDesc, messages);
-          await this.ensureTopic(payload.topic);
-          await tx.send(payload);
-        }
-      };
-      await fn(ctx);
-      await tx.commit();
-    } catch (error) {
-      await tx.abort();
-      throw error;
-    }
-  }
-  // ── Producer lifecycle ───────────────────────────────────────────
-  /** Connect the idempotent producer. Called automatically by `KafkaModule.register()`. */
-  async connectProducer() {
-    await this.producer.connect();
-    this.logger.log("Producer connected");
-  }
-  async disconnectProducer() {
-    await this.producer.disconnect();
-    this.logger.log("Producer disconnected");
-  }
-  async startConsumer(topics, handleMessage, options = {}) {
-    const { consumer, schemaMap, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", options);
-    await consumer.run({
-      autoCommit: options.autoCommit ?? true,
-      eachMessage: async ({ topic: topic2, message }) => {
-        if (!message.value) {
-          this.logger.warn(`Received empty message from topic ${topic2}`);
-          return;
-        }
-        const raw = message.value.toString();
-        const parsed = this.parseJsonMessage(raw, topic2);
-        if (parsed === null) return;
-        const validated = await this.validateWithSchema(
-          parsed,
-          raw,
-          topic2,
-          schemaMap,
-          interceptors,
-          dlq
-        );
-        if (validated === null) return;
-        await this.executeWithRetry(
-          () => handleMessage(validated, topic2),
-          { topic: topic2, messages: validated, rawMessages: [raw], interceptors, dlq, retry }
-        );
-      }
-    });
-    this.runningConsumers.set(gid, "eachMessage");
-  }
-  async startBatchConsumer(topics, handleBatch, options = {}) {
-    const { consumer, schemaMap, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", options);
-    await consumer.run({
-      autoCommit: options.autoCommit ?? true,
-      eachBatch: async ({
-        batch,
-        heartbeat,
-        resolveOffset,
-        commitOffsetsIfNecessary
-      }) => {
-        const validMessages = [];
-        const rawMessages = [];
-        for (const message of batch.messages) {
-          if (!message.value) {
-            this.logger.warn(
-              `Received empty message from topic ${batch.topic}`
-            );
-            continue;
-          }
-          const raw = message.value.toString();
-          const parsed = this.parseJsonMessage(raw, batch.topic);
-          if (parsed === null) continue;
-          const validated = await this.validateWithSchema(
-            parsed,
-            raw,
-            batch.topic,
-            schemaMap,
-            interceptors,
-            dlq
-          );
-          if (validated === null) continue;
-          validMessages.push(validated);
-          rawMessages.push(raw);
-        }
-        if (validMessages.length === 0) return;
-        const meta = {
-          partition: batch.partition,
-          highWatermark: batch.highWatermark,
-          heartbeat,
-          resolveOffset,
-          commitOffsetsIfNecessary
-        };
-        await this.executeWithRetry(
-          () => handleBatch(validMessages, batch.topic, meta),
-          {
-            topic: batch.topic,
-            messages: validMessages,
-            rawMessages: batch.messages.filter((m) => m.value).map((m) => m.value.toString()),
-            interceptors,
-            dlq,
-            retry,
-            isBatch: true
-          }
-        );
-      }
-    });
-    this.runningConsumers.set(gid, "eachBatch");
-  }
-  // ── Consumer lifecycle ───────────────────────────────────────────
-  async stopConsumer() {
-    const tasks = [];
-    for (const consumer of this.consumers.values()) {
-      tasks.push(consumer.disconnect());
-    }
-    await Promise.allSettled(tasks);
-    this.consumers.clear();
-    this.runningConsumers.clear();
-    this.logger.log("All consumers disconnected");
-  }
-  /** Check broker connectivity and return available topics. */
-  async checkStatus() {
-    if (!this.isAdminConnected) {
-      await this.admin.connect();
-      this.isAdminConnected = true;
-    }
-    const topics = await this.admin.listTopics();
-    return { topics };
-  }
-  getClientId() {
-    return this.clientId;
-  }
-  /** Gracefully disconnect producer, all consumers, and admin. */
-  async disconnect() {
-    const tasks = [this.producer.disconnect()];
-    for (const consumer of this.consumers.values()) {
-      tasks.push(consumer.disconnect());
-    }
-    if (this.isAdminConnected) {
-      tasks.push(this.admin.disconnect());
-      this.isAdminConnected = false;
-    }
-    await Promise.allSettled(tasks);
-    this.consumers.clear();
-    this.runningConsumers.clear();
-    this.logger.log("All connections closed");
-  }
-  // ── Private helpers ──────────────────────────────────────────────
-  getOrCreateConsumer(groupId) {
-    const gid = groupId || this.defaultGroupId;
-    if (!this.consumers.has(gid)) {
-      this.consumers.set(gid, this.kafka.consumer({ groupId: gid }));
-    }
-    return this.consumers.get(gid);
-  }
-  resolveTopicName(topicOrDescriptor) {
-    if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
-    if (topicOrDescriptor && typeof topicOrDescriptor === "object" && "__topic" in topicOrDescriptor) {
-      return topicOrDescriptor.__topic;
-    }
-    return String(topicOrDescriptor);
-  }
-  async ensureTopic(topic2) {
-    if (!this.autoCreateTopicsEnabled || this.ensuredTopics.has(topic2)) return;
-    if (!this.isAdminConnected) {
-      await this.admin.connect();
-      this.isAdminConnected = true;
-    }
-    await this.admin.createTopics({
-      topics: [{ topic: topic2, numPartitions: 1 }]
-    });
-    this.ensuredTopics.add(topic2);
-  }
-  /** Register schema from descriptor into global registry (side-effect). */
-  registerSchema(topicOrDesc) {
-    if (topicOrDesc?.__schema) {
-      const topic2 = this.resolveTopicName(topicOrDesc);
-      this.schemaRegistry.set(topic2, topicOrDesc.__schema);
-    }
-  }
-  /** Validate message against schema. Pure — no side-effects on registry. */
-  validateMessage(topicOrDesc, message) {
-    if (topicOrDesc?.__schema) {
-      return topicOrDesc.__schema.parse(message);
-    }
-    if (this.strictSchemasEnabled && typeof topicOrDesc === "string") {
-      const schema = this.schemaRegistry.get(topicOrDesc);
-      if (schema) return schema.parse(message);
-    }
-    return message;
-  }
-  /**
-   * Build a kafkajs-ready send payload.
-   * Handles: topic resolution, schema registration, validation, JSON serialization.
-   */
-  buildSendPayload(topicOrDesc, messages) {
-    this.registerSchema(topicOrDesc);
-    const topic2 = this.resolveTopicName(topicOrDesc);
-    return {
-      topic: topic2,
-      messages: messages.map((m) => ({
-        value: JSON.stringify(this.validateMessage(topicOrDesc, m.value)),
-        key: m.key ?? null,
-        headers: m.headers
-      })),
-      acks: ACKS_ALL
-    };
-  }
-  /** Shared consumer setup: groupId check, schema map, connect, subscribe. */
-  async setupConsumer(topics, mode, options) {
-    const {
-      groupId: optGroupId,
-      fromBeginning = false,
-      retry,
-      dlq = false,
-      interceptors = [],
-      schemas: optionSchemas
-    } = options;
-    const gid = optGroupId || this.defaultGroupId;
-    const existingMode = this.runningConsumers.get(gid);
-    const oppositeMode = mode === "eachMessage" ? "eachBatch" : "eachMessage";
-    if (existingMode === oppositeMode) {
-      throw new Error(
-        `Cannot use ${mode} on consumer group "${gid}" \u2014 it is already running with ${oppositeMode}. Use a different groupId for this consumer.`
-      );
-    }
-    const consumer = this.getOrCreateConsumer(optGroupId);
-    const schemaMap = this.buildSchemaMap(topics, optionSchemas);
-    const topicNames = topics.map(
-      (t) => this.resolveTopicName(t)
-    );
-    await consumer.connect();
-    await this.subscribeWithRetry(consumer, topicNames, fromBeginning, options.subscribeRetry);
-    this.logger.log(
-      `${mode === "eachBatch" ? "Batch consumer" : "Consumer"} subscribed to topics: ${topicNames.join(", ")}`
-    );
-    return { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry };
-  }
-  buildSchemaMap(topics, optionSchemas) {
-    const schemaMap = /* @__PURE__ */ new Map();
-    for (const t of topics) {
-      if (t?.__schema) {
-        const name = this.resolveTopicName(t);
-        schemaMap.set(name, t.__schema);
-        this.schemaRegistry.set(name, t.__schema);
-      }
-    }
-    if (optionSchemas) {
-      for (const [k, v] of optionSchemas) {
-        schemaMap.set(k, v);
-        this.schemaRegistry.set(k, v);
-      }
-    }
-    return schemaMap;
-  }
-  /** Parse raw message as JSON. Returns null on failure (logs error). */
-  parseJsonMessage(raw, topic2) {
-    try {
-      return JSON.parse(raw);
-    } catch (error) {
-      this.logger.error(
-        `Failed to parse message from topic ${topic2}:`,
-        toError(error).stack
-      );
-      return null;
-    }
-  }
-  /**
-   * Validate a parsed message against the schema map.
-   * On failure: logs error, sends to DLQ if enabled, calls interceptor.onError.
-   * Returns validated message or null.
-   */
-  async validateWithSchema(message, raw, topic2, schemaMap, interceptors, dlq) {
-    const schema = schemaMap.get(topic2);
-    if (!schema) return message;
-    try {
-      return schema.parse(message);
-    } catch (error) {
-      const err = toError(error);
-      const validationError = new KafkaValidationError(topic2, message, {
-        cause: err
-      });
-      this.logger.error(
-        `Schema validation failed for topic ${topic2}:`,
-        err.message
-      );
-      if (dlq) await this.sendToDlq(topic2, raw);
-      for (const interceptor of interceptors) {
-        await interceptor.onError?.(message, topic2, validationError);
-      }
-      return null;
-    }
-  }
-  /**
-   * Execute a handler with retry, interceptors, and DLQ support.
-   * Used by both single-message and batch consumers.
-   */
-  async executeWithRetry(fn, ctx) {
-    const { topic: topic2, messages, rawMessages, interceptors, dlq, retry, isBatch } = ctx;
-    const maxAttempts = retry ? retry.maxRetries + 1 : 1;
-    const backoffMs = retry?.backoffMs ?? 1e3;
-    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
-      try {
-        if (isBatch) {
-          for (const interceptor of interceptors) {
-            for (const msg of messages) {
-              await interceptor.before?.(msg, topic2);
-            }
-          }
-        } else {
-          for (const interceptor of interceptors) {
-            await interceptor.before?.(messages, topic2);
-          }
-        }
-        await fn();
-        if (isBatch) {
-          for (const interceptor of interceptors) {
-            for (const msg of messages) {
-              await interceptor.after?.(msg, topic2);
-            }
-          }
-        } else {
-          for (const interceptor of interceptors) {
-            await interceptor.after?.(messages, topic2);
-          }
-        }
-        return;
-      } catch (error) {
-        const err = toError(error);
-        const isLastAttempt = attempt === maxAttempts;
-        if (isLastAttempt && maxAttempts > 1) {
-          const exhaustedError = new KafkaRetryExhaustedError(
-            topic2,
-            messages,
-            maxAttempts,
-            { cause: err }
-          );
-          for (const interceptor of interceptors) {
-            await interceptor.onError?.(messages, topic2, exhaustedError);
-          }
-        } else {
-          for (const interceptor of interceptors) {
-            await interceptor.onError?.(messages, topic2, err);
-          }
-        }
-        this.logger.error(
-          `Error processing ${isBatch ? "batch" : "message"} from topic ${topic2} (attempt ${attempt}/${maxAttempts}):`,
-          err.stack
-        );
-        if (isLastAttempt) {
-          if (dlq) {
-            for (const raw of rawMessages) {
-              await this.sendToDlq(topic2, raw);
-            }
-          }
-        } else {
-          await this.sleep(backoffMs * attempt);
-        }
-      }
-    }
-  }
-  async sendToDlq(topic2, rawMessage) {
-    const dlqTopic = `${topic2}.dlq`;
-    try {
-      await this.producer.send({
-        topic: dlqTopic,
-        messages: [{ value: rawMessage }],
-        acks: ACKS_ALL
-      });
-      this.logger.warn(`Message sent to DLQ: ${dlqTopic}`);
-    } catch (error) {
-      this.logger.error(
-        `Failed to send message to DLQ ${dlqTopic}:`,
-        toError(error).stack
-      );
-    }
-  }
-  async subscribeWithRetry(consumer, topics, fromBeginning, retryOpts) {
-    const maxAttempts = retryOpts?.retries ?? 5;
-    const backoffMs = retryOpts?.backoffMs ?? 5e3;
-    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
-      try {
-        await consumer.subscribe({ topics, fromBeginning });
-        return;
-      } catch (error) {
-        if (attempt === maxAttempts) throw error;
-        const msg = toError(error).message;
-        this.logger.warn(
-          `Failed to subscribe to [${topics.join(", ")}] (attempt ${attempt}/${maxAttempts}): ${msg}. Retrying in ${backoffMs}ms...`
-        );
-        await this.sleep(backoffMs);
-      }
-    }
-  }
-  sleep(ms) {
-    return new Promise((resolve) => setTimeout(resolve, ms));
-  }
-};
-
-// src/module/kafka.constants.ts
+// src/nest/kafka.constants.ts
 var KAFKA_CLIENT = "KAFKA_CLIENT";
 var getKafkaClientToken = (name) => name ? `KAFKA_CLIENT_${name}` : KAFKA_CLIENT;
 
-// src/module/kafka.explorer.ts
-import { Inject as Inject2, Injectable, Logger as Logger2 } from "@nestjs/common";
+// src/nest/kafka.explorer.ts
+import { Inject as Inject2, Injectable, Logger } from "@nestjs/common";
 import { DiscoveryService, ModuleRef } from "@nestjs/core";
 
-// src/decorators/kafka.decorator.ts
+// src/nest/kafka.decorator.ts
 import { Inject } from "@nestjs/common";
 var KAFKA_SUBSCRIBER_METADATA = "KAFKA_SUBSCRIBER_METADATA";
 var InjectKafkaClient = (name) => Inject(getKafkaClientToken(name));
@@ -553,13 +56,13 @@ var SubscribeTo = (topics, options) => {
   };
 };
 
-// src/module/kafka.explorer.ts
+// src/nest/kafka.explorer.ts
 var KafkaExplorer = class {
   constructor(discoveryService, moduleRef) {
     this.discoveryService = discoveryService;
     this.moduleRef = moduleRef;
   }
-  logger = new Logger2(KafkaExplorer.name);
+  logger = new Logger(KafkaExplorer.name);
   async onModuleInit() {
     const providers = this.discoveryService.getProviders();
     for (const wrapper of providers) {
@@ -616,7 +119,7 @@ KafkaExplorer = __decorateClass([
   __decorateParam(1, Inject2(ModuleRef))
 ], KafkaExplorer);
 
-// src/module/kafka.module.ts
+// src/nest/kafka.module.ts
 var KafkaModule = class {
   /** Register a Kafka client with static options. */
   static register(options) {
@@ -628,7 +131,10 @@ var KafkaModule = class {
           options.clientId,
           options.groupId,
           options.brokers,
-          { autoCreateTopics: options.autoCreateTopics }
+          {
+            autoCreateTopics: options.autoCreateTopics,
+            logger: new Logger2(`KafkaClient:${options.clientId}`)
+          }
         );
         await client.connectProducer();
         return client;
@@ -660,7 +166,10 @@ var KafkaModule = class {
           options.clientId,
           options.groupId,
           options.brokers,
-          { autoCreateTopics: options.autoCreateTopics }
+          {
+            autoCreateTopics: options.autoCreateTopics,
+            logger: new Logger2(`KafkaClient:${options.clientId}`)
+          }
         );
         await client.connectProducer();
         return client;
@@ -687,21 +196,7 @@ KafkaModule = __decorateClass([
   Module({})
 ], KafkaModule);
 
-// src/client/topic.ts
-function topic(name) {
-  const fn = () => ({
-    __topic: name,
-    __type: void 0
-  });
-  fn.schema = (schema) => ({
-    __topic: name,
-    __type: void 0,
-    __schema: schema
-  });
-  return fn;
-}
-
-// src/health/kafka.health.ts
+// src/nest/kafka.health.ts
 import { Injectable as Injectable2 } from "@nestjs/common";
 var KafkaHealthIndicator = class {
   async check(client) {