@drarzter/kafka-client 0.1.9 → 0.2.0

package/dist/index.js

@@ -38,6 +38,7 @@ __export(index_exports, {
   KafkaModule: () => KafkaModule,
   KafkaProcessingError: () => KafkaProcessingError,
   KafkaRetryExhaustedError: () => KafkaRetryExhaustedError,
+  KafkaValidationError: () => KafkaValidationError,
   SubscribeTo: () => SubscribeTo,
   getKafkaClientToken: () => getKafkaClientToken,
   topic: () => topic
@@ -62,6 +63,15 @@ var KafkaProcessingError = class extends Error {
     if (options?.cause) this.cause = options.cause;
   }
 };
+var KafkaValidationError = class extends Error {
+  constructor(topic2, originalMessage, options) {
+    super(`Schema validation failed for topic "${topic2}"`, options);
+    this.topic = topic2;
+    this.originalMessage = originalMessage;
+    this.name = "KafkaValidationError";
+    if (options?.cause) this.cause = options.cause;
+  }
+};
 var KafkaRetryExhaustedError = class extends KafkaProcessingError {
   constructor(topic2, originalMessage, attempts, options) {
     super(
@@ -79,15 +89,17 @@ var KafkaRetryExhaustedError = class extends KafkaProcessingError {
 var KafkaClient = class {
   kafka;
   producer;
-  consumer;
+  consumers = /* @__PURE__ */ new Map();
   admin;
   logger;
   autoCreateTopicsEnabled;
   ensuredTopics = /* @__PURE__ */ new Set();
+  defaultGroupId;
   isAdminConnected = false;
   clientId;
   constructor(clientId, groupId, brokers, options) {
     this.clientId = clientId;
+    this.defaultGroupId = groupId;
     this.logger = new import_common.Logger(`KafkaClient:${clientId}`);
     this.autoCreateTopicsEnabled = options?.autoCreateTopics ?? false;
     this.kafka = new import_kafkajs.Kafka({
@@ -100,9 +112,15 @@ var KafkaClient = class {
       transactionalId: `${clientId}-tx`,
       maxInFlightRequests: 1
     });
-    this.consumer = this.kafka.consumer({ groupId });
     this.admin = this.kafka.admin();
   }
+  getOrCreateConsumer(groupId) {
+    const gid = groupId || this.defaultGroupId;
+    if (!this.consumers.has(gid)) {
+      this.consumers.set(gid, this.kafka.consumer({ groupId: gid }));
+    }
+    return this.consumers.get(gid);
+  }
   resolveTopicName(topicOrDescriptor) {
     if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
     if (topicOrDescriptor && typeof topicOrDescriptor === "object" && "__topic" in topicOrDescriptor) {
@@ -121,14 +139,21 @@ var KafkaClient = class {
     });
     this.ensuredTopics.add(topic2);
   }
+  validateMessage(topicOrDesc, message) {
+    if (topicOrDesc?.__schema) {
+      return topicOrDesc.__schema.parse(message);
+    }
+    return message;
+  }
   async sendMessage(topicOrDesc, message, options = {}) {
+    const validated = this.validateMessage(topicOrDesc, message);
     const topic2 = this.resolveTopicName(topicOrDesc);
     await this.ensureTopic(topic2);
     await this.producer.send({
       topic: topic2,
       messages: [
         {
-          value: JSON.stringify(message),
+          value: JSON.stringify(validated),
           key: options.key ?? null,
           headers: options.headers
         }
@@ -142,7 +167,7 @@ var KafkaClient = class {
     await this.producer.send({
       topic: topic2,
       messages: messages.map((m) => ({
-        value: JSON.stringify(m.value),
+        value: JSON.stringify(this.validateMessage(topicOrDesc, m.value)),
         key: m.key ?? null,
         headers: m.headers
       })),
@@ -155,13 +180,14 @@ var KafkaClient = class {
     try {
       const ctx = {
         send: async (topicOrDesc, message, options = {}) => {
+          const validated = this.validateMessage(topicOrDesc, message);
           const topic2 = this.resolveTopicName(topicOrDesc);
           await this.ensureTopic(topic2);
           await tx.send({
             topic: topic2,
             messages: [
               {
-                value: JSON.stringify(message),
+                value: JSON.stringify(validated),
                 key: options.key ?? null,
                 headers: options.headers
               }
@@ -175,7 +201,7 @@ var KafkaClient = class {
           await tx.send({
             topic: topic2,
             messages: messages.map((m) => ({
-              value: JSON.stringify(m.value),
+              value: JSON.stringify(this.validateMessage(topicOrDesc, m.value)),
               key: m.key ?? null,
               headers: m.headers
             })),
@@ -201,22 +227,26 @@ var KafkaClient = class {
   }
   async startConsumer(topics, handleMessage, options = {}) {
     const {
+      groupId: optGroupId,
       fromBeginning = false,
       autoCommit = true,
       retry,
       dlq = false,
-      interceptors = []
+      interceptors = [],
+      schemas: optionSchemas
     } = options;
+    const consumer = this.getOrCreateConsumer(optGroupId);
+    const schemaMap = this.buildSchemaMap(topics, optionSchemas);
     const topicNames = topics.map(
       (t) => this.resolveTopicName(t)
     );
-    await this.consumer.connect();
+    await consumer.connect();
     for (const t of topicNames) {
       await this.ensureTopic(t);
     }
-    await this.consumer.subscribe({ topics: topicNames, fromBeginning });
+    await consumer.subscribe({ topics: topicNames, fromBeginning });
     this.logger.log(`Consumer subscribed to topics: ${topicNames.join(", ")}`);
-    await this.consumer.run({
+    await consumer.run({
       autoCommit,
       eachMessage: async ({ topic: topic2, message }) => {
         if (!message.value) {
@@ -234,6 +264,32 @@ var KafkaClient = class {
           );
           return;
         }
+        const schema = schemaMap.get(topic2);
+        if (schema) {
+          try {
+            parsedMessage = schema.parse(parsedMessage);
+          } catch (error) {
+            const err = error instanceof Error ? error : new Error(String(error));
+            const validationError = new KafkaValidationError(
+              topic2,
+              parsedMessage,
+              { cause: err }
+            );
+            this.logger.error(
+              `Schema validation failed for topic ${topic2}:`,
+              err.message
+            );
+            if (dlq) await this.sendToDlq(topic2, raw);
+            for (const interceptor of interceptors) {
+              await interceptor.onError?.(
+                parsedMessage,
+                topic2,
+                validationError
+              );
+            }
+            return;
+          }
+        }
         await this.processMessage(parsedMessage, raw, topic2, handleMessage, {
           retry,
           dlq,
@@ -242,9 +298,162 @@ var KafkaClient = class {
       }
     });
   }
+  async startBatchConsumer(topics, handleBatch, options = {}) {
+    const {
+      groupId: optGroupId,
+      fromBeginning = false,
+      autoCommit = true,
+      retry,
+      dlq = false,
+      interceptors = [],
+      schemas: optionSchemas
+    } = options;
+    const consumer = this.getOrCreateConsumer(optGroupId);
+    const schemaMap = this.buildSchemaMap(topics, optionSchemas);
+    const topicNames = topics.map(
+      (t) => this.resolveTopicName(t)
+    );
+    await consumer.connect();
+    for (const t of topicNames) {
+      await this.ensureTopic(t);
+    }
+    await consumer.subscribe({ topics: topicNames, fromBeginning });
+    this.logger.log(
+      `Batch consumer subscribed to topics: ${topicNames.join(", ")}`
+    );
+    await consumer.run({
+      autoCommit,
+      eachBatch: async ({
+        batch,
+        heartbeat,
+        resolveOffset,
+        commitOffsetsIfNecessary
+      }) => {
+        const validMessages = [];
+        for (const message of batch.messages) {
+          if (!message.value) {
+            this.logger.warn(
+              `Received empty message from topic ${batch.topic}`
+            );
+            continue;
+          }
+          const raw = message.value.toString();
+          let parsedMessage;
+          try {
+            parsedMessage = JSON.parse(raw);
+          } catch (error) {
+            this.logger.error(
+              `Failed to parse message from topic ${batch.topic}:`,
+              error instanceof Error ? error.stack : String(error)
+            );
+            continue;
+          }
+          const schema = schemaMap.get(batch.topic);
+          if (schema) {
+            try {
+              parsedMessage = schema.parse(parsedMessage);
+            } catch (error) {
+              const err = error instanceof Error ? error : new Error(String(error));
+              const validationError = new KafkaValidationError(
+                batch.topic,
+                parsedMessage,
+                { cause: err }
+              );
+              this.logger.error(
+                `Schema validation failed for topic ${batch.topic}:`,
+                err.message
+              );
+              if (dlq) await this.sendToDlq(batch.topic, raw);
+              for (const interceptor of interceptors) {
+                await interceptor.onError?.(
+                  parsedMessage,
+                  batch.topic,
+                  validationError
+                );
+              }
+              continue;
+            }
+          }
+          validMessages.push(parsedMessage);
+        }
+        if (validMessages.length === 0) return;
+        const meta = {
+          partition: batch.partition,
+          highWatermark: batch.highWatermark,
+          heartbeat,
+          resolveOffset,
+          commitOffsetsIfNecessary
+        };
+        const maxAttempts = retry ? retry.maxRetries + 1 : 1;
+        const backoffMs = retry?.backoffMs ?? 1e3;
+        for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+          try {
+            for (const interceptor of interceptors) {
+              for (const msg of validMessages) {
+                await interceptor.before?.(msg, batch.topic);
+              }
+            }
+            await handleBatch(validMessages, batch.topic, meta);
+            for (const interceptor of interceptors) {
+              for (const msg of validMessages) {
+                await interceptor.after?.(msg, batch.topic);
+              }
+            }
+            return;
+          } catch (error) {
+            const err = error instanceof Error ? error : new Error(String(error));
+            const isLastAttempt = attempt === maxAttempts;
+            if (isLastAttempt && maxAttempts > 1) {
+              const exhaustedError = new KafkaRetryExhaustedError(
+                batch.topic,
+                validMessages,
+                maxAttempts,
+                { cause: err }
+              );
+              for (const interceptor of interceptors) {
+                await interceptor.onError?.(
+                  validMessages,
+                  batch.topic,
+                  exhaustedError
+                );
+              }
+            } else {
+              for (const interceptor of interceptors) {
+                await interceptor.onError?.(
+                  validMessages,
+                  batch.topic,
+                  err
+                );
+              }
+            }
+            this.logger.error(
+              `Error processing batch from topic ${batch.topic} (attempt ${attempt}/${maxAttempts}):`,
+              err.stack
+            );
+            if (isLastAttempt) {
+              if (dlq) {
+                for (const msg of batch.messages) {
+                  if (msg.value) {
+                    await this.sendToDlq(batch.topic, msg.value.toString());
+                  }
+                }
+              }
+            } else {
+              await this.sleep(backoffMs * attempt);
+            }
+          }
+        }
+      }
+    });
+  }
   async stopConsumer() {
-    await this.consumer.disconnect();
-    this.logger.log("Consumer disconnected");
+    const tasks = [];
+    for (const consumer of this.consumers.values()) {
+      tasks.push(consumer.disconnect());
+    }
+    await Promise.allSettled(tasks);
+    this.consumers.clear();
+    this.logger.log("All consumers disconnected");
   }
   /** Check broker connectivity and return available topics. */
   async checkStatus() {
@@ -258,17 +467,35 @@ var KafkaClient = class {
   getClientId() {
     return this.clientId;
   }
-  /** Gracefully disconnect producer, consumer, and admin. */
+  /** Gracefully disconnect producer, all consumers, and admin. */
   async disconnect() {
-    const tasks = [this.producer.disconnect(), this.consumer.disconnect()];
+    const tasks = [this.producer.disconnect()];
+    for (const consumer of this.consumers.values()) {
+      tasks.push(consumer.disconnect());
+    }
     if (this.isAdminConnected) {
       tasks.push(this.admin.disconnect());
       this.isAdminConnected = false;
     }
     await Promise.allSettled(tasks);
+    this.consumers.clear();
     this.logger.log("All connections closed");
   }
   // --- Private helpers ---
+  buildSchemaMap(topics, optionSchemas) {
+    const schemaMap = /* @__PURE__ */ new Map();
+    for (const t of topics) {
+      if (t?.__schema) {
+        schemaMap.set(this.resolveTopicName(t), t.__schema);
+      }
+    }
+    if (optionSchemas) {
+      for (const [k, v] of optionSchemas) {
+        schemaMap.set(k, v);
+      }
+    }
+    return schemaMap;
+  }
   async processMessage(parsedMessage, raw, topic2, handleMessage, opts) {
     const { retry, dlq = false, interceptors = [] } = opts;
     const maxAttempts = retry ? retry.maxRetries + 1 : 1;
@@ -348,10 +575,14 @@ var KAFKA_SUBSCRIBER_METADATA = "KAFKA_SUBSCRIBER_METADATA";
 var InjectKafkaClient = (name) => (0, import_common2.Inject)(getKafkaClientToken(name));
 var SubscribeTo = (topics, options) => {
   const arr = Array.isArray(topics) ? topics : [topics];
-  const topicsArray = arr.map(
-    (t) => typeof t === "string" ? t : t.__topic
-  );
-  const { clientName, ...consumerOptions } = options || {};
+  const topicsArray = arr.map((t) => typeof t === "string" ? t : t.__topic);
+  const schemas = /* @__PURE__ */ new Map();
+  for (const t of arr) {
+    if (typeof t !== "string" && t.__schema) {
+      schemas.set(t.__topic, t.__schema);
+    }
+  }
+  const { clientName, batch, ...consumerOptions } = options || {};
   return (target, propertyKey, _descriptor) => {
     const existing = Reflect.getMetadata(KAFKA_SUBSCRIBER_METADATA, target.constructor) || [];
     Reflect.defineMetadata(
@@ -360,8 +591,10 @@ var SubscribeTo = (topics, options) => {
         ...existing,
         {
           topics: topicsArray,
+          schemas: schemas.size > 0 ? schemas : void 0,
           options: Object.keys(consumerOptions).length ? consumerOptions : void 0,
           clientName,
+          batch,
           methodName: propertyKey
         }
       ],
@@ -399,15 +632,29 @@ var KafkaExplorer = class {
           continue;
         }
         const handler = instance[entry.methodName].bind(instance);
-        await client.startConsumer(
-          entry.topics,
-          async (message, topic2) => {
-            await handler(message, topic2);
-          },
-          entry.options
-        );
+        const consumerOptions = { ...entry.options };
+        if (entry.schemas) {
+          consumerOptions.schemas = entry.schemas;
+        }
+        if (entry.batch) {
+          await client.startBatchConsumer(
+            entry.topics,
+            async (messages, topic2, meta) => {
+              await handler(messages, topic2, meta);
+            },
+            consumerOptions
+          );
+        } else {
+          await client.startConsumer(
+            entry.topics,
+            async (message, topic2) => {
+              await handler(message, topic2);
+            },
+            consumerOptions
+          );
+        }
         this.logger.log(
-          `Registered @SubscribeTo(${entry.topics.join(", ")}) on ${instance.constructor.name}.${String(entry.methodName)}`
+          `Registered @SubscribeTo(${entry.topics.join(", ")})${entry.batch ? " [batch]" : ""} on ${instance.constructor.name}.${String(entry.methodName)}`
         );
       }
     }
@@ -492,10 +739,16 @@ KafkaModule = __decorateClass([
 
 // src/client/topic.ts
 function topic(name) {
-  return () => ({
+  const fn = () => ({
     __topic: name,
     __type: void 0
   });
+  fn.schema = (schema) => ({
+    __topic: name,
+    __type: void 0,
+    __schema: schema
+  });
+  return fn;
 }
 
 // src/health/kafka.health.ts
@@ -532,6 +785,7 @@ KafkaHealthIndicator = __decorateClass([
   KafkaModule,
   KafkaProcessingError,
   KafkaRetryExhaustedError,
+  KafkaValidationError,
   SubscribeTo,
   getKafkaClientToken,
   topic