@drarzter/kafka-client 0.2.0 → 0.2.2

package/dist/index.js

@@ -86,6 +86,10 @@ var KafkaRetryExhaustedError = class extends KafkaProcessingError {
 };
 
 // src/client/kafka.client.ts
+var ACKS_ALL = -1;
+function toError(error) {
+  return error instanceof Error ? error : new Error(String(error));
+}
 var KafkaClient = class {
   kafka;
   producer;
@@ -93,8 +97,11 @@ var KafkaClient = class {
   admin;
   logger;
   autoCreateTopicsEnabled;
+  strictSchemasEnabled;
   ensuredTopics = /* @__PURE__ */ new Set();
   defaultGroupId;
+  schemaRegistry = /* @__PURE__ */ new Map();
+  runningConsumers = /* @__PURE__ */ new Map();
   isAdminConnected = false;
   clientId;
   constructor(clientId, groupId, brokers, options) {
@@ -102,6 +109,7 @@ var KafkaClient = class {
     this.defaultGroupId = groupId;
     this.logger = new import_common.Logger(`KafkaClient:${clientId}`);
     this.autoCreateTopicsEnabled = options?.autoCreateTopics ?? false;
+    this.strictSchemasEnabled = options?.strictSchemas ?? true;
     this.kafka = new import_kafkajs.Kafka({
       clientId: this.clientId,
       brokers
@@ -114,65 +122,17 @@ var KafkaClient = class {
     });
     this.admin = this.kafka.admin();
   }
-  getOrCreateConsumer(groupId) {
-    const gid = groupId || this.defaultGroupId;
-    if (!this.consumers.has(gid)) {
-      this.consumers.set(gid, this.kafka.consumer({ groupId: gid }));
-    }
-    return this.consumers.get(gid);
-  }
-  resolveTopicName(topicOrDescriptor) {
-    if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
-    if (topicOrDescriptor && typeof topicOrDescriptor === "object" && "__topic" in topicOrDescriptor) {
-      return topicOrDescriptor.__topic;
-    }
-    return String(topicOrDescriptor);
-  }
-  async ensureTopic(topic2) {
-    if (!this.autoCreateTopicsEnabled || this.ensuredTopics.has(topic2)) return;
-    if (!this.isAdminConnected) {
-      await this.admin.connect();
-      this.isAdminConnected = true;
-    }
-    await this.admin.createTopics({
-      topics: [{ topic: topic2, numPartitions: 1 }]
-    });
-    this.ensuredTopics.add(topic2);
-  }
-  validateMessage(topicOrDesc, message) {
-    if (topicOrDesc?.__schema) {
-      return topicOrDesc.__schema.parse(message);
-    }
-    return message;
-  }
   async sendMessage(topicOrDesc, message, options = {}) {
-    const validated = this.validateMessage(topicOrDesc, message);
-    const topic2 = this.resolveTopicName(topicOrDesc);
-    await this.ensureTopic(topic2);
-    await this.producer.send({
-      topic: topic2,
-      messages: [
-        {
-          value: JSON.stringify(validated),
-          key: options.key ?? null,
-          headers: options.headers
-        }
-      ],
-      acks: -1
-    });
+    const payload = this.buildSendPayload(topicOrDesc, [
+      { value: message, key: options.key, headers: options.headers }
+    ]);
+    await this.ensureTopic(payload.topic);
+    await this.producer.send(payload);
   }
   async sendBatch(topicOrDesc, messages) {
-    const topic2 = this.resolveTopicName(topicOrDesc);
-    await this.ensureTopic(topic2);
-    await this.producer.send({
-      topic: topic2,
-      messages: messages.map((m) => ({
-        value: JSON.stringify(this.validateMessage(topicOrDesc, m.value)),
-        key: m.key ?? null,
-        headers: m.headers
-      })),
-      acks: -1
-    });
+    const payload = this.buildSendPayload(topicOrDesc, messages);
+    await this.ensureTopic(payload.topic);
+    await this.producer.send(payload);
   }
   /** Execute multiple sends atomically. Commits on success, aborts on error. */
   async transaction(fn) {
@@ -180,33 +140,16 @@ var KafkaClient = class {
     try {
       const ctx = {
         send: async (topicOrDesc, message, options = {}) => {
-          const validated = this.validateMessage(topicOrDesc, message);
-          const topic2 = this.resolveTopicName(topicOrDesc);
-          await this.ensureTopic(topic2);
-          await tx.send({
-            topic: topic2,
-            messages: [
-              {
-                value: JSON.stringify(validated),
-                key: options.key ?? null,
-                headers: options.headers
-              }
-            ],
-            acks: -1
-          });
+          const payload = this.buildSendPayload(topicOrDesc, [
+            { value: message, key: options.key, headers: options.headers }
+          ]);
+          await this.ensureTopic(payload.topic);
+          await tx.send(payload);
         },
         sendBatch: async (topicOrDesc, messages) => {
-          const topic2 = this.resolveTopicName(topicOrDesc);
-          await this.ensureTopic(topic2);
-          await tx.send({
-            topic: topic2,
-            messages: messages.map((m) => ({
-              value: JSON.stringify(this.validateMessage(topicOrDesc, m.value)),
-              key: m.key ?? null,
-              headers: m.headers
-            })),
-            acks: -1
-          });
+          const payload = this.buildSendPayload(topicOrDesc, messages);
+          await this.ensureTopic(payload.topic);
+          await tx.send(payload);
         }
       };
       await fn(ctx);
@@ -216,6 +159,7 @@ var KafkaClient = class {
       throw error;
     }
   }
+  // ── Producer lifecycle ───────────────────────────────────────────
   /** Connect the idempotent producer. Called automatically by `KafkaModule.register()`. */
   async connectProducer() {
     await this.producer.connect();
@@ -226,103 +170,38 @@ var KafkaClient = class {
     this.logger.log("Producer disconnected");
   }
   async startConsumer(topics, handleMessage, options = {}) {
-    const {
-      groupId: optGroupId,
-      fromBeginning = false,
-      autoCommit = true,
-      retry,
-      dlq = false,
-      interceptors = [],
-      schemas: optionSchemas
-    } = options;
-    const consumer = this.getOrCreateConsumer(optGroupId);
-    const schemaMap = this.buildSchemaMap(topics, optionSchemas);
-    const topicNames = topics.map(
-      (t) => this.resolveTopicName(t)
-    );
-    await consumer.connect();
-    for (const t of topicNames) {
-      await this.ensureTopic(t);
-    }
-    await consumer.subscribe({ topics: topicNames, fromBeginning });
-    this.logger.log(`Consumer subscribed to topics: ${topicNames.join(", ")}`);
+    const { consumer, schemaMap, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", options);
     await consumer.run({
-      autoCommit,
+      autoCommit: options.autoCommit ?? true,
       eachMessage: async ({ topic: topic2, message }) => {
         if (!message.value) {
           this.logger.warn(`Received empty message from topic ${topic2}`);
           return;
         }
         const raw = message.value.toString();
-        let parsedMessage;
-        try {
-          parsedMessage = JSON.parse(raw);
-        } catch (error) {
-          this.logger.error(
-            `Failed to parse message from topic ${topic2}:`,
-            error instanceof Error ? error.stack : String(error)
-          );
-          return;
-        }
-        const schema = schemaMap.get(topic2);
-        if (schema) {
-          try {
-            parsedMessage = schema.parse(parsedMessage);
-          } catch (error) {
-            const err = error instanceof Error ? error : new Error(String(error));
-            const validationError = new KafkaValidationError(
-              topic2,
-              parsedMessage,
-              { cause: err }
-            );
-            this.logger.error(
-              `Schema validation failed for topic ${topic2}:`,
-              err.message
-            );
-            if (dlq) await this.sendToDlq(topic2, raw);
-            for (const interceptor of interceptors) {
-              await interceptor.onError?.(
-                parsedMessage,
-                topic2,
-                validationError
-              );
-            }
-            return;
-          }
-        }
-        await this.processMessage(parsedMessage, raw, topic2, handleMessage, {
-          retry,
-          dlq,
-          interceptors
-        });
+        const parsed = this.parseJsonMessage(raw, topic2);
+        if (parsed === null) return;
+        const validated = await this.validateWithSchema(
+          parsed,
+          raw,
+          topic2,
+          schemaMap,
+          interceptors,
+          dlq
+        );
+        if (validated === null) return;
+        await this.executeWithRetry(
+          () => handleMessage(validated, topic2),
+          { topic: topic2, messages: validated, rawMessages: [raw], interceptors, dlq, retry }
+        );
       }
     });
+    this.runningConsumers.set(gid, "eachMessage");
   }
   async startBatchConsumer(topics, handleBatch, options = {}) {
-    const {
-      groupId: optGroupId,
-      fromBeginning = false,
-      autoCommit = true,
-      retry,
-      dlq = false,
-      interceptors = [],
-      schemas: optionSchemas
-    } = options;
-    const consumer = this.getOrCreateConsumer(optGroupId);
-    const schemaMap = this.buildSchemaMap(topics, optionSchemas);
-    const topicNames = topics.map(
-      (t) => this.resolveTopicName(t)
-    );
-    await consumer.connect();
-    for (const t of topicNames) {
-      await this.ensureTopic(t);
-    }
-    await consumer.subscribe({ topics: topicNames, fromBeginning });
-    this.logger.log(
-      `Batch consumer subscribed to topics: ${topicNames.join(", ")}`
-    );
+    const { consumer, schemaMap, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", options);
     await consumer.run({
-      autoCommit,
+      autoCommit: options.autoCommit ?? true,
       eachBatch: async ({
         batch,
         heartbeat,
@@ -330,6 +209,7 @@ var KafkaClient = class {
         commitOffsetsIfNecessary
       }) => {
         const validMessages = [];
+        const rawMessages = [];
         for (const message of batch.messages) {
           if (!message.value) {
             this.logger.warn(
@@ -338,43 +218,19 @@ var KafkaClient = class {
             continue;
           }
           const raw = message.value.toString();
-          let parsedMessage;
-          try {
-            parsedMessage = JSON.parse(raw);
-          } catch (error) {
-            this.logger.error(
-              `Failed to parse message from topic ${batch.topic}:`,
-              error instanceof Error ? error.stack : String(error)
-            );
-            continue;
-          }
-          const schema = schemaMap.get(batch.topic);
-          if (schema) {
-            try {
-              parsedMessage = schema.parse(parsedMessage);
-            } catch (error) {
-              const err = error instanceof Error ? error : new Error(String(error));
-              const validationError = new KafkaValidationError(
-                batch.topic,
-                parsedMessage,
-                { cause: err }
-              );
-              this.logger.error(
-                `Schema validation failed for topic ${batch.topic}:`,
-                err.message
-              );
-              if (dlq) await this.sendToDlq(batch.topic, raw);
-              for (const interceptor of interceptors) {
-                await interceptor.onError?.(
-                  parsedMessage,
-                  batch.topic,
-                  validationError
-                );
-              }
-              continue;
-            }
-          }
-          validMessages.push(parsedMessage);
+          const parsed = this.parseJsonMessage(raw, batch.topic);
+          if (parsed === null) continue;
+          const validated = await this.validateWithSchema(
+            parsed,
+            raw,
+            batch.topic,
+            schemaMap,
+            interceptors,
+            dlq
+          );
+          if (validated === null) continue;
+          validMessages.push(validated);
+          rawMessages.push(raw);
         }
         if (validMessages.length === 0) return;
         const meta = {
@@ -384,68 +240,23 @@ var KafkaClient = class {
           resolveOffset,
           commitOffsetsIfNecessary
         };
-        const maxAttempts = retry ? retry.maxRetries + 1 : 1;
-        const backoffMs = retry?.backoffMs ?? 1e3;
-        for (let attempt = 1; attempt <= maxAttempts; attempt++) {
-          try {
-            for (const interceptor of interceptors) {
-              for (const msg of validMessages) {
-                await interceptor.before?.(msg, batch.topic);
-              }
-            }
-            await handleBatch(validMessages, batch.topic, meta);
-            for (const interceptor of interceptors) {
-              for (const msg of validMessages) {
-                await interceptor.after?.(msg, batch.topic);
-              }
-            }
-            return;
-          } catch (error) {
-            const err = error instanceof Error ? error : new Error(String(error));
-            const isLastAttempt = attempt === maxAttempts;
-            if (isLastAttempt && maxAttempts > 1) {
-              const exhaustedError = new KafkaRetryExhaustedError(
-                batch.topic,
-                validMessages,
-                maxAttempts,
-                { cause: err }
-              );
-              for (const interceptor of interceptors) {
-                await interceptor.onError?.(
-                  validMessages,
-                  batch.topic,
-                  exhaustedError
-                );
-              }
-            } else {
-              for (const interceptor of interceptors) {
-                await interceptor.onError?.(
-                  validMessages,
-                  batch.topic,
-                  err
-                );
-              }
-            }
-            this.logger.error(
-              `Error processing batch from topic ${batch.topic} (attempt ${attempt}/${maxAttempts}):`,
-              err.stack
-            );
-            if (isLastAttempt) {
-              if (dlq) {
-                for (const msg of batch.messages) {
-                  if (msg.value) {
-                    await this.sendToDlq(batch.topic, msg.value.toString());
-                  }
-                }
-              }
-            } else {
-              await this.sleep(backoffMs * attempt);
-            }
+        await this.executeWithRetry(
+          () => handleBatch(validMessages, batch.topic, meta),
+          {
+            topic: batch.topic,
+            messages: validMessages,
+            rawMessages: batch.messages.filter((m) => m.value).map((m) => m.value.toString()),
+            interceptors,
+            dlq,
+            retry,
+            isBatch: true
           }
-        }
+        );
       }
     });
+    this.runningConsumers.set(gid, "eachBatch");
   }
+  // ── Consumer lifecycle ───────────────────────────────────────────
   async stopConsumer() {
     const tasks = [];
     for (const consumer of this.consumers.values()) {
@@ -453,6 +264,7 @@ var KafkaClient = class {
     }
     await Promise.allSettled(tasks);
     this.consumers.clear();
+    this.runningConsumers.clear();
     this.logger.log("All consumers disconnected");
   }
   /** Check broker connectivity and return available topics. */
@@ -479,61 +291,217 @@ var KafkaClient = class {
     }
     await Promise.allSettled(tasks);
     this.consumers.clear();
+    this.runningConsumers.clear();
     this.logger.log("All connections closed");
   }
-  // --- Private helpers ---
+  // ── Private helpers ──────────────────────────────────────────────
+  getOrCreateConsumer(groupId) {
+    const gid = groupId || this.defaultGroupId;
+    if (!this.consumers.has(gid)) {
+      this.consumers.set(gid, this.kafka.consumer({ groupId: gid }));
+    }
+    return this.consumers.get(gid);
+  }
+  resolveTopicName(topicOrDescriptor) {
+    if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
+    if (topicOrDescriptor && typeof topicOrDescriptor === "object" && "__topic" in topicOrDescriptor) {
+      return topicOrDescriptor.__topic;
+    }
+    return String(topicOrDescriptor);
+  }
+  async ensureTopic(topic2) {
+    if (!this.autoCreateTopicsEnabled || this.ensuredTopics.has(topic2)) return;
+    if (!this.isAdminConnected) {
+      await this.admin.connect();
+      this.isAdminConnected = true;
+    }
+    await this.admin.createTopics({
+      topics: [{ topic: topic2, numPartitions: 1 }]
+    });
+    this.ensuredTopics.add(topic2);
+  }
+  /** Register schema from descriptor into global registry (side-effect). */
+  registerSchema(topicOrDesc) {
+    if (topicOrDesc?.__schema) {
+      const topic2 = this.resolveTopicName(topicOrDesc);
+      this.schemaRegistry.set(topic2, topicOrDesc.__schema);
+    }
+  }
+  /** Validate message against schema. Pure — no side-effects on registry. */
+  validateMessage(topicOrDesc, message) {
+    if (topicOrDesc?.__schema) {
+      return topicOrDesc.__schema.parse(message);
+    }
+    if (this.strictSchemasEnabled && typeof topicOrDesc === "string") {
+      const schema = this.schemaRegistry.get(topicOrDesc);
+      if (schema) return schema.parse(message);
+    }
+    return message;
+  }
+  /**
+   * Build a kafkajs-ready send payload.
+   * Handles: topic resolution, schema registration, validation, JSON serialization.
+   */
+  buildSendPayload(topicOrDesc, messages) {
+    this.registerSchema(topicOrDesc);
+    const topic2 = this.resolveTopicName(topicOrDesc);
+    return {
+      topic: topic2,
+      messages: messages.map((m) => ({
+        value: JSON.stringify(this.validateMessage(topicOrDesc, m.value)),
+        key: m.key ?? null,
+        headers: m.headers
+      })),
+      acks: ACKS_ALL
+    };
+  }
+  /** Shared consumer setup: groupId check, schema map, connect, subscribe. */
+  async setupConsumer(topics, mode, options) {
+    const {
+      groupId: optGroupId,
+      fromBeginning = false,
+      retry,
+      dlq = false,
+      interceptors = [],
+      schemas: optionSchemas
+    } = options;
+    const gid = optGroupId || this.defaultGroupId;
+    const existingMode = this.runningConsumers.get(gid);
+    const oppositeMode = mode === "eachMessage" ? "eachBatch" : "eachMessage";
+    if (existingMode === oppositeMode) {
+      throw new Error(
+        `Cannot use ${mode} on consumer group "${gid}" \u2014 it is already running with ${oppositeMode}. Use a different groupId for this consumer.`
+      );
+    }
+    const consumer = this.getOrCreateConsumer(optGroupId);
+    const schemaMap = this.buildSchemaMap(topics, optionSchemas);
+    const topicNames = topics.map(
+      (t) => this.resolveTopicName(t)
+    );
+    await consumer.connect();
+    await this.subscribeWithRetry(consumer, topicNames, fromBeginning, options.subscribeRetry);
+    this.logger.log(
+      `${mode === "eachBatch" ? "Batch consumer" : "Consumer"} subscribed to topics: ${topicNames.join(", ")}`
+    );
+    return { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry };
+  }
   buildSchemaMap(topics, optionSchemas) {
     const schemaMap = /* @__PURE__ */ new Map();
     for (const t of topics) {
       if (t?.__schema) {
-        schemaMap.set(this.resolveTopicName(t), t.__schema);
+        const name = this.resolveTopicName(t);
+        schemaMap.set(name, t.__schema);
+        this.schemaRegistry.set(name, t.__schema);
       }
     }
     if (optionSchemas) {
       for (const [k, v] of optionSchemas) {
         schemaMap.set(k, v);
+        this.schemaRegistry.set(k, v);
       }
     }
     return schemaMap;
   }
-  async processMessage(parsedMessage, raw, topic2, handleMessage, opts) {
-    const { retry, dlq = false, interceptors = [] } = opts;
+  /** Parse raw message as JSON. Returns null on failure (logs error). */
+  parseJsonMessage(raw, topic2) {
+    try {
+      return JSON.parse(raw);
+    } catch (error) {
+      this.logger.error(
+        `Failed to parse message from topic ${topic2}:`,
+        toError(error).stack
+      );
+      return null;
+    }
+  }
+  /**
+   * Validate a parsed message against the schema map.
+   * On failure: logs error, sends to DLQ if enabled, calls interceptor.onError.
+   * Returns validated message or null.
+   */
+  async validateWithSchema(message, raw, topic2, schemaMap, interceptors, dlq) {
+    const schema = schemaMap.get(topic2);
+    if (!schema) return message;
+    try {
+      return schema.parse(message);
+    } catch (error) {
+      const err = toError(error);
+      const validationError = new KafkaValidationError(topic2, message, {
+        cause: err
+      });
+      this.logger.error(
+        `Schema validation failed for topic ${topic2}:`,
+        err.message
+      );
+      if (dlq) await this.sendToDlq(topic2, raw);
+      for (const interceptor of interceptors) {
+        await interceptor.onError?.(message, topic2, validationError);
+      }
+      return null;
+    }
+  }
+  /**
+   * Execute a handler with retry, interceptors, and DLQ support.
+   * Used by both single-message and batch consumers.
+   */
+  async executeWithRetry(fn, ctx) {
+    const { topic: topic2, messages, rawMessages, interceptors, dlq, retry, isBatch } = ctx;
     const maxAttempts = retry ? retry.maxRetries + 1 : 1;
     const backoffMs = retry?.backoffMs ?? 1e3;
     for (let attempt = 1; attempt <= maxAttempts; attempt++) {
       try {
-        for (const interceptor of interceptors) {
-          await interceptor.before?.(parsedMessage, topic2);
+        if (isBatch) {
+          for (const interceptor of interceptors) {
+            for (const msg of messages) {
+              await interceptor.before?.(msg, topic2);
+            }
+          }
+        } else {
+          for (const interceptor of interceptors) {
+            await interceptor.before?.(messages, topic2);
+          }
         }
-        await handleMessage(parsedMessage, topic2);
-        for (const interceptor of interceptors) {
-          await interceptor.after?.(parsedMessage, topic2);
+        await fn();
+        if (isBatch) {
+          for (const interceptor of interceptors) {
+            for (const msg of messages) {
+              await interceptor.after?.(msg, topic2);
+            }
+          }
+        } else {
+          for (const interceptor of interceptors) {
+            await interceptor.after?.(messages, topic2);
+          }
         }
         return;
       } catch (error) {
-        const err = error instanceof Error ? error : new Error(String(error));
+        const err = toError(error);
         const isLastAttempt = attempt === maxAttempts;
         if (isLastAttempt && maxAttempts > 1) {
           const exhaustedError = new KafkaRetryExhaustedError(
             topic2,
-            parsedMessage,
+            messages,
             maxAttempts,
             { cause: err }
           );
           for (const interceptor of interceptors) {
-            await interceptor.onError?.(parsedMessage, topic2, exhaustedError);
+            await interceptor.onError?.(messages, topic2, exhaustedError);
           }
         } else {
           for (const interceptor of interceptors) {
-            await interceptor.onError?.(parsedMessage, topic2, err);
+            await interceptor.onError?.(messages, topic2, err);
           }
         }
         this.logger.error(
-          `Error processing message from topic ${topic2} (attempt ${attempt}/${maxAttempts}):`,
+          `Error processing ${isBatch ? "batch" : "message"} from topic ${topic2} (attempt ${attempt}/${maxAttempts}):`,
           err.stack
         );
         if (isLastAttempt) {
-          if (dlq) await this.sendToDlq(topic2, raw);
+          if (dlq) {
+            for (const raw of rawMessages) {
+              await this.sendToDlq(topic2, raw);
+            }
+          }
         } else {
           await this.sleep(backoffMs * attempt);
         }
@@ -546,16 +514,33 @@ var KafkaClient = class {
       await this.producer.send({
         topic: dlqTopic,
         messages: [{ value: rawMessage }],
-        acks: -1
+        acks: ACKS_ALL
       });
       this.logger.warn(`Message sent to DLQ: ${dlqTopic}`);
     } catch (error) {
       this.logger.error(
         `Failed to send message to DLQ ${dlqTopic}:`,
-        error instanceof Error ? error.stack : String(error)
+        toError(error).stack
       );
     }
   }
+  async subscribeWithRetry(consumer, topics, fromBeginning, retryOpts) {
+    const maxAttempts = retryOpts?.retries ?? 5;
+    const backoffMs = retryOpts?.backoffMs ?? 5e3;
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+      try {
+        await consumer.subscribe({ topics, fromBeginning });
+        return;
+      } catch (error) {
+        if (attempt === maxAttempts) throw error;
+        const msg = toError(error).message;
+        this.logger.warn(
+          `Failed to subscribe to [${topics.join(", ")}] (attempt ${attempt}/${maxAttempts}): ${msg}. Retrying in ${backoffMs}ms...`
+        );
+        await this.sleep(backoffMs);
+      }
+    }
+  }
   sleep(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
   }