@drarzter/kafka-client 0.5.6 → 0.5.7

package/dist/index.js

@@ -118,6 +118,38 @@ function extractEnvelope(payload, headers, topic2, partition, offset) {
   };
 }
 
+// src/client/errors.ts
+var KafkaProcessingError = class extends Error {
+  constructor(message, topic2, originalMessage, options) {
+    super(message, options);
+    this.topic = topic2;
+    this.originalMessage = originalMessage;
+    this.name = "KafkaProcessingError";
+    if (options?.cause) this.cause = options.cause;
+  }
+};
+var KafkaValidationError = class extends Error {
+  constructor(topic2, originalMessage, options) {
+    super(`Schema validation failed for topic "${topic2}"`, options);
+    this.topic = topic2;
+    this.originalMessage = originalMessage;
+    this.name = "KafkaValidationError";
+    if (options?.cause) this.cause = options.cause;
+  }
+};
+var KafkaRetryExhaustedError = class extends KafkaProcessingError {
+  constructor(topic2, originalMessage, attempts, options) {
+    super(
+      `Message processing failed after ${attempts} attempts on topic "${topic2}"`,
+      topic2,
+      originalMessage,
+      options
+    );
+    this.attempts = attempts;
+    this.name = "KafkaRetryExhaustedError";
+  }
+};
+
 // src/client/kafka.client/producer-ops.ts
 function resolveTopicName(topicOrDescriptor) {
   if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
@@ -133,17 +165,31 @@ function registerSchema(topicOrDesc, schemaRegistry) {
   }
 }
 async function validateMessage(topicOrDesc, message, deps) {
+  const topicName = resolveTopicName(topicOrDesc);
   if (topicOrDesc?.__schema) {
-    return await topicOrDesc.__schema.parse(message);
+    try {
+      return await topicOrDesc.__schema.parse(message);
+    } catch (error) {
+      throw new KafkaValidationError(topicName, message, {
+        cause: error instanceof Error ? error : new Error(String(error))
+      });
+    }
   }
   if (deps.strictSchemasEnabled && typeof topicOrDesc === "string") {
     const schema = deps.schemaRegistry.get(topicOrDesc);
-    if (schema) return await schema.parse(message);
+    if (schema) {
+      try {
+        return await schema.parse(message);
+      } catch (error) {
+        throw new KafkaValidationError(topicName, message, {
+          cause: error instanceof Error ? error : new Error(String(error))
+        });
+      }
+    }
   }
   return message;
 }
 async function buildSendPayload(topicOrDesc, messages, deps) {
-  registerSchema(topicOrDesc, deps.schemaRegistry);
   const topic2 = resolveTopicName(topicOrDesc);
   const builtMessages = await Promise.all(
     messages.map(async (m) => {
@@ -220,38 +266,6 @@ function buildSchemaMap(topics, schemaRegistry, optionSchemas) {
   return schemaMap;
 }
 
-// src/client/errors.ts
-var KafkaProcessingError = class extends Error {
-  constructor(message, topic2, originalMessage, options) {
-    super(message, options);
-    this.topic = topic2;
-    this.originalMessage = originalMessage;
-    this.name = "KafkaProcessingError";
-    if (options?.cause) this.cause = options.cause;
-  }
-};
-var KafkaValidationError = class extends Error {
-  constructor(topic2, originalMessage, options) {
-    super(`Schema validation failed for topic "${topic2}"`, options);
-    this.topic = topic2;
-    this.originalMessage = originalMessage;
-    this.name = "KafkaValidationError";
-    if (options?.cause) this.cause = options.cause;
-  }
-};
-var KafkaRetryExhaustedError = class extends KafkaProcessingError {
-  constructor(topic2, originalMessage, attempts, options) {
-    super(
-      `Message processing failed after ${attempts} attempts on topic "${topic2}"`,
-      topic2,
-      originalMessage,
-      options
-    );
-    this.attempts = attempts;
-    this.name = "KafkaRetryExhaustedError";
-  }
-};
-
 // src/client/consumer/pipeline.ts
 function toError(error) {
   return error instanceof Error ? error : new Error(String(error));
@@ -339,7 +353,7 @@ var RETRY_HEADER_AFTER = "x-retry-after";
 var RETRY_HEADER_MAX_RETRIES = "x-retry-max-retries";
 var RETRY_HEADER_ORIGINAL_TOPIC = "x-retry-original-topic";
 async function sendToRetryTopic(originalTopic, rawMessages, attempt, maxRetries, delayMs, originalHeaders, deps) {
-  const retryTopic = `${originalTopic}.retry`;
+  const retryTopic = `${originalTopic}.retry.${attempt}`;
   const {
     [RETRY_HEADER_ATTEMPT]: _a,
     [RETRY_HEADER_AFTER]: _b,
@@ -643,7 +657,7 @@ async function waitForPartitionAssignment(consumer, topics, logger, timeoutMs =
     `Retry consumer did not receive partition assignments for [${topics.join(", ")}] within ${timeoutMs}ms`
   );
 }
-async function startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap, deps) {
+async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopics, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs) {
   const {
     logger,
     producer,
@@ -653,43 +667,48 @@ async function startRetryTopicConsumers(originalTopics, originalGroupId, handleM
     getOrCreateConsumer: getOrCreateConsumer2,
     runningConsumers
   } = deps;
-  const retryTopicNames = originalTopics.map((t) => `${t}.retry`);
-  const retryGroupId = `${originalGroupId}-retry`;
   const backoffMs = retry.backoffMs ?? 1e3;
   const maxBackoffMs = retry.maxBackoffMs ?? 3e4;
   const pipelineDeps = { logger, producer, instrumentation, onMessageLost };
-  for (const rt of retryTopicNames) {
-    await ensureTopic(rt);
+  for (const lt of levelTopics) {
+    await ensureTopic(lt);
   }
-  const consumer = getOrCreateConsumer2(retryGroupId, false, true);
+  const consumer = getOrCreateConsumer2(levelGroupId, false, false);
   await consumer.connect();
-  await subscribeWithRetry(consumer, retryTopicNames, logger);
+  await subscribeWithRetry(consumer, levelTopics, logger);
   await consumer.run({
-    eachMessage: async ({ topic: retryTopic, partition, message }) => {
-      if (!message.value) return;
-      const raw = message.value.toString();
-      const parsed = parseJsonMessage(raw, retryTopic, logger);
-      if (parsed === null) return;
+    eachMessage: async ({ topic: levelTopic, partition, message }) => {
+      const nextOffset = {
+        topic: levelTopic,
+        partition,
+        offset: (parseInt(message.offset, 10) + 1).toString()
+      };
+      if (!message.value) {
+        await consumer.commitOffsets([nextOffset]);
+        return;
+      }
       const headers = decodeHeaders(message.headers);
-      const originalTopic = headers[RETRY_HEADER_ORIGINAL_TOPIC] ?? retryTopic.replace(/\.retry$/, "");
-      const currentAttempt = parseInt(
-        headers[RETRY_HEADER_ATTEMPT] ?? "1",
-        10
-      );
-      const maxRetries = parseInt(
-        headers[RETRY_HEADER_MAX_RETRIES] ?? String(retry.maxRetries),
-        10
-      );
       const retryAfter = parseInt(
         headers[RETRY_HEADER_AFTER] ?? "0",
         10
       );
       const remaining = retryAfter - Date.now();
       if (remaining > 0) {
-        consumer.pause([{ topic: retryTopic, partitions: [partition] }]);
+        consumer.pause([{ topic: levelTopic, partitions: [partition] }]);
         await sleep(remaining);
-        consumer.resume([{ topic: retryTopic, partitions: [partition] }]);
+        consumer.resume([{ topic: levelTopic, partitions: [partition] }]);
       }
+      const raw = message.value.toString();
+      const parsed = parseJsonMessage(raw, levelTopic, logger);
+      if (parsed === null) {
+        await consumer.commitOffsets([nextOffset]);
+        return;
+      }
+      const currentMaxRetries = parseInt(
+        headers[RETRY_HEADER_MAX_RETRIES] ?? String(retry.maxRetries),
+        10
+      );
+      const originalTopic = headers[RETRY_HEADER_ORIGINAL_TOPIC] ?? levelTopic.replace(/\.retry\.\d+$/, "");
       const validated = await validateWithSchema(
         parsed,
         raw,
@@ -699,7 +718,10 @@ async function startRetryTopicConsumers(originalTopics, originalGroupId, handleM
         dlq,
         { ...pipelineDeps, originalHeaders: headers }
       );
-      if (validated === null) return;
+      if (validated === null) {
+        await consumer.commitOffsets([nextOffset]);
+        return;
+      }
       const envelope = extractEnvelope(
         validated,
         headers,
@@ -719,62 +741,81 @@ async function startRetryTopicConsumers(originalTopics, originalGroupId, handleM
         interceptors,
         instrumentation
       );
-      if (error) {
-        const nextAttempt = currentAttempt + 1;
-        const exhausted = currentAttempt >= maxRetries;
-        const reportedError = exhausted && maxRetries > 1 ? new KafkaRetryExhaustedError(
+      if (!error) {
+        await consumer.commitOffsets([nextOffset]);
+        return;
+      }
+      const exhausted = level >= currentMaxRetries;
+      const reportedError = exhausted && currentMaxRetries > 1 ? new KafkaRetryExhaustedError(
+        originalTopic,
+        [envelope.payload],
+        currentMaxRetries,
+        { cause: error }
+      ) : error;
+      await notifyInterceptorsOnError([envelope], interceptors, reportedError);
+      logger.error(
+        `Retry consumer error for ${originalTopic} (level ${level}/${currentMaxRetries}):`,
+        error.stack
+      );
+      if (!exhausted) {
+        const nextLevel = level + 1;
+        const cap = Math.min(backoffMs * 2 ** level, maxBackoffMs);
+        const delay = Math.floor(Math.random() * cap);
+        await sendToRetryTopic(
           originalTopic,
-          [envelope.payload],
-          maxRetries,
-          { cause: error }
-        ) : error;
-        await notifyInterceptorsOnError(
-          [envelope],
-          interceptors,
-          reportedError
+          [raw],
+          nextLevel,
+          currentMaxRetries,
+          delay,
+          headers,
+          pipelineDeps
         );
-        logger.error(
-          `Retry consumer error for ${originalTopic} (attempt ${currentAttempt}/${maxRetries}):`,
-          error.stack
-        );
-        if (!exhausted) {
-          const cap = Math.min(backoffMs * 2 ** currentAttempt, maxBackoffMs);
-          const delay = Math.floor(Math.random() * cap);
-          await sendToRetryTopic(
-            originalTopic,
-            [raw],
-            nextAttempt,
-            maxRetries,
-            delay,
-            headers,
-            pipelineDeps
-          );
-        } else if (dlq) {
-          await sendToDlq(originalTopic, raw, pipelineDeps, {
-            error,
-            // +1 to account for the main consumer's initial attempt before
-            // routing to the retry topic, making this consistent with the
-            // in-process retry path where attempt counts all tries.
-            attempt: currentAttempt + 1,
-            originalHeaders: headers
-          });
-        } else {
-          await onMessageLost?.({
-            topic: originalTopic,
-            error,
-            attempt: currentAttempt,
-            headers
-          });
-        }
+      } else if (dlq) {
+        await sendToDlq(originalTopic, raw, pipelineDeps, {
+          error,
+          // +1 to account for the main consumer's initial attempt before routing.
+          attempt: level + 1,
+          originalHeaders: headers
+        });
+      } else {
+        await onMessageLost?.({
+          topic: originalTopic,
+          error,
+          attempt: level,
+          headers
+        });
       }
+      await consumer.commitOffsets([nextOffset]);
     }
   });
-  runningConsumers.set(retryGroupId, "eachMessage");
-  await waitForPartitionAssignment(consumer, retryTopicNames, logger);
+  runningConsumers.set(levelGroupId, "eachMessage");
+  await waitForPartitionAssignment(consumer, levelTopics, logger, assignmentTimeoutMs);
   logger.log(
-    `Retry topic consumers started for: ${originalTopics.join(", ")} (group: ${retryGroupId})`
+    `Retry level ${level}/${retry.maxRetries} consumer started for: ${originalTopics.join(", ")} (group: ${levelGroupId})`
   );
 }
+async function startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs) {
+  const levelGroupIds = [];
+  for (let level = 1; level <= retry.maxRetries; level++) {
+    const levelTopics = originalTopics.map((t) => `${t}.retry.${level}`);
+    const levelGroupId = `${originalGroupId}-retry.${level}`;
+    await startLevelConsumer(
+      level,
+      levelTopics,
+      levelGroupId,
+      originalTopics,
+      handleMessage,
+      retry,
+      dlq,
+      interceptors,
+      schemaMap,
+      deps,
+      assignmentTimeoutMs
+    );
+    levelGroupIds.push(levelGroupId);
+  }
+  return levelGroupIds;
+}
 
 // src/client/kafka.client/index.ts
 var { Kafka: KafkaClass, logLevel: KafkaLogLevel } = import_kafka_javascript.KafkaJS;
@@ -793,6 +834,8 @@ var KafkaClient = class {
   schemaRegistry = /* @__PURE__ */ new Map();
   runningConsumers = /* @__PURE__ */ new Map();
   consumerCreationOptions = /* @__PURE__ */ new Map();
+  /** Maps each main consumer groupId to its companion retry level groupIds. */
+  companionGroupIds = /* @__PURE__ */ new Map();
   instrumentation;
   onMessageLost;
   onRebalance;
@@ -929,7 +972,7 @@ var KafkaClient = class {
     });
     this.runningConsumers.set(gid, "eachMessage");
     if (options.retryTopics && retry) {
-      await startRetryTopicConsumers(
+      const companions = await startRetryTopicConsumers(
         topicNames,
         gid,
         handleMessage,
@@ -937,8 +980,10 @@ var KafkaClient = class {
         dlq,
         interceptors,
         schemaMap,
-        this.retryTopicDeps
+        this.retryTopicDeps,
+        options.retryTopicAssignmentTimeoutMs
       );
+      this.companionGroupIds.set(gid, companions);
     }
     return { groupId: gid, stop: () => this.stopConsumer(gid) };
   }
@@ -980,6 +1025,19 @@ var KafkaClient = class {
       this.runningConsumers.delete(groupId);
       this.consumerCreationOptions.delete(groupId);
       this.logger.log(`Consumer disconnected: group "${groupId}"`);
+      const companions = this.companionGroupIds.get(groupId) ?? [];
+      for (const cGroupId of companions) {
+        const cConsumer = this.consumers.get(cGroupId);
+        if (cConsumer) {
+          await cConsumer.disconnect().catch(() => {
+          });
+          this.consumers.delete(cGroupId);
+          this.runningConsumers.delete(cGroupId);
+          this.consumerCreationOptions.delete(cGroupId);
+          this.logger.log(`Retry consumer disconnected: group "${cGroupId}"`);
+        }
+      }
+      this.companionGroupIds.delete(groupId);
     } else {
       const tasks = Array.from(this.consumers.values()).map(
         (c) => c.disconnect().catch(() => {
@@ -989,6 +1047,7 @@ var KafkaClient = class {
       this.consumers.clear();
       this.runningConsumers.clear();
       this.consumerCreationOptions.clear();
+      this.companionGroupIds.clear();
       this.logger.log("All consumers disconnected");
     }
   }
@@ -1018,14 +1077,22 @@ var KafkaClient = class {
     }
     return result;
   }
-  /** Check broker connectivity and return status, clientId, and available topics. */
+  /** Check broker connectivity. Never throws — returns a discriminated union. */
   async checkStatus() {
-    if (!this.isAdminConnected) {
-      await this.admin.connect();
-      this.isAdminConnected = true;
+    try {
+      if (!this.isAdminConnected) {
+        await this.admin.connect();
+        this.isAdminConnected = true;
+      }
+      const topics = await this.admin.listTopics();
+      return { status: "up", clientId: this.clientId, topics };
+    } catch (error) {
+      return {
+        status: "down",
+        clientId: this.clientId,
+        error: error instanceof Error ? error.message : String(error)
+      };
     }
-    const topics = await this.admin.listTopics();
-    return { status: "up", clientId: this.clientId, topics };
   }
   getClientId() {
     return this.clientId;
@@ -1048,10 +1115,12 @@ var KafkaClient = class {
     this.consumers.clear();
     this.runningConsumers.clear();
     this.consumerCreationOptions.clear();
+    this.companionGroupIds.clear();
     this.logger.log("All connections closed");
   }
   // ── Private helpers ──────────────────────────────────────────────
   async preparePayload(topicOrDesc, messages) {
+    registerSchema(topicOrDesc, this.schemaRegistry);
     const payload = await buildSendPayload(
       topicOrDesc,
       messages,
@@ -1185,16 +1254,18 @@ var KafkaClient = class {
 
 // src/client/message/topic.ts
 function topic(name) {
-  const fn = () => ({
-    __topic: name,
-    __type: void 0
-  });
-  fn.schema = (schema) => ({
-    __topic: name,
-    __type: void 0,
-    __schema: schema
-  });
-  return fn;
+  return {
+    /** Provide an explicit message type without a runtime schema. */
+    type: () => ({
+      __topic: name,
+      __type: void 0
+    }),
+    schema: (schema) => ({
+      __topic: name,
+      __type: void 0,
+      __schema: schema
+    })
+  };
 }
 
 // src/nest/kafka.module.ts
@@ -1383,15 +1454,7 @@ KafkaModule = __decorateClass([
 var import_common4 = require("@nestjs/common");
 var KafkaHealthIndicator = class {
   async check(client) {
-    try {
-      return await client.checkStatus();
-    } catch (error) {
-      return {
-        status: "down",
-        clientId: client.clientId,
-        error: error instanceof Error ? error.message : String(error)
-      };
-    }
+    return client.checkStatus();
   }
 };
 KafkaHealthIndicator = __decorateClass([