@drarzter/kafka-client 0.5.2 → 0.5.4

package/dist/index.js

@@ -221,9 +221,43 @@ async function sendToDlq(topic2, rawMessage, deps, meta) {
     );
   }
 }
+var RETRY_HEADER_ATTEMPT = "x-retry-attempt";
+var RETRY_HEADER_AFTER = "x-retry-after";
+var RETRY_HEADER_MAX_RETRIES = "x-retry-max-retries";
+var RETRY_HEADER_ORIGINAL_TOPIC = "x-retry-original-topic";
+async function sendToRetryTopic(originalTopic, rawMessages, attempt, maxRetries, delayMs, originalHeaders, deps) {
+  const retryTopic = `${originalTopic}.retry`;
+  const {
+    [RETRY_HEADER_ATTEMPT]: _a,
+    [RETRY_HEADER_AFTER]: _b,
+    [RETRY_HEADER_MAX_RETRIES]: _c,
+    [RETRY_HEADER_ORIGINAL_TOPIC]: _d,
+    ...userHeaders
+  } = originalHeaders;
+  const headers = {
+    ...userHeaders,
+    [RETRY_HEADER_ATTEMPT]: String(attempt),
+    [RETRY_HEADER_AFTER]: String(Date.now() + delayMs),
+    [RETRY_HEADER_MAX_RETRIES]: String(maxRetries),
+    [RETRY_HEADER_ORIGINAL_TOPIC]: originalTopic
+  };
+  try {
+    for (const raw of rawMessages) {
+      await deps.producer.send({ topic: retryTopic, messages: [{ value: raw, headers }] });
+    }
+    deps.logger.warn(
+      `Message queued in retry topic ${retryTopic} (attempt ${attempt}/${maxRetries})`
+    );
+  } catch (error) {
+    deps.logger.error(
+      `Failed to send message to retry topic ${retryTopic}:`,
+      toError(error).stack
+    );
+  }
+}
 async function executeWithRetry(fn, ctx, deps) {
-  const { envelope, rawMessages, interceptors, dlq, retry, isBatch } = ctx;
-  const maxAttempts = retry ? retry.maxRetries + 1 : 1;
+  const { envelope, rawMessages, interceptors, dlq, retry, isBatch, retryTopics } = ctx;
+  const maxAttempts = retryTopics ? 1 : retry ? retry.maxRetries + 1 : 1;
   const backoffMs = retry?.backoffMs ?? 1e3;
   const maxBackoffMs = retry?.maxBackoffMs ?? 3e4;
   const envelopes = Array.isArray(envelope) ? envelope : [envelope];
@@ -282,7 +316,19 @@ async function executeWithRetry(fn, ctx, deps) {
         `Error processing ${isBatch ? "batch" : "message"} from topic ${topic2} (attempt ${attempt}/${maxAttempts}):`,
         err.stack
       );
-      if (isLastAttempt) {
+      if (retryTopics && retry) {
+        const cap = Math.min(backoffMs, maxBackoffMs);
+        const delay = Math.floor(Math.random() * cap);
+        await sendToRetryTopic(
+          topic2,
+          rawMessages,
+          1,
+          retry.maxRetries,
+          delay,
+          envelopes[0]?.headers ?? {},
+          deps
+        );
+      } else if (isLastAttempt) {
         if (dlq) {
           const dlqMeta = {
             error: err,
@@ -460,7 +506,12 @@ var KafkaClient = class {
     this.logger.log("Producer disconnected");
   }
   async startConsumer(topics, handleMessage, options = {}) {
-    const { consumer, schemaMap, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", options);
+    if (options.retryTopics && !options.retry) {
+      throw new Error(
+        "retryTopics requires retry to be configured \u2014 set retry.maxRetries to enable the retry topic chain"
+      );
+    }
+    const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", options);
     const deps = { logger: this.logger, producer: this.producer, instrumentation: this.instrumentation, onMessageLost: this.onMessageLost };
     await consumer.run({
       eachMessage: async ({ topic: topic2, partition, message }) => {
@@ -494,12 +545,23 @@ var KafkaClient = class {
             { correlationId: envelope.correlationId, traceparent: envelope.traceparent },
             () => handleMessage(envelope)
           ),
-          { envelope, rawMessages: [raw], interceptors, dlq, retry },
+          { envelope, rawMessages: [raw], interceptors, dlq, retry, retryTopics: options.retryTopics },
           deps
         );
       }
     });
     this.runningConsumers.set(gid, "eachMessage");
+    if (options.retryTopics && retry) {
+      await this.startRetryTopicConsumers(
+        topicNames,
+        gid,
+        handleMessage,
+        retry,
+        dlq,
+        interceptors,
+        schemaMap
+      );
+    }
   }
   async startBatchConsumer(topics, handleBatch, options = {}) {
     const { consumer, schemaMap, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", options);
@@ -564,15 +626,28 @@ var KafkaClient = class {
     this.runningConsumers.set(gid, "eachBatch");
   }
   // ── Consumer lifecycle ───────────────────────────────────────────
-  async stopConsumer() {
-    const tasks = [];
-    for (const consumer of this.consumers.values()) {
-      tasks.push(consumer.disconnect());
+  async stopConsumer(groupId) {
+    if (groupId !== void 0) {
+      const consumer = this.consumers.get(groupId);
+      if (!consumer) {
+        this.logger.warn(`stopConsumer: no active consumer for group "${groupId}"`);
+        return;
+      }
+      await consumer.disconnect().catch(() => {
+      });
+      this.consumers.delete(groupId);
+      this.runningConsumers.delete(groupId);
+      this.logger.log(`Consumer disconnected: group "${groupId}"`);
+    } else {
+      const tasks = Array.from(this.consumers.values()).map(
+        (c) => c.disconnect().catch(() => {
+        })
+      );
+      await Promise.allSettled(tasks);
+      this.consumers.clear();
+      this.runningConsumers.clear();
+      this.logger.log("All consumers disconnected");
     }
-    await Promise.allSettled(tasks);
-    this.consumers.clear();
-    this.runningConsumers.clear();
-    this.logger.log("All consumers disconnected");
   }
   /** Check broker connectivity and return status, clientId, and available topics. */
   async checkStatus() {
@@ -605,7 +680,164 @@ var KafkaClient = class {
     this.runningConsumers.clear();
     this.logger.log("All connections closed");
   }
+  // ── Retry topic chain ────────────────────────────────────────────
+  /**
+   * Auto-start companion consumers on `<topic>.retry` for each original topic.
+   * Called by `startConsumer` when `retryTopics: true`.
+   *
+   * Flow per message:
+   *   1. Sleep until `x-retry-after` (scheduled by the main consumer or previous retry hop)
+   *   2. Call the original handler
+   *   3. On failure: if retries remain → re-send to `<originalTopic>.retry` with incremented attempt
+   *                  if exhausted       → DLQ or onMessageLost
+   */
+  async startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap) {
+    const retryTopicNames = originalTopics.map((t) => `${t}.retry`);
+    const retryGroupId = `${originalGroupId}-retry`;
+    const backoffMs = retry.backoffMs ?? 1e3;
+    const maxBackoffMs = retry.maxBackoffMs ?? 3e4;
+    const deps = {
+      logger: this.logger,
+      producer: this.producer,
+      instrumentation: this.instrumentation,
+      onMessageLost: this.onMessageLost
+    };
+    for (const rt of retryTopicNames) {
+      await this.ensureTopic(rt);
+    }
+    const consumer = this.getOrCreateConsumer(retryGroupId, false, true);
+    await consumer.connect();
+    await subscribeWithRetry(consumer, retryTopicNames, this.logger);
+    await consumer.run({
+      eachMessage: async ({ topic: retryTopic, partition, message }) => {
+        if (!message.value) return;
+        const raw = message.value.toString();
+        const parsed = parseJsonMessage(raw, retryTopic, this.logger);
+        if (parsed === null) return;
+        const headers = decodeHeaders(message.headers);
+        const originalTopic = headers[RETRY_HEADER_ORIGINAL_TOPIC] ?? retryTopic.replace(/\.retry$/, "");
+        const currentAttempt = parseInt(
+          headers[RETRY_HEADER_ATTEMPT] ?? "1",
+          10
+        );
+        const maxRetries = parseInt(
+          headers[RETRY_HEADER_MAX_RETRIES] ?? String(retry.maxRetries),
+          10
+        );
+        const retryAfter = parseInt(
+          headers[RETRY_HEADER_AFTER] ?? "0",
+          10
+        );
+        const remaining = retryAfter - Date.now();
+        if (remaining > 0) {
+          consumer.pause([{ topic: retryTopic, partitions: [partition] }]);
+          await sleep(remaining);
+          consumer.resume([{ topic: retryTopic, partitions: [partition] }]);
+        }
+        const validated = await validateWithSchema(
+          parsed,
+          raw,
+          originalTopic,
+          schemaMap,
+          interceptors,
+          dlq,
+          { ...deps, originalHeaders: headers }
+        );
+        if (validated === null) return;
+        const envelope = extractEnvelope(
+          validated,
+          headers,
+          originalTopic,
+          partition,
+          message.offset
+        );
+        try {
+          const cleanups = [];
+          for (const inst of this.instrumentation) {
+            const c = inst.beforeConsume?.(envelope);
+            if (typeof c === "function") cleanups.push(c);
+          }
+          for (const interceptor of interceptors) await interceptor.before?.(envelope);
+          await runWithEnvelopeContext(
+            { correlationId: envelope.correlationId, traceparent: envelope.traceparent },
+            () => handleMessage(envelope)
+          );
+          for (const interceptor of interceptors) await interceptor.after?.(envelope);
+          for (const cleanup of cleanups) cleanup();
+        } catch (error) {
+          const err = toError(error);
+          const nextAttempt = currentAttempt + 1;
+          const exhausted = currentAttempt >= maxRetries;
+          for (const inst of this.instrumentation) inst.onConsumeError?.(envelope, err);
+          const reportedError = exhausted && maxRetries > 1 ? new KafkaRetryExhaustedError(originalTopic, [envelope.payload], maxRetries, { cause: err }) : err;
+          for (const interceptor of interceptors) {
+            await interceptor.onError?.(envelope, reportedError);
+          }
+          this.logger.error(
+            `Retry consumer error for ${originalTopic} (attempt ${currentAttempt}/${maxRetries}):`,
+            err.stack
+          );
+          if (!exhausted) {
+            const cap = Math.min(backoffMs * 2 ** currentAttempt, maxBackoffMs);
+            const delay = Math.floor(Math.random() * cap);
+            await sendToRetryTopic(
+              originalTopic,
+              [raw],
+              nextAttempt,
+              maxRetries,
+              delay,
+              headers,
+              deps
+            );
+          } else if (dlq) {
+            await sendToDlq(originalTopic, raw, deps, {
+              error: err,
+              // +1 to account for the main consumer's initial attempt before
+              // routing to the retry topic, making this consistent with the
+              // in-process retry path where attempt counts all tries.
+              attempt: currentAttempt + 1,
+              originalHeaders: headers
+            });
+          } else {
+            await deps.onMessageLost?.({
+              topic: originalTopic,
+              error: err,
+              attempt: currentAttempt,
+              headers
+            });
+          }
+        }
+      }
+    });
+    this.runningConsumers.set(retryGroupId, "eachMessage");
+    await this.waitForPartitionAssignment(consumer, retryTopicNames);
+    this.logger.log(
+      `Retry topic consumers started for: ${originalTopics.join(", ")} (group: ${retryGroupId})`
+    );
+  }
   // ── Private helpers ──────────────────────────────────────────────
+  /**
+   * Poll `consumer.assignment()` until the consumer has received at least one
+   * partition for the given topics, then return. Logs a warning and returns
+   * (rather than throwing) on timeout so that a slow broker does not break
+   * the caller — in the worst case a message sent immediately after would be
+   * missed, which is the same behaviour as before this guard was added.
+   */
+  async waitForPartitionAssignment(consumer, topics, timeoutMs = 1e4) {
+    const topicSet = new Set(topics);
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+      try {
+        const assigned = consumer.assignment();
+        if (assigned.some((a) => topicSet.has(a.topic))) return;
+      } catch {
+      }
+      await sleep(200);
+    }
+    this.logger.warn(
+      `Retry consumer did not receive partition assignments for [${topics.join(", ")}] within ${timeoutMs}ms`
+    );
+  }
   getOrCreateConsumer(groupId, fromBeginning, autoCommit) {
     if (!this.consumers.has(groupId)) {
       this.consumers.set(