@drarzter/kafka-client 0.5.1 → 0.5.4

package/dist/index.js

@@ -172,7 +172,7 @@ async function validateWithSchema(message, raw, topic2, schemaMap, interceptors,
   const schema = schemaMap.get(topic2);
   if (!schema) return message;
   try {
-    return schema.parse(message);
+    return await schema.parse(message);
   } catch (error) {
     const err = toError(error);
     const validationError = new KafkaValidationError(topic2, message, {
@@ -182,20 +182,36 @@ async function validateWithSchema(message, raw, topic2, schemaMap, interceptors,
       `Schema validation failed for topic ${topic2}:`,
       err.message
     );
-    if (dlq) await sendToDlq(topic2, raw, deps);
-    const errorEnvelope = extractEnvelope(message, {}, topic2, -1, "");
+    if (dlq) {
+      await sendToDlq(topic2, raw, deps, {
+        error: validationError,
+        attempt: 0,
+        originalHeaders: deps.originalHeaders
+      });
+    } else {
+      await deps.onMessageLost?.({ topic: topic2, error: validationError, attempt: 0, headers: deps.originalHeaders ?? {} });
+    }
+    const errorEnvelope = extractEnvelope(message, deps.originalHeaders ?? {}, topic2, -1, "");
     for (const interceptor of interceptors) {
       await interceptor.onError?.(errorEnvelope, validationError);
     }
     return null;
   }
 }
-async function sendToDlq(topic2, rawMessage, deps) {
+async function sendToDlq(topic2, rawMessage, deps, meta) {
   const dlqTopic = `${topic2}.dlq`;
+  const headers = {
+    ...meta?.originalHeaders ?? {},
+    "x-dlq-original-topic": topic2,
+    "x-dlq-failed-at": (/* @__PURE__ */ new Date()).toISOString(),
+    "x-dlq-error-message": meta?.error.message ?? "unknown",
+    "x-dlq-error-stack": meta?.error.stack?.slice(0, 2e3) ?? "",
+    "x-dlq-attempt-count": String(meta?.attempt ?? 0)
+  };
   try {
     await deps.producer.send({
       topic: dlqTopic,
-      messages: [{ value: rawMessage }]
+      messages: [{ value: rawMessage, headers }]
     });
     deps.logger.warn(`Message sent to DLQ: ${dlqTopic}`);
   } catch (error) {
@@ -205,10 +221,45 @@ async function sendToDlq(topic2, rawMessage, deps) {
     );
   }
 }
+var RETRY_HEADER_ATTEMPT = "x-retry-attempt";
+var RETRY_HEADER_AFTER = "x-retry-after";
+var RETRY_HEADER_MAX_RETRIES = "x-retry-max-retries";
+var RETRY_HEADER_ORIGINAL_TOPIC = "x-retry-original-topic";
+async function sendToRetryTopic(originalTopic, rawMessages, attempt, maxRetries, delayMs, originalHeaders, deps) {
+  const retryTopic = `${originalTopic}.retry`;
+  const {
+    [RETRY_HEADER_ATTEMPT]: _a,
+    [RETRY_HEADER_AFTER]: _b,
+    [RETRY_HEADER_MAX_RETRIES]: _c,
+    [RETRY_HEADER_ORIGINAL_TOPIC]: _d,
+    ...userHeaders
+  } = originalHeaders;
+  const headers = {
+    ...userHeaders,
+    [RETRY_HEADER_ATTEMPT]: String(attempt),
+    [RETRY_HEADER_AFTER]: String(Date.now() + delayMs),
+    [RETRY_HEADER_MAX_RETRIES]: String(maxRetries),
+    [RETRY_HEADER_ORIGINAL_TOPIC]: originalTopic
+  };
+  try {
+    for (const raw of rawMessages) {
+      await deps.producer.send({ topic: retryTopic, messages: [{ value: raw, headers }] });
+    }
+    deps.logger.warn(
+      `Message queued in retry topic ${retryTopic} (attempt ${attempt}/${maxRetries})`
+    );
+  } catch (error) {
+    deps.logger.error(
+      `Failed to send message to retry topic ${retryTopic}:`,
+      toError(error).stack
+    );
+  }
+}
 async function executeWithRetry(fn, ctx, deps) {
-  const { envelope, rawMessages, interceptors, dlq, retry, isBatch } = ctx;
-  const maxAttempts = retry ? retry.maxRetries + 1 : 1;
+  const { envelope, rawMessages, interceptors, dlq, retry, isBatch, retryTopics } = ctx;
+  const maxAttempts = retryTopics ? 1 : retry ? retry.maxRetries + 1 : 1;
   const backoffMs = retry?.backoffMs ?? 1e3;
+  const maxBackoffMs = retry?.maxBackoffMs ?? 3e4;
   const envelopes = Array.isArray(envelope) ? envelope : [envelope];
   const topic2 = envelopes[0]?.topic ?? "unknown";
   for (let attempt = 1; attempt <= maxAttempts; attempt++) {
@@ -265,14 +316,39 @@ async function executeWithRetry(fn, ctx, deps) {
         `Error processing ${isBatch ? "batch" : "message"} from topic ${topic2} (attempt ${attempt}/${maxAttempts}):`,
         err.stack
       );
-      if (isLastAttempt) {
+      if (retryTopics && retry) {
+        const cap = Math.min(backoffMs, maxBackoffMs);
+        const delay = Math.floor(Math.random() * cap);
+        await sendToRetryTopic(
+          topic2,
+          rawMessages,
+          1,
+          retry.maxRetries,
+          delay,
+          envelopes[0]?.headers ?? {},
+          deps
+        );
+      } else if (isLastAttempt) {
         if (dlq) {
+          const dlqMeta = {
+            error: err,
+            attempt,
+            originalHeaders: envelopes[0]?.headers
+          };
           for (const raw of rawMessages) {
-            await sendToDlq(topic2, raw, deps);
+            await sendToDlq(topic2, raw, deps, dlqMeta);
           }
+        } else {
+          await deps.onMessageLost?.({
+            topic: topic2,
+            error: err,
+            attempt,
+            headers: envelopes[0]?.headers ?? {}
+          });
         }
       } else {
-        await sleep(backoffMs * attempt);
+        const cap = Math.min(backoffMs * 2 ** (attempt - 1), maxBackoffMs);
+        await sleep(Math.random() * cap);
       }
     }
   }
@@ -314,6 +390,7 @@ var KafkaClient = class {
   schemaRegistry = /* @__PURE__ */ new Map();
   runningConsumers = /* @__PURE__ */ new Map();
   instrumentation;
+  onMessageLost;
   isAdminConnected = false;
   clientId;
   constructor(clientId, groupId, brokers, options) {
@@ -328,6 +405,7 @@ var KafkaClient = class {
     this.strictSchemasEnabled = options?.strictSchemas ?? true;
     this.numPartitions = options?.numPartitions ?? 1;
     this.instrumentation = options?.instrumentation ?? [];
+    this.onMessageLost = options?.onMessageLost;
     this.kafka = new KafkaClass({
       kafkaJS: {
         clientId: this.clientId,
@@ -343,7 +421,7 @@ var KafkaClient = class {
     this.admin = this.kafka.admin();
   }
   async sendMessage(topicOrDesc, message, options = {}) {
-    const payload = this.buildSendPayload(topicOrDesc, [
+    const payload = await this.buildSendPayload(topicOrDesc, [
       {
         value: message,
         key: options.key,
@@ -360,7 +438,7 @@ var KafkaClient = class {
     }
   }
   async sendBatch(topicOrDesc, messages) {
-    const payload = this.buildSendPayload(topicOrDesc, messages);
+    const payload = await this.buildSendPayload(topicOrDesc, messages);
     await this.ensureTopic(payload.topic);
     await this.producer.send(payload);
     for (const inst of this.instrumentation) {
@@ -384,7 +462,7 @@ var KafkaClient = class {
     try {
       const ctx = {
         send: async (topicOrDesc, message, options = {}) => {
-          const payload = this.buildSendPayload(topicOrDesc, [
+          const payload = await this.buildSendPayload(topicOrDesc, [
             {
               value: message,
               key: options.key,
@@ -398,7 +476,7 @@ var KafkaClient = class {
           await tx.send(payload);
         },
         sendBatch: async (topicOrDesc, messages) => {
-          const payload = this.buildSendPayload(topicOrDesc, messages);
+          const payload = await this.buildSendPayload(topicOrDesc, messages);
           await this.ensureTopic(payload.topic);
           await tx.send(payload);
         }
@@ -428,8 +506,13 @@ var KafkaClient = class {
     this.logger.log("Producer disconnected");
   }
   async startConsumer(topics, handleMessage, options = {}) {
-    const { consumer, schemaMap, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", options);
-    const deps = { logger: this.logger, producer: this.producer, instrumentation: this.instrumentation };
+    if (options.retryTopics && !options.retry) {
+      throw new Error(
+        "retryTopics requires retry to be configured \u2014 set retry.maxRetries to enable the retry topic chain"
+      );
+    }
+    const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", options);
+    const deps = { logger: this.logger, producer: this.producer, instrumentation: this.instrumentation, onMessageLost: this.onMessageLost };
     await consumer.run({
       eachMessage: async ({ topic: topic2, partition, message }) => {
         if (!message.value) {
@@ -439,6 +522,7 @@ var KafkaClient = class {
         const raw = message.value.toString();
         const parsed = parseJsonMessage(raw, topic2, this.logger);
         if (parsed === null) return;
+        const headers = decodeHeaders(message.headers);
         const validated = await validateWithSchema(
           parsed,
           raw,
@@ -446,10 +530,9 @@ var KafkaClient = class {
           schemaMap,
           interceptors,
           dlq,
-          deps
+          { ...deps, originalHeaders: headers }
         );
         if (validated === null) return;
-        const headers = decodeHeaders(message.headers);
         const envelope = extractEnvelope(
           validated,
           headers,
@@ -462,16 +545,27 @@ var KafkaClient = class {
             { correlationId: envelope.correlationId, traceparent: envelope.traceparent },
             () => handleMessage(envelope)
           ),
-          { envelope, rawMessages: [raw], interceptors, dlq, retry },
+          { envelope, rawMessages: [raw], interceptors, dlq, retry, retryTopics: options.retryTopics },
           deps
         );
       }
     });
     this.runningConsumers.set(gid, "eachMessage");
+    if (options.retryTopics && retry) {
+      await this.startRetryTopicConsumers(
+        topicNames,
+        gid,
+        handleMessage,
+        retry,
+        dlq,
+        interceptors,
+        schemaMap
+      );
+    }
   }
   async startBatchConsumer(topics, handleBatch, options = {}) {
     const { consumer, schemaMap, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", options);
-    const deps = { logger: this.logger, producer: this.producer, instrumentation: this.instrumentation };
+    const deps = { logger: this.logger, producer: this.producer, instrumentation: this.instrumentation, onMessageLost: this.onMessageLost };
     await consumer.run({
       eachBatch: async ({
         batch,
@@ -491,6 +585,7 @@ var KafkaClient = class {
           const raw = message.value.toString();
           const parsed = parseJsonMessage(raw, batch.topic, this.logger);
           if (parsed === null) continue;
+          const headers = decodeHeaders(message.headers);
           const validated = await validateWithSchema(
             parsed,
             raw,
@@ -498,10 +593,9 @@ var KafkaClient = class {
             schemaMap,
             interceptors,
             dlq,
-            deps
+            { ...deps, originalHeaders: headers }
           );
           if (validated === null) continue;
-          const headers = decodeHeaders(message.headers);
           envelopes.push(
             extractEnvelope(validated, headers, batch.topic, batch.partition, message.offset)
           );
@@ -532,15 +626,28 @@ var KafkaClient = class {
     this.runningConsumers.set(gid, "eachBatch");
   }
   // ── Consumer lifecycle ───────────────────────────────────────────
-  async stopConsumer() {
-    const tasks = [];
-    for (const consumer of this.consumers.values()) {
-      tasks.push(consumer.disconnect());
+  async stopConsumer(groupId) {
+    if (groupId !== void 0) {
+      const consumer = this.consumers.get(groupId);
+      if (!consumer) {
+        this.logger.warn(`stopConsumer: no active consumer for group "${groupId}"`);
+        return;
+      }
+      await consumer.disconnect().catch(() => {
+      });
+      this.consumers.delete(groupId);
+      this.runningConsumers.delete(groupId);
+      this.logger.log(`Consumer disconnected: group "${groupId}"`);
+    } else {
+      const tasks = Array.from(this.consumers.values()).map(
+        (c) => c.disconnect().catch(() => {
+        })
+      );
+      await Promise.allSettled(tasks);
+      this.consumers.clear();
+      this.runningConsumers.clear();
+      this.logger.log("All consumers disconnected");
     }
-    await Promise.allSettled(tasks);
-    this.consumers.clear();
-    this.runningConsumers.clear();
-    this.logger.log("All consumers disconnected");
   }
   /** Check broker connectivity and return status, clientId, and available topics. */
   async checkStatus() {
@@ -573,7 +680,164 @@ var KafkaClient = class {
     this.runningConsumers.clear();
     this.logger.log("All connections closed");
   }
+  // ── Retry topic chain ────────────────────────────────────────────
+  /**
+   * Auto-start companion consumers on `<topic>.retry` for each original topic.
+   * Called by `startConsumer` when `retryTopics: true`.
+   *
+   * Flow per message:
+   *   1. Sleep until `x-retry-after` (scheduled by the main consumer or previous retry hop)
+   *   2. Call the original handler
+   *   3. On failure: if retries remain → re-send to `<originalTopic>.retry` with incremented attempt
+   *                  if exhausted       → DLQ or onMessageLost
+   */
+  async startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap) {
+    const retryTopicNames = originalTopics.map((t) => `${t}.retry`);
+    const retryGroupId = `${originalGroupId}-retry`;
+    const backoffMs = retry.backoffMs ?? 1e3;
+    const maxBackoffMs = retry.maxBackoffMs ?? 3e4;
+    const deps = {
+      logger: this.logger,
+      producer: this.producer,
+      instrumentation: this.instrumentation,
+      onMessageLost: this.onMessageLost
+    };
+    for (const rt of retryTopicNames) {
+      await this.ensureTopic(rt);
+    }
+    const consumer = this.getOrCreateConsumer(retryGroupId, false, true);
+    await consumer.connect();
+    await subscribeWithRetry(consumer, retryTopicNames, this.logger);
+    await consumer.run({
+      eachMessage: async ({ topic: retryTopic, partition, message }) => {
+        if (!message.value) return;
+        const raw = message.value.toString();
+        const parsed = parseJsonMessage(raw, retryTopic, this.logger);
+        if (parsed === null) return;
+        const headers = decodeHeaders(message.headers);
+        const originalTopic = headers[RETRY_HEADER_ORIGINAL_TOPIC] ?? retryTopic.replace(/\.retry$/, "");
+        const currentAttempt = parseInt(
+          headers[RETRY_HEADER_ATTEMPT] ?? "1",
+          10
+        );
+        const maxRetries = parseInt(
+          headers[RETRY_HEADER_MAX_RETRIES] ?? String(retry.maxRetries),
+          10
+        );
+        const retryAfter = parseInt(
+          headers[RETRY_HEADER_AFTER] ?? "0",
+          10
+        );
+        const remaining = retryAfter - Date.now();
+        if (remaining > 0) {
+          consumer.pause([{ topic: retryTopic, partitions: [partition] }]);
+          await sleep(remaining);
+          consumer.resume([{ topic: retryTopic, partitions: [partition] }]);
+        }
+        const validated = await validateWithSchema(
+          parsed,
+          raw,
+          originalTopic,
+          schemaMap,
+          interceptors,
+          dlq,
+          { ...deps, originalHeaders: headers }
+        );
+        if (validated === null) return;
+        const envelope = extractEnvelope(
+          validated,
+          headers,
+          originalTopic,
+          partition,
+          message.offset
+        );
+        try {
+          const cleanups = [];
+          for (const inst of this.instrumentation) {
+            const c = inst.beforeConsume?.(envelope);
+            if (typeof c === "function") cleanups.push(c);
+          }
+          for (const interceptor of interceptors) await interceptor.before?.(envelope);
+          await runWithEnvelopeContext(
+            { correlationId: envelope.correlationId, traceparent: envelope.traceparent },
+            () => handleMessage(envelope)
+          );
+          for (const interceptor of interceptors) await interceptor.after?.(envelope);
+          for (const cleanup of cleanups) cleanup();
+        } catch (error) {
+          const err = toError(error);
+          const nextAttempt = currentAttempt + 1;
+          const exhausted = currentAttempt >= maxRetries;
+          for (const inst of this.instrumentation) inst.onConsumeError?.(envelope, err);
+          const reportedError = exhausted && maxRetries > 1 ? new KafkaRetryExhaustedError(originalTopic, [envelope.payload], maxRetries, { cause: err }) : err;
+          for (const interceptor of interceptors) {
+            await interceptor.onError?.(envelope, reportedError);
+          }
+          this.logger.error(
+            `Retry consumer error for ${originalTopic} (attempt ${currentAttempt}/${maxRetries}):`,
+            err.stack
+          );
+          if (!exhausted) {
+            const cap = Math.min(backoffMs * 2 ** currentAttempt, maxBackoffMs);
+            const delay = Math.floor(Math.random() * cap);
+            await sendToRetryTopic(
+              originalTopic,
+              [raw],
+              nextAttempt,
+              maxRetries,
+              delay,
+              headers,
+              deps
+            );
+          } else if (dlq) {
+            await sendToDlq(originalTopic, raw, deps, {
+              error: err,
+              // +1 to account for the main consumer's initial attempt before
+              // routing to the retry topic, making this consistent with the
+              // in-process retry path where attempt counts all tries.
+              attempt: currentAttempt + 1,
+              originalHeaders: headers
+            });
+          } else {
+            await deps.onMessageLost?.({
+              topic: originalTopic,
+              error: err,
+              attempt: currentAttempt,
+              headers
+            });
+          }
+        }
+      }
+    });
+    this.runningConsumers.set(retryGroupId, "eachMessage");
+    await this.waitForPartitionAssignment(consumer, retryTopicNames);
+    this.logger.log(
+      `Retry topic consumers started for: ${originalTopics.join(", ")} (group: ${retryGroupId})`
+    );
+  }
   // ── Private helpers ──────────────────────────────────────────────
+  /**
+   * Poll `consumer.assignment()` until the consumer has received at least one
+   * partition for the given topics, then return. Logs a warning and returns
+   * (rather than throwing) on timeout so that a slow broker does not break
+   * the caller — in the worst case a message sent immediately after would be
+   * missed, which is the same behaviour as before this guard was added.
+   */
+  async waitForPartitionAssignment(consumer, topics, timeoutMs = 1e4) {
+    const topicSet = new Set(topics);
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+      try {
+        const assigned = consumer.assignment();
+        if (assigned.some((a) => topicSet.has(a.topic))) return;
+      } catch {
+      }
+      await sleep(200);
+    }
+    this.logger.warn(
+      `Retry consumer did not receive partition assignments for [${topics.join(", ")}] within ${timeoutMs}ms`
+    );
+  }
   getOrCreateConsumer(groupId, fromBeginning, autoCommit) {
     if (!this.consumers.has(groupId)) {
       this.consumers.set(
@@ -611,13 +875,13 @@ var KafkaClient = class {
     }
   }
   /** Validate message against schema. Pure — no side-effects on registry. */
-  validateMessage(topicOrDesc, message) {
+  async validateMessage(topicOrDesc, message) {
     if (topicOrDesc?.__schema) {
-      return topicOrDesc.__schema.parse(message);
+      return await topicOrDesc.__schema.parse(message);
     }
     if (this.strictSchemasEnabled && typeof topicOrDesc === "string") {
       const schema = this.schemaRegistry.get(topicOrDesc);
-      if (schema) return schema.parse(message);
+      if (schema) return await schema.parse(message);
     }
     return message;
   }
@@ -626,12 +890,11 @@ var KafkaClient = class {
    * Handles: topic resolution, schema registration, validation, JSON serialization,
    * envelope header generation, and instrumentation hooks.
    */
-  buildSendPayload(topicOrDesc, messages) {
+  async buildSendPayload(topicOrDesc, messages) {
     this.registerSchema(topicOrDesc);
     const topic2 = this.resolveTopicName(topicOrDesc);
-    return {
-      topic: topic2,
-      messages: messages.map((m) => {
+    const builtMessages = await Promise.all(
+      messages.map(async (m) => {
         const envelopeHeaders = buildEnvelopeHeaders({
           correlationId: m.correlationId,
           schemaVersion: m.schemaVersion,
@@ -642,12 +905,13 @@ var KafkaClient = class {
           inst.beforeSend?.(topic2, envelopeHeaders);
         }
         return {
-          value: JSON.stringify(this.validateMessage(topicOrDesc, m.value)),
+          value: JSON.stringify(await this.validateMessage(topicOrDesc, m.value)),
           key: m.key ?? null,
           headers: envelopeHeaders
         };
       })
-    };
+    );
+    return { topic: topic2, messages: builtMessages };
   }
   /** Shared consumer setup: groupId check, schema map, connect, subscribe. */
   async setupConsumer(topics, mode, options) {