@drarzter/kafka-client 0.5.2 → 0.5.5

package/dist/core.js

@@ -172,9 +172,20 @@ async function validateWithSchema(message, raw, topic2, schemaMap, interceptors,
         originalHeaders: deps.originalHeaders
       });
     } else {
-      await deps.onMessageLost?.({ topic: topic2, error: validationError, attempt: 0, headers: deps.originalHeaders ?? {} });
+      await deps.onMessageLost?.({
+        topic: topic2,
+        error: validationError,
+        attempt: 0,
+        headers: deps.originalHeaders ?? {}
+      });
     }
-    const errorEnvelope = extractEnvelope(message, deps.originalHeaders ?? {}, topic2, -1, "");
+    const errorEnvelope = extractEnvelope(
+      message,
+      deps.originalHeaders ?? {},
+      topic2,
+      -1,
+      ""
+    );
     for (const interceptor of interceptors) {
       await interceptor.onError?.(errorEnvelope, validationError);
     }
@@ -204,9 +215,54 @@ async function sendToDlq(topic2, rawMessage, deps, meta) {
     );
   }
 }
+var RETRY_HEADER_ATTEMPT = "x-retry-attempt";
+var RETRY_HEADER_AFTER = "x-retry-after";
+var RETRY_HEADER_MAX_RETRIES = "x-retry-max-retries";
+var RETRY_HEADER_ORIGINAL_TOPIC = "x-retry-original-topic";
+async function sendToRetryTopic(originalTopic, rawMessages, attempt, maxRetries, delayMs, originalHeaders, deps) {
+  const retryTopic = `${originalTopic}.retry`;
+  const {
+    [RETRY_HEADER_ATTEMPT]: _a,
+    [RETRY_HEADER_AFTER]: _b,
+    [RETRY_HEADER_MAX_RETRIES]: _c,
+    [RETRY_HEADER_ORIGINAL_TOPIC]: _d,
+    ...userHeaders
+  } = originalHeaders;
+  const headers = {
+    ...userHeaders,
+    [RETRY_HEADER_ATTEMPT]: String(attempt),
+    [RETRY_HEADER_AFTER]: String(Date.now() + delayMs),
+    [RETRY_HEADER_MAX_RETRIES]: String(maxRetries),
+    [RETRY_HEADER_ORIGINAL_TOPIC]: originalTopic
+  };
+  try {
+    for (const raw of rawMessages) {
+      await deps.producer.send({
+        topic: retryTopic,
+        messages: [{ value: raw, headers }]
+      });
+    }
+    deps.logger.warn(
+      `Message queued in retry topic ${retryTopic} (attempt ${attempt}/${maxRetries})`
+    );
+  } catch (error) {
+    deps.logger.error(
+      `Failed to send message to retry topic ${retryTopic}:`,
+      toError(error).stack
+    );
+  }
+}
 async function executeWithRetry(fn, ctx, deps) {
-  const { envelope, rawMessages, interceptors, dlq, retry, isBatch } = ctx;
-  const maxAttempts = retry ? retry.maxRetries + 1 : 1;
+  const {
+    envelope,
+    rawMessages,
+    interceptors,
+    dlq,
+    retry,
+    isBatch,
+    retryTopics
+  } = ctx;
+  const maxAttempts = retryTopics ? 1 : retry ? retry.maxRetries + 1 : 1;
   const backoffMs = retry?.backoffMs ?? 1e3;
   const maxBackoffMs = retry?.maxBackoffMs ?? 3e4;
   const envelopes = Array.isArray(envelope) ? envelope : [envelope];
@@ -265,7 +321,19 @@ async function executeWithRetry(fn, ctx, deps) {
         `Error processing ${isBatch ? "batch" : "message"} from topic ${topic2} (attempt ${attempt}/${maxAttempts}):`,
         err.stack
       );
-      if (isLastAttempt) {
+      if (retryTopics && retry) {
+        const cap = Math.min(backoffMs, maxBackoffMs);
+        const delay = Math.floor(Math.random() * cap);
+        await sendToRetryTopic(
+          topic2,
+          rawMessages,
+          1,
+          retry.maxRetries,
+          delay,
+          envelopes[0]?.headers ?? {},
+          deps
+        );
+      } else if (isLastAttempt) {
         if (dlq) {
           const dlqMeta = {
             error: err,
@@ -328,6 +396,7 @@ var KafkaClient = class {
   runningConsumers = /* @__PURE__ */ new Map();
   instrumentation;
   onMessageLost;
+  onRebalance;
   isAdminConnected = false;
   clientId;
   constructor(clientId, groupId, brokers, options) {
@@ -343,6 +412,7 @@ var KafkaClient = class {
     this.numPartitions = options?.numPartitions ?? 1;
     this.instrumentation = options?.instrumentation ?? [];
     this.onMessageLost = options?.onMessageLost;
+    this.onRebalance = options?.onRebalance;
     this.kafka = new KafkaClass({
       kafkaJS: {
         clientId: this.clientId,
@@ -443,8 +513,19 @@ var KafkaClient = class {
     this.logger.log("Producer disconnected");
   }
   async startConsumer(topics, handleMessage, options = {}) {
-    const { consumer, schemaMap, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", options);
-    const deps = { logger: this.logger, producer: this.producer, instrumentation: this.instrumentation, onMessageLost: this.onMessageLost };
+    if (options.retryTopics && !options.retry) {
+      throw new Error(
+        "retryTopics requires retry to be configured \u2014 set retry.maxRetries to enable the retry topic chain"
+      );
+    }
+    const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", options);
+    const deps = {
+      logger: this.logger,
+      producer: this.producer,
+      instrumentation: this.instrumentation,
+      onMessageLost: this.onMessageLost
+    };
+    const timeoutMs = options.handlerTimeoutMs;
     await consumer.run({
       eachMessage: async ({ topic: topic2, partition, message }) => {
         if (!message.value) {
@@ -473,20 +554,51 @@ var KafkaClient = class {
           message.offset
         );
         await executeWithRetry(
-          () => runWithEnvelopeContext(
-            { correlationId: envelope.correlationId, traceparent: envelope.traceparent },
-            () => handleMessage(envelope)
-          ),
-          { envelope, rawMessages: [raw], interceptors, dlq, retry },
+          () => {
+            const fn = () => runWithEnvelopeContext(
+              {
+                correlationId: envelope.correlationId,
+                traceparent: envelope.traceparent
+              },
+              () => handleMessage(envelope)
+            );
+            return timeoutMs ? this.wrapWithTimeoutWarning(fn, timeoutMs, topic2) : fn();
+          },
+          {
+            envelope,
+            rawMessages: [raw],
+            interceptors,
+            dlq,
+            retry,
+            retryTopics: options.retryTopics
+          },
           deps
         );
       }
     });
     this.runningConsumers.set(gid, "eachMessage");
+    if (options.retryTopics && retry) {
+      await this.startRetryTopicConsumers(
+        topicNames,
+        gid,
+        handleMessage,
+        retry,
+        dlq,
+        interceptors,
+        schemaMap
+      );
+    }
+    return { groupId: gid, stop: () => this.stopConsumer(gid) };
   }
   async startBatchConsumer(topics, handleBatch, options = {}) {
     const { consumer, schemaMap, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", options);
-    const deps = { logger: this.logger, producer: this.producer, instrumentation: this.instrumentation, onMessageLost: this.onMessageLost };
+    const deps = {
+      logger: this.logger,
+      producer: this.producer,
+      instrumentation: this.instrumentation,
+      onMessageLost: this.onMessageLost
+    };
+    const timeoutMs = options.handlerTimeoutMs;
     await consumer.run({
       eachBatch: async ({
         batch,
@@ -518,7 +630,13 @@ var KafkaClient = class {
           );
           if (validated === null) continue;
           envelopes.push(
-            extractEnvelope(validated, headers, batch.topic, batch.partition, message.offset)
+            extractEnvelope(
+              validated,
+              headers,
+              batch.topic,
+              batch.partition,
+              message.offset
+            )
           );
           rawMessages.push(raw);
         }
@@ -531,7 +649,10 @@ var KafkaClient = class {
           commitOffsetsIfNecessary
         };
         await executeWithRetry(
-          () => handleBatch(envelopes, meta),
+          () => {
+            const fn = () => handleBatch(envelopes, meta);
+            return timeoutMs ? this.wrapWithTimeoutWarning(fn, timeoutMs, batch.topic) : fn();
+          },
           {
             envelope: envelopes,
             rawMessages: batch.messages.filter((m) => m.value).map((m) => m.value.toString()),
@@ -545,17 +666,59 @@ var KafkaClient = class {
       }
     });
     this.runningConsumers.set(gid, "eachBatch");
+    return { groupId: gid, stop: () => this.stopConsumer(gid) };
   }
   // ── Consumer lifecycle ───────────────────────────────────────────
-  async stopConsumer() {
-    const tasks = [];
-    for (const consumer of this.consumers.values()) {
-      tasks.push(consumer.disconnect());
+  async stopConsumer(groupId) {
+    if (groupId !== void 0) {
+      const consumer = this.consumers.get(groupId);
+      if (!consumer) {
+        this.logger.warn(
+          `stopConsumer: no active consumer for group "${groupId}"`
+        );
+        return;
+      }
+      await consumer.disconnect().catch(() => {
+      });
+      this.consumers.delete(groupId);
+      this.runningConsumers.delete(groupId);
+      this.logger.log(`Consumer disconnected: group "${groupId}"`);
+    } else {
+      const tasks = Array.from(this.consumers.values()).map(
+        (c) => c.disconnect().catch(() => {
+        })
+      );
+      await Promise.allSettled(tasks);
+      this.consumers.clear();
+      this.runningConsumers.clear();
+      this.logger.log("All consumers disconnected");
     }
-    await Promise.allSettled(tasks);
-    this.consumers.clear();
-    this.runningConsumers.clear();
-    this.logger.log("All consumers disconnected");
+  }
+  /**
+   * Query consumer group lag per partition.
+   * Lag = broker high-watermark − last committed offset.
+   * A committed offset of -1 (nothing committed yet) counts as full lag.
+   */
+  async getConsumerLag(groupId) {
+    const gid = groupId ?? this.defaultGroupId;
+    if (!this.isAdminConnected) {
+      await this.admin.connect();
+      this.isAdminConnected = true;
+    }
+    const committedByTopic = await this.admin.fetchOffsets({ groupId: gid });
+    const result = [];
+    for (const { topic: topic2, partitions } of committedByTopic) {
+      const brokerOffsets = await this.admin.fetchTopicOffsets(topic2);
+      for (const { partition, offset } of partitions) {
+        const broker = brokerOffsets.find((o) => o.partition === partition);
+        if (!broker) continue;
+        const committed = parseInt(offset, 10);
+        const high = parseInt(broker.high, 10);
+        const lag = committed === -1 ? high : Math.max(0, high - committed);
+        result.push({ topic: topic2, partition, lag });
+      }
+    }
+    return result;
   }
   /** Check broker connectivity and return status, clientId, and available topics. */
   async checkStatus() {
@@ -588,18 +751,219 @@ var KafkaClient = class {
     this.runningConsumers.clear();
     this.logger.log("All connections closed");
   }
+  // ── Retry topic chain ────────────────────────────────────────────
+  /**
+   * Auto-start companion consumers on `<topic>.retry` for each original topic.
+   * Called by `startConsumer` when `retryTopics: true`.
+   *
+   * Flow per message:
+   *   1. Sleep until `x-retry-after` (scheduled by the main consumer or previous retry hop)
+   *   2. Call the original handler
+   *   3. On failure: if retries remain → re-send to `<originalTopic>.retry` with incremented attempt
+   *                  if exhausted       → DLQ or onMessageLost
+   */
+  async startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap) {
+    const retryTopicNames = originalTopics.map((t) => `${t}.retry`);
+    const retryGroupId = `${originalGroupId}-retry`;
+    const backoffMs = retry.backoffMs ?? 1e3;
+    const maxBackoffMs = retry.maxBackoffMs ?? 3e4;
+    const deps = {
+      logger: this.logger,
+      producer: this.producer,
+      instrumentation: this.instrumentation,
+      onMessageLost: this.onMessageLost
+    };
+    for (const rt of retryTopicNames) {
+      await this.ensureTopic(rt);
+    }
+    const consumer = this.getOrCreateConsumer(retryGroupId, false, true);
+    await consumer.connect();
+    await subscribeWithRetry(consumer, retryTopicNames, this.logger);
+    await consumer.run({
+      eachMessage: async ({ topic: retryTopic, partition, message }) => {
+        if (!message.value) return;
+        const raw = message.value.toString();
+        const parsed = parseJsonMessage(raw, retryTopic, this.logger);
+        if (parsed === null) return;
+        const headers = decodeHeaders(message.headers);
+        const originalTopic = headers[RETRY_HEADER_ORIGINAL_TOPIC] ?? retryTopic.replace(/\.retry$/, "");
+        const currentAttempt = parseInt(
+          headers[RETRY_HEADER_ATTEMPT] ?? "1",
+          10
+        );
+        const maxRetries = parseInt(
+          headers[RETRY_HEADER_MAX_RETRIES] ?? String(retry.maxRetries),
+          10
+        );
+        const retryAfter = parseInt(
+          headers[RETRY_HEADER_AFTER] ?? "0",
+          10
+        );
+        const remaining = retryAfter - Date.now();
+        if (remaining > 0) {
+          consumer.pause([{ topic: retryTopic, partitions: [partition] }]);
+          await sleep(remaining);
+          consumer.resume([{ topic: retryTopic, partitions: [partition] }]);
+        }
+        const validated = await validateWithSchema(
+          parsed,
+          raw,
+          originalTopic,
+          schemaMap,
+          interceptors,
+          dlq,
+          { ...deps, originalHeaders: headers }
+        );
+        if (validated === null) return;
+        const envelope = extractEnvelope(
+          validated,
+          headers,
+          originalTopic,
+          partition,
+          message.offset
+        );
+        try {
+          const cleanups = [];
+          for (const inst of this.instrumentation) {
+            const c = inst.beforeConsume?.(envelope);
+            if (typeof c === "function") cleanups.push(c);
+          }
+          for (const interceptor of interceptors)
+            await interceptor.before?.(envelope);
+          await runWithEnvelopeContext(
+            {
+              correlationId: envelope.correlationId,
+              traceparent: envelope.traceparent
+            },
+            () => handleMessage(envelope)
+          );
+          for (const interceptor of interceptors)
+            await interceptor.after?.(envelope);
+          for (const cleanup of cleanups) cleanup();
+        } catch (error) {
+          const err = toError(error);
+          const nextAttempt = currentAttempt + 1;
+          const exhausted = currentAttempt >= maxRetries;
+          for (const inst of this.instrumentation)
+            inst.onConsumeError?.(envelope, err);
+          const reportedError = exhausted && maxRetries > 1 ? new KafkaRetryExhaustedError(
+            originalTopic,
+            [envelope.payload],
+            maxRetries,
+            { cause: err }
+          ) : err;
+          for (const interceptor of interceptors) {
+            await interceptor.onError?.(envelope, reportedError);
+          }
+          this.logger.error(
+            `Retry consumer error for ${originalTopic} (attempt ${currentAttempt}/${maxRetries}):`,
+            err.stack
+          );
+          if (!exhausted) {
+            const cap = Math.min(backoffMs * 2 ** currentAttempt, maxBackoffMs);
+            const delay = Math.floor(Math.random() * cap);
+            await sendToRetryTopic(
+              originalTopic,
+              [raw],
+              nextAttempt,
+              maxRetries,
+              delay,
+              headers,
+              deps
+            );
+          } else if (dlq) {
+            await sendToDlq(originalTopic, raw, deps, {
+              error: err,
+              // +1 to account for the main consumer's initial attempt before
+              // routing to the retry topic, making this consistent with the
+              // in-process retry path where attempt counts all tries.
+              attempt: currentAttempt + 1,
+              originalHeaders: headers
+            });
+          } else {
+            await deps.onMessageLost?.({
+              topic: originalTopic,
+              error: err,
+              attempt: currentAttempt,
+              headers
+            });
+          }
+        }
+      }
+    });
+    this.runningConsumers.set(retryGroupId, "eachMessage");
+    await this.waitForPartitionAssignment(consumer, retryTopicNames);
+    this.logger.log(
+      `Retry topic consumers started for: ${originalTopics.join(", ")} (group: ${retryGroupId})`
+    );
+  }
   // ── Private helpers ──────────────────────────────────────────────
+  /**
+   * Poll `consumer.assignment()` until the consumer has received at least one
+   * partition for the given topics, then return. Logs a warning and returns
+   * (rather than throwing) on timeout so that a slow broker does not break
+   * the caller — in the worst case a message sent immediately after would be
+   * missed, which is the same behaviour as before this guard was added.
+   */
+  async waitForPartitionAssignment(consumer, topics, timeoutMs = 1e4) {
+    const topicSet = new Set(topics);
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+      try {
+        const assigned = consumer.assignment();
+        if (assigned.some((a) => topicSet.has(a.topic))) return;
+      } catch {
+      }
+      await sleep(200);
+    }
+    this.logger.warn(
+      `Retry consumer did not receive partition assignments for [${topics.join(", ")}] within ${timeoutMs}ms`
+    );
+  }
   getOrCreateConsumer(groupId, fromBeginning, autoCommit) {
     if (!this.consumers.has(groupId)) {
-      this.consumers.set(
-        groupId,
-        this.kafka.consumer({
-          kafkaJS: { groupId, fromBeginning, autoCommit }
-        })
-      );
+      const config = {
+        kafkaJS: { groupId, fromBeginning, autoCommit }
+      };
+      if (this.onRebalance) {
+        const onRebalance = this.onRebalance;
+        config["rebalance_cb"] = (err, assignment) => {
+          const type = err.code === -175 ? "assign" : "revoke";
+          try {
+            onRebalance(
+              type,
+              assignment.map((p) => ({
+                topic: p.topic,
+                partition: p.partition
+              }))
+            );
+          } catch (e) {
+            this.logger.warn(
+              `onRebalance callback threw: ${e.message}`
+            );
+          }
+        };
+      }
+      this.consumers.set(groupId, this.kafka.consumer(config));
     }
     return this.consumers.get(groupId);
   }
+  /**
+   * Start a timer that logs a warning if `fn` hasn't resolved within `timeoutMs`.
+   * The handler itself is not cancelled — the warning is diagnostic only.
+   */
+  wrapWithTimeoutWarning(fn, timeoutMs, topic2) {
+    let timer;
+    const promise = fn().finally(() => {
+      if (timer !== void 0) clearTimeout(timer);
+    });
+    timer = setTimeout(() => {
+      this.logger.warn(
+        `Handler for topic "${topic2}" has not resolved after ${timeoutMs}ms \u2014 possible stuck handler`
+      );
+    }, timeoutMs);
+    return promise;
+  }
   resolveTopicName(topicOrDescriptor) {
     if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
     if (topicOrDescriptor && typeof topicOrDescriptor === "object" && "__topic" in topicOrDescriptor) {
@@ -656,7 +1020,9 @@ var KafkaClient = class {
           inst.beforeSend?.(topic2, envelopeHeaders);
         }
         return {
-          value: JSON.stringify(await this.validateMessage(topicOrDesc, m.value)),
+          value: JSON.stringify(
+            await this.validateMessage(topicOrDesc, m.value)
+          ),
           key: m.key ?? null,
           headers: envelopeHeaders
         };
@@ -682,7 +1048,11 @@ var KafkaClient = class {
         `Cannot use ${mode} on consumer group "${gid}" \u2014 it is already running with ${oppositeMode}. Use a different groupId for this consumer.`
       );
     }
-    const consumer = this.getOrCreateConsumer(gid, fromBeginning, options.autoCommit ?? true);
+    const consumer = this.getOrCreateConsumer(
+      gid,
+      fromBeginning,
+      options.autoCommit ?? true
+    );
     const schemaMap = this.buildSchemaMap(topics, optionSchemas);
     const topicNames = topics.map(
       (t) => this.resolveTopicName(t)
@@ -696,7 +1066,12 @@ var KafkaClient = class {
       }
     }
     await consumer.connect();
-    await subscribeWithRetry(consumer, topicNames, this.logger, options.subscribeRetry);
+    await subscribeWithRetry(
+      consumer,
+      topicNames,
+      this.logger,
+      options.subscribeRetry
+    );
     this.logger.log(
       `${mode === "eachBatch" ? "Batch consumer" : "Consumer"} subscribed to topics: ${topicNames.join(", ")}`
     );