@drarzter/kafka-client 0.5.7 → 0.6.4

package/dist/index.js

@@ -96,7 +96,8 @@ function decodeHeaders(raw) {
   for (const [key, value] of Object.entries(raw)) {
     if (value === void 0) continue;
     if (Array.isArray(value)) {
-      result[key] = value.map((v) => Buffer.isBuffer(v) ? v.toString() : v).join(",");
+      const items = value.map((v) => Buffer.isBuffer(v) ? v.toString() : v);
+      result[key] = items[items.length - 1] ?? "";
     } else {
       result[key] = Buffer.isBuffer(value) ? value.toString() : value;
     }
@@ -158,17 +159,23 @@ function resolveTopicName(topicOrDescriptor) {
   }
   return String(topicOrDescriptor);
 }
-function registerSchema(topicOrDesc, schemaRegistry) {
+function registerSchema(topicOrDesc, schemaRegistry, logger) {
   if (topicOrDesc?.__schema) {
     const topic2 = resolveTopicName(topicOrDesc);
+    const existing = schemaRegistry.get(topic2);
+    if (existing && existing !== topicOrDesc.__schema) {
+      logger?.warn(
+        `Schema conflict for topic "${topic2}": a different schema is already registered. Using the new schema \u2014 ensure consistent schemas to avoid silent validation mismatches.`
+      );
+    }
     schemaRegistry.set(topic2, topicOrDesc.__schema);
   }
 }
-async function validateMessage(topicOrDesc, message, deps) {
+async function validateMessage(topicOrDesc, message, deps, ctx) {
   const topicName = resolveTopicName(topicOrDesc);
   if (topicOrDesc?.__schema) {
     try {
-      return await topicOrDesc.__schema.parse(message);
+      return await topicOrDesc.__schema.parse(message, ctx);
     } catch (error) {
       throw new KafkaValidationError(topicName, message, {
         cause: error instanceof Error ? error : new Error(String(error))
@@ -179,7 +186,7 @@ async function validateMessage(topicOrDesc, message, deps) {
     const schema = deps.schemaRegistry.get(topicOrDesc);
     if (schema) {
       try {
-        return await schema.parse(message);
+        return await schema.parse(message, ctx);
       } catch (error) {
         throw new KafkaValidationError(topicName, message, {
           cause: error instanceof Error ? error : new Error(String(error))
@@ -202,9 +209,14 @@ async function buildSendPayload(topicOrDesc, messages, deps) {
       for (const inst of deps.instrumentation) {
         inst.beforeSend?.(topic2, envelopeHeaders);
       }
+      const sendCtx = {
+        topic: topic2,
+        headers: envelopeHeaders,
+        version: m.schemaVersion ?? 1
+      };
       return {
         value: JSON.stringify(
-          await validateMessage(topicOrDesc, m.value, deps)
+          await validateMessage(topicOrDesc, m.value, deps, sendCtx)
         ),
         key: m.key ?? null,
         headers: envelopeHeaders
@@ -248,19 +260,26 @@ function getOrCreateConsumer(groupId, fromBeginning, autoCommit, deps) {
   consumers.set(groupId, consumer);
   return consumer;
 }
-function buildSchemaMap(topics, schemaRegistry, optionSchemas) {
+function buildSchemaMap(topics, schemaRegistry, optionSchemas, logger) {
   const schemaMap = /* @__PURE__ */ new Map();
+  const registerChecked = (name, schema) => {
+    const existing = schemaRegistry.get(name);
+    if (existing && existing !== schema) {
+      logger?.warn(
+        `Schema conflict for topic "${name}": a different schema is already registered. Using the new schema \u2014 ensure consistent schemas to avoid silent validation mismatches.`
+      );
+    }
+    schemaMap.set(name, schema);
+    schemaRegistry.set(name, schema);
+  };
   for (const t of topics) {
     if (t?.__schema) {
-      const name = resolveTopicName(t);
-      schemaMap.set(name, t.__schema);
-      schemaRegistry.set(name, t.__schema);
+      registerChecked(resolveTopicName(t), t.__schema);
     }
   }
   if (optionSchemas) {
     for (const [k, v] of optionSchemas) {
-      schemaMap.set(k, v);
-      schemaRegistry.set(k, v);
+      registerChecked(k, v);
     }
   }
   return schemaMap;
@@ -287,8 +306,13 @@ function parseJsonMessage(raw, topic2, logger) {
 async function validateWithSchema(message, raw, topic2, schemaMap, interceptors, dlq, deps) {
   const schema = schemaMap.get(topic2);
   if (!schema) return message;
+  const ctx = {
+    topic: topic2,
+    headers: deps.originalHeaders ?? {},
+    version: Number(deps.originalHeaders?.["x-schema-version"] ?? 1)
+  };
   try {
-    return await schema.parse(message);
+    return await schema.parse(message, ctx);
   } catch (error) {
     const err = toError(error);
     const validationError = new KafkaValidationError(topic2, message, {
@@ -325,7 +349,7 @@ async function validateWithSchema(message, raw, topic2, schemaMap, interceptors,
     return null;
   }
 }
-async function sendToDlq(topic2, rawMessage, deps, meta) {
+function buildDlqPayload(topic2, rawMessage, meta) {
   const dlqTopic = `${topic2}.dlq`;
   const headers = {
     ...meta?.originalHeaders ?? {},
@@ -335,54 +359,82 @@ async function sendToDlq(topic2, rawMessage, deps, meta) {
     "x-dlq-error-stack": meta?.error.stack?.slice(0, 2e3) ?? "",
     "x-dlq-attempt-count": String(meta?.attempt ?? 0)
   };
+  return { topic: dlqTopic, messages: [{ value: rawMessage, headers }] };
+}
+async function sendToDlq(topic2, rawMessage, deps, meta) {
+  const payload = buildDlqPayload(topic2, rawMessage, meta);
   try {
-    await deps.producer.send({
-      topic: dlqTopic,
-      messages: [{ value: rawMessage, headers }]
-    });
-    deps.logger.warn(`Message sent to DLQ: ${dlqTopic}`);
+    await deps.producer.send(payload);
+    deps.logger.warn(`Message sent to DLQ: ${payload.topic}`);
   } catch (error) {
-    deps.logger.error(
-      `Failed to send message to DLQ ${dlqTopic}:`,
-      toError(error).stack
-    );
+    const err = toError(error);
+    deps.logger.error(`Failed to send message to DLQ ${payload.topic}:`, err.stack);
+    await deps.onMessageLost?.({
+      topic: topic2,
+      error: err,
+      attempt: meta?.attempt ?? 0,
+      headers: meta?.originalHeaders ?? {}
+    });
   }
 }
 var RETRY_HEADER_ATTEMPT = "x-retry-attempt";
 var RETRY_HEADER_AFTER = "x-retry-after";
 var RETRY_HEADER_MAX_RETRIES = "x-retry-max-retries";
 var RETRY_HEADER_ORIGINAL_TOPIC = "x-retry-original-topic";
-async function sendToRetryTopic(originalTopic, rawMessages, attempt, maxRetries, delayMs, originalHeaders, deps) {
+function buildRetryTopicPayload(originalTopic, rawMessages, attempt, maxRetries, delayMs, originalHeaders) {
   const retryTopic = `${originalTopic}.retry.${attempt}`;
-  const {
-    [RETRY_HEADER_ATTEMPT]: _a,
-    [RETRY_HEADER_AFTER]: _b,
-    [RETRY_HEADER_MAX_RETRIES]: _c,
-    [RETRY_HEADER_ORIGINAL_TOPIC]: _d,
-    ...userHeaders
-  } = originalHeaders;
-  const headers = {
-    ...userHeaders,
-    [RETRY_HEADER_ATTEMPT]: String(attempt),
-    [RETRY_HEADER_AFTER]: String(Date.now() + delayMs),
-    [RETRY_HEADER_MAX_RETRIES]: String(maxRetries),
-    [RETRY_HEADER_ORIGINAL_TOPIC]: originalTopic
+  function buildHeaders(hdr) {
+    const {
+      [RETRY_HEADER_ATTEMPT]: _a,
+      [RETRY_HEADER_AFTER]: _b,
+      [RETRY_HEADER_MAX_RETRIES]: _c,
+      [RETRY_HEADER_ORIGINAL_TOPIC]: _d,
+      ...userHeaders
+    } = hdr;
+    return {
+      ...userHeaders,
+      [RETRY_HEADER_ATTEMPT]: String(attempt),
+      [RETRY_HEADER_AFTER]: String(Date.now() + delayMs),
+      [RETRY_HEADER_MAX_RETRIES]: String(maxRetries),
+      [RETRY_HEADER_ORIGINAL_TOPIC]: originalTopic
+    };
+  }
+  return {
+    topic: retryTopic,
+    messages: rawMessages.map((value, i) => ({
+      value,
+      headers: buildHeaders(
+        Array.isArray(originalHeaders) ? originalHeaders[i] ?? {} : originalHeaders
+      )
+    }))
   };
+}
+async function sendToRetryTopic(originalTopic, rawMessages, attempt, maxRetries, delayMs, originalHeaders, deps) {
+  const payload = buildRetryTopicPayload(
+    originalTopic,
+    rawMessages,
+    attempt,
+    maxRetries,
+    delayMs,
+    originalHeaders
+  );
   try {
-    for (const raw of rawMessages) {
-      await deps.producer.send({
-        topic: retryTopic,
-        messages: [{ value: raw, headers }]
-      });
-    }
+    await deps.producer.send(payload);
     deps.logger.warn(
-      `Message queued in retry topic ${retryTopic} (attempt ${attempt}/${maxRetries})`
+      `Message queued in retry topic ${payload.topic} (attempt ${attempt}/${maxRetries})`
     );
   } catch (error) {
+    const err = toError(error);
     deps.logger.error(
-      `Failed to send message to retry topic ${retryTopic}:`,
-      toError(error).stack
+      `Failed to send message to retry topic ${payload.topic}:`,
+      err.stack
     );
+    await deps.onMessageLost?.({
+      topic: originalTopic,
+      error: err,
+      attempt,
+      headers: Array.isArray(originalHeaders) ? originalHeaders[0] ?? {} : originalHeaders
+    });
   }
 }
 async function broadcastToInterceptors(envelopes, interceptors, cb) {
@@ -394,11 +446,17 @@ async function broadcastToInterceptors(envelopes, interceptors, cb) {
 }
 async function runHandlerWithPipeline(fn, envelopes, interceptors, instrumentation) {
   const cleanups = [];
+  const wraps = [];
   try {
     for (const env of envelopes) {
       for (const inst of instrumentation) {
-        const cleanup = inst.beforeConsume?.(env);
-        if (typeof cleanup === "function") cleanups.push(cleanup);
+        const result = inst.beforeConsume?.(env);
+        if (typeof result === "function") {
+          cleanups.push(result);
+        } else if (result) {
+          if (result.cleanup) cleanups.push(result.cleanup);
+          if (result.wrap) wraps.push(result.wrap);
+        }
       }
     }
     for (const env of envelopes) {
@@ -406,7 +464,13 @@ async function runHandlerWithPipeline(fn, envelopes, interceptors, instrumentati
         await interceptor.before?.(env);
       }
     }
-    await fn();
+    let runFn = fn;
+    for (let i = wraps.length - 1; i >= 0; i--) {
+      const wrap = wraps[i];
+      const inner = runFn;
+      runFn = () => wrap(inner);
+    }
+    await runFn();
     for (const env of envelopes) {
       for (const interceptor of interceptors) {
         await interceptor.after?.(env);
@@ -476,7 +540,7 @@ async function executeWithRetry(fn, ctx, deps) {
         1,
         retry.maxRetries,
         delay,
-        envelopes[0]?.headers ?? {},
+        isBatch ? envelopes.map((e) => e.headers) : envelopes[0]?.headers ?? {},
         deps
       );
     } else if (isLastAttempt) {
@@ -499,7 +563,7 @@ async function executeWithRetry(fn, ctx, deps) {
       }
     } else {
       const cap = Math.min(backoffMs * 2 ** (attempt - 1), maxBackoffMs);
-      await sleep(Math.random() * cap);
+      await sleep(Math.floor(Math.random() * cap));
     }
   }
 }
@@ -578,6 +642,7 @@ async function handleEachBatch(payload, opts, deps) {
     interceptors,
     dlq,
     retry,
+    retryTopics,
     timeoutMs,
     wrapWithTimeout
   } = opts;
@@ -612,11 +677,12 @@ async function handleEachBatch(payload, opts, deps) {
     },
     {
       envelope: envelopes,
-      rawMessages: batch.messages.filter((m) => m.value).map((m) => m.value.toString()),
+      rawMessages,
       interceptors,
       dlq,
       retry,
-      isBatch: true
+      isBatch: true,
+      retryTopics
     },
     deps
   );
@@ -633,10 +699,11 @@ async function subscribeWithRetry(consumer, topics, logger, retryOpts) {
     } catch (error) {
       if (attempt === maxAttempts) throw error;
       const msg = toError(error).message;
+      const delay = Math.floor(Math.random() * backoffMs);
       logger.warn(
-        `Failed to subscribe to [${topics.join(", ")}] (attempt ${attempt}/${maxAttempts}): ${msg}. Retrying in ${backoffMs}ms...`
+        `Failed to subscribe to [${topics.join(", ")}] (attempt ${attempt}/${maxAttempts}): ${msg}. Retrying in ${delay}ms...`
       );
-      await sleep(backoffMs);
+      await sleep(delay);
     }
   }
 }
@@ -665,7 +732,8 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
     onMessageLost,
     ensureTopic,
     getOrCreateConsumer: getOrCreateConsumer2,
-    runningConsumers
+    runningConsumers,
+    createRetryTxProducer
   } = deps;
   const backoffMs = retry.backoffMs ?? 1e3;
   const maxBackoffMs = retry.maxBackoffMs ?? 3e4;
@@ -673,6 +741,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
   for (const lt of levelTopics) {
     await ensureTopic(lt);
   }
+  const levelTxProducer = await createRetryTxProducer(`${levelGroupId}-tx`);
   const consumer = getOrCreateConsumer2(levelGroupId, false, false);
   await consumer.connect();
   await subscribeWithRetry(consumer, levelTopics, logger);
@@ -761,22 +830,67 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
         const nextLevel = level + 1;
         const cap = Math.min(backoffMs * 2 ** level, maxBackoffMs);
         const delay = Math.floor(Math.random() * cap);
-        await sendToRetryTopic(
+        const { topic: rtTopic, messages: rtMsgs } = buildRetryTopicPayload(
           originalTopic,
           [raw],
           nextLevel,
           currentMaxRetries,
           delay,
-          headers,
-          pipelineDeps
+          headers
         );
+        const tx = await levelTxProducer.transaction();
+        try {
+          await tx.send({ topic: rtTopic, messages: rtMsgs });
+          await tx.sendOffsets({
+            consumer,
+            topics: [{ topic: nextOffset.topic, partitions: [{ partition: nextOffset.partition, offset: nextOffset.offset }] }]
+          });
+          await tx.commit();
+          logger.warn(
+            `Message routed to ${rtTopic} (EOS, level ${nextLevel}/${currentMaxRetries})`
+          );
+        } catch (txErr) {
+          try {
+            await tx.abort();
+          } catch {
+          }
+          logger.error(
+            `EOS routing to ${rtTopic} failed \u2014 message will be redelivered:`,
+            toError(txErr).stack
+          );
+          return;
+        }
       } else if (dlq) {
-        await sendToDlq(originalTopic, raw, pipelineDeps, {
-          error,
-          // +1 to account for the main consumer's initial attempt before routing.
-          attempt: level + 1,
-          originalHeaders: headers
-        });
+        const { topic: dTopic, messages: dMsgs } = buildDlqPayload(
+          originalTopic,
+          raw,
+          {
+            error,
+            // +1 to account for the main consumer's initial attempt before routing.
+            attempt: level + 1,
+            originalHeaders: headers
+          }
+        );
+        const tx = await levelTxProducer.transaction();
+        try {
+          await tx.send({ topic: dTopic, messages: dMsgs });
+          await tx.sendOffsets({
+            consumer,
+            topics: [{ topic: nextOffset.topic, partitions: [{ partition: nextOffset.partition, offset: nextOffset.offset }] }]
+          });
+          await tx.commit();
+          logger.warn(`Message sent to DLQ: ${dTopic} (EOS)`);
+        } catch (txErr) {
+          try {
+            await tx.abort();
+          } catch {
+          }
+          logger.error(
+            `EOS DLQ routing to ${dTopic} failed \u2014 message will be redelivered:`,
+            toError(txErr).stack
+          );
+          return;
+        }
       } else {
         await onMessageLost?.({
           topic: originalTopic,
@@ -784,8 +898,8 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
           attempt: level,
           headers
         });
+        await consumer.commitOffsets([nextOffset]);
       }
-      await consumer.commitOffsets([nextOffset]);
     }
   });
   runningConsumers.set(levelGroupId, "eachMessage");
@@ -823,6 +937,8 @@ var KafkaClient = class {
   kafka;
   producer;
   txProducer;
+  /** Maps transactionalId → Producer for each active retry level consumer. */
+  retryTxProducers = /* @__PURE__ */ new Map();
   consumers = /* @__PURE__ */ new Map();
   admin;
   logger;
@@ -840,6 +956,8 @@ var KafkaClient = class {
   onMessageLost;
   onRebalance;
   isAdminConnected = false;
+  inFlightTotal = 0;
+  drainResolvers = [];
   clientId;
   constructor(clientId, groupId, brokers, options) {
     this.clientId = clientId;
@@ -847,7 +965,8 @@ var KafkaClient = class {
     this.logger = options?.logger ?? {
       log: (msg) => console.log(`[KafkaClient:${clientId}] ${msg}`),
       warn: (msg, ...args) => console.warn(`[KafkaClient:${clientId}] ${msg}`, ...args),
-      error: (msg, ...args) => console.error(`[KafkaClient:${clientId}] ${msg}`, ...args)
+      error: (msg, ...args) => console.error(`[KafkaClient:${clientId}] ${msg}`, ...args),
+      debug: (msg, ...args) => console.debug(`[KafkaClient:${clientId}] ${msg}`, ...args)
     };
     this.autoCreateTopicsEnabled = options?.autoCreateTopics ?? false;
     this.strictSchemasEnabled = options?.strictSchemas ?? true;
@@ -891,7 +1010,7 @@ var KafkaClient = class {
   /** Execute multiple sends atomically. Commits on success, aborts on error. */
   async transaction(fn) {
     if (!this.txProducer) {
-      this.txProducer = this.kafka.producer({
+      const p = this.kafka.producer({
         kafkaJS: {
           acks: -1,
           idempotent: true,
@@ -899,7 +1018,8 @@ var KafkaClient = class {
           maxInFlightRequests: 1
         }
       });
-      await this.txProducer.connect();
+      await p.connect();
+      this.txProducer = p;
     }
     const tx = await this.txProducer.transaction();
     try {
@@ -916,9 +1036,12 @@ var KafkaClient = class {
             }
           ]);
           await tx.send(payload);
+          this.notifyAfterSend(payload.topic, payload.messages.length);
         },
         sendBatch: async (topicOrDesc, messages) => {
-          await tx.send(await this.preparePayload(topicOrDesc, messages));
+          const payload = await this.preparePayload(topicOrDesc, messages);
+          await tx.send(payload);
+          this.notifyAfterSend(payload.topic, payload.messages.length);
         }
       };
       await fn(ctx);
@@ -955,23 +1078,28 @@ var KafkaClient = class {
     const deps = this.messageDeps;
     const timeoutMs = options.handlerTimeoutMs;
     await consumer.run({
-      eachMessage: (payload) => handleEachMessage(
-        payload,
-        {
-          schemaMap,
-          handleMessage,
-          interceptors,
-          dlq,
-          retry,
-          retryTopics: options.retryTopics,
-          timeoutMs,
-          wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this)
-        },
-        deps
+      eachMessage: (payload) => this.trackInFlight(
+        () => handleEachMessage(
+          payload,
+          {
+            schemaMap,
+            handleMessage,
+            interceptors,
+            dlq,
+            retry,
+            retryTopics: options.retryTopics,
+            timeoutMs,
+            wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this)
+          },
+          deps
+        )
       )
     });
     this.runningConsumers.set(gid, "eachMessage");
     if (options.retryTopics && retry) {
+      if (!this.autoCreateTopicsEnabled) {
+        await this.validateRetryTopicsExist(topicNames, retry.maxRetries);
+      }
       const companions = await startRetryTopicConsumers(
         topicNames,
         gid,
@@ -988,25 +1116,65 @@ var KafkaClient = class {
     return { groupId: gid, stop: () => this.stopConsumer(gid) };
   }
   async startBatchConsumer(topics, handleBatch, options = {}) {
-    const { consumer, schemaMap, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", options);
+    if (options.retryTopics && !options.retry) {
+      throw new Error(
+        "retryTopics requires retry to be configured \u2014 set retry.maxRetries to enable the retry topic chain"
+      );
+    }
+    if (options.autoCommit !== false) {
+      this.logger.debug?.(
+        `startBatchConsumer: autoCommit is enabled (default true). If your handler calls resolveOffset() or commitOffsetsIfNecessary(), set autoCommit: false to avoid offset conflicts.`
+      );
+    }
+    const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", options);
     const deps = this.messageDeps;
     const timeoutMs = options.handlerTimeoutMs;
     await consumer.run({
-      eachBatch: (payload) => handleEachBatch(
-        payload,
-        {
-          schemaMap,
-          handleBatch,
-          interceptors,
-          dlq,
-          retry,
-          timeoutMs,
-          wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this)
-        },
-        deps
+      eachBatch: (payload) => this.trackInFlight(
+        () => handleEachBatch(
+          payload,
+          {
+            schemaMap,
+            handleBatch,
+            interceptors,
+            dlq,
+            retry,
+            retryTopics: options.retryTopics,
+            timeoutMs,
+            wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this)
+          },
+          deps
+        )
       )
     });
     this.runningConsumers.set(gid, "eachBatch");
+    if (options.retryTopics && retry) {
+      if (!this.autoCreateTopicsEnabled) {
+        await this.validateRetryTopicsExist(topicNames, retry.maxRetries);
+      }
+      const handleMessageForRetry = (env) => handleBatch([env], {
+        partition: env.partition,
+        highWatermark: env.offset,
+        heartbeat: async () => {
+        },
+        resolveOffset: () => {
+        },
+        commitOffsetsIfNecessary: async () => {
+        }
+      });
+      const companions = await startRetryTopicConsumers(
+        topicNames,
+        gid,
+        handleMessageForRetry,
+        retry,
+        dlq,
+        interceptors,
+        schemaMap,
+        this.retryTopicDeps,
+        options.retryTopicAssignmentTimeoutMs
+      );
+      this.companionGroupIds.set(gid, companions);
+    }
     return { groupId: gid, stop: () => this.stopConsumer(gid) };
   }
   // ── Consumer lifecycle ───────────────────────────────────────────
@@ -1036,18 +1204,32 @@ var KafkaClient = class {
           this.consumerCreationOptions.delete(cGroupId);
           this.logger.log(`Retry consumer disconnected: group "${cGroupId}"`);
         }
+        const txId = `${cGroupId}-tx`;
+        const txProducer = this.retryTxProducers.get(txId);
+        if (txProducer) {
+          await txProducer.disconnect().catch(() => {
+          });
+          this.retryTxProducers.delete(txId);
+        }
       }
       this.companionGroupIds.delete(groupId);
     } else {
-      const tasks = Array.from(this.consumers.values()).map(
-        (c) => c.disconnect().catch(() => {
-        })
-      );
+      const tasks = [
+        ...Array.from(this.consumers.values()).map(
+          (c) => c.disconnect().catch(() => {
+          })
+        ),
+        ...Array.from(this.retryTxProducers.values()).map(
+          (p) => p.disconnect().catch(() => {
+          })
+        )
+      ];
       await Promise.allSettled(tasks);
       this.consumers.clear();
       this.runningConsumers.clear();
       this.consumerCreationOptions.clear();
       this.companionGroupIds.clear();
+      this.retryTxProducers.clear();
       this.logger.log("All consumers disconnected");
     }
   }
@@ -1055,17 +1237,24 @@ var KafkaClient = class {
    * Query consumer group lag per partition.
    * Lag = broker high-watermark − last committed offset.
    * A committed offset of -1 (nothing committed yet) counts as full lag.
+   *
+   * Returns an empty array when the consumer group has never committed any
+   * offsets (freshly created group, `autoCommit: false` with no manual commits,
+   * or group not yet assigned). This is a Kafka protocol limitation:
+   * `fetchOffsets` only returns data for topic-partitions that have at least one
+   * committed offset. Use `checkStatus()` to verify broker connectivity in that case.
    */
   async getConsumerLag(groupId) {
     const gid = groupId ?? this.defaultGroupId;
-    if (!this.isAdminConnected) {
-      await this.admin.connect();
-      this.isAdminConnected = true;
-    }
+    await this.ensureAdminConnected();
     const committedByTopic = await this.admin.fetchOffsets({ groupId: gid });
+    const brokerOffsetsAll = await Promise.all(
+      committedByTopic.map(({ topic: topic2 }) => this.admin.fetchTopicOffsets(topic2))
+    );
     const result = [];
-    for (const { topic: topic2, partitions } of committedByTopic) {
-      const brokerOffsets = await this.admin.fetchTopicOffsets(topic2);
+    for (let i = 0; i < committedByTopic.length; i++) {
+      const { topic: topic2, partitions } = committedByTopic[i];
+      const brokerOffsets = brokerOffsetsAll[i];
       for (const { partition, offset } of partitions) {
         const broker = brokerOffsets.find((o) => o.partition === partition);
         if (!broker) continue;
@@ -1080,10 +1269,7 @@ var KafkaClient = class {
   /** Check broker connectivity. Never throws — returns a discriminated union. */
   async checkStatus() {
     try {
-      if (!this.isAdminConnected) {
-        await this.admin.connect();
-        this.isAdminConnected = true;
-      }
+      await this.ensureAdminConnected();
       const topics = await this.admin.listTopics();
       return { status: "up", clientId: this.clientId, topics };
     } catch (error) {
@@ -1098,12 +1284,17 @@ var KafkaClient = class {
     return this.clientId;
   }
   /** Gracefully disconnect producer, all consumers, and admin. */
-  async disconnect() {
+  async disconnect(drainTimeoutMs = 3e4) {
+    await this.waitForDrain(drainTimeoutMs);
     const tasks = [this.producer.disconnect()];
     if (this.txProducer) {
       tasks.push(this.txProducer.disconnect());
       this.txProducer = void 0;
     }
+    for (const p of this.retryTxProducers.values()) {
+      tasks.push(p.disconnect());
+    }
+    this.retryTxProducers.clear();
     for (const consumer of this.consumers.values()) {
       tasks.push(consumer.disconnect());
     }
@@ -1118,9 +1309,67 @@ var KafkaClient = class {
     this.companionGroupIds.clear();
     this.logger.log("All connections closed");
   }
+  // ── Graceful shutdown ────────────────────────────────────────────
+  /**
+   * NestJS lifecycle hook — called automatically when the host module is torn down.
+   * Drains in-flight handlers and disconnects all producers, consumers, and admin.
+   * `KafkaModule` relies on this method; no separate destroy provider is needed.
+   */
+  async onModuleDestroy() {
+    await this.disconnect();
+  }
+  /**
+   * Register SIGTERM / SIGINT handlers that drain in-flight messages before
+   * disconnecting. Call this once after constructing the client in non-NestJS apps.
+   * NestJS apps get drain for free via `onModuleDestroy` → `disconnect()`.
+   */
+  enableGracefulShutdown(signals = ["SIGTERM", "SIGINT"], drainTimeoutMs = 3e4) {
+    const handler = () => {
+      this.logger.log(
+        "Shutdown signal received \u2014 draining in-flight handlers..."
+      );
+      this.disconnect(drainTimeoutMs).catch(
+        (err) => this.logger.error(
+          "Error during graceful shutdown:",
+          toError(err).message
+        )
+      );
+    };
+    for (const signal of signals) {
+      process.once(signal, handler);
+    }
+  }
+  trackInFlight(fn) {
+    this.inFlightTotal++;
+    return fn().finally(() => {
+      this.inFlightTotal--;
+      if (this.inFlightTotal === 0) {
+        this.drainResolvers.splice(0).forEach((r) => r());
+      }
+    });
+  }
+  waitForDrain(timeoutMs) {
+    if (this.inFlightTotal === 0) return Promise.resolve();
+    return new Promise((resolve) => {
+      let handle;
+      const onDrain = () => {
+        clearTimeout(handle);
+        resolve();
+      };
+      this.drainResolvers.push(onDrain);
+      handle = setTimeout(() => {
+        const idx = this.drainResolvers.indexOf(onDrain);
+        if (idx !== -1) this.drainResolvers.splice(idx, 1);
+        this.logger.warn(
+          `Drain timed out after ${timeoutMs}ms \u2014 ${this.inFlightTotal} handler(s) still in flight`
+        );
+        resolve();
+      }, timeoutMs);
+    });
+  }
   // ── Private helpers ──────────────────────────────────────────────
   async preparePayload(topicOrDesc, messages) {
-    registerSchema(topicOrDesc, this.schemaRegistry);
+    registerSchema(topicOrDesc, this.schemaRegistry, this.logger);
     const payload = await buildSendPayload(
       topicOrDesc,
       messages,
@@ -1153,12 +1402,78 @@ var KafkaClient = class {
     }, timeoutMs);
     return promise;
   }
-  async ensureTopic(topic2) {
-    if (!this.autoCreateTopicsEnabled || this.ensuredTopics.has(topic2)) return;
-    if (!this.isAdminConnected) {
+  /**
+   * When `retryTopics: true` and `autoCreateTopics: false`, verify that every
+   * `<topic>.retry.<level>` topic already exists. Throws a clear error at startup
+   * rather than silently discovering missing topics on the first handler failure.
+   */
+  async validateRetryTopicsExist(topicNames, maxRetries) {
+    await this.ensureAdminConnected();
+    const existing = new Set(await this.admin.listTopics());
+    const missing = [];
+    for (const t of topicNames) {
+      for (let level = 1; level <= maxRetries; level++) {
+        const retryTopic = `${t}.retry.${level}`;
+        if (!existing.has(retryTopic)) missing.push(retryTopic);
+      }
+    }
+    if (missing.length > 0) {
+      throw new Error(
+        `retryTopics: true but the following retry topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
+      );
+    }
+  }
+  /**
+   * When `autoCreateTopics` is disabled, verify that `<topic>.dlq` exists for every
+   * consumed topic. Throws a clear error at startup rather than silently discovering
+   * missing DLQ topics on the first handler failure.
+   */
+  async validateDlqTopicsExist(topicNames) {
+    await this.ensureAdminConnected();
+    const existing = new Set(await this.admin.listTopics());
+    const missing = topicNames.filter((t) => !existing.has(`${t}.dlq`)).map((t) => `${t}.dlq`);
+    if (missing.length > 0) {
+      throw new Error(
+        `dlq: true but the following DLQ topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
+      );
+    }
+  }
+  /**
+   * Connect the admin client if not already connected.
+   * The flag is only set to `true` after a successful connect — if `admin.connect()`
+   * throws the flag remains `false` so the next call will retry the connection.
+   */
+  async ensureAdminConnected() {
+    if (this.isAdminConnected) return;
+    try {
       await this.admin.connect();
       this.isAdminConnected = true;
+    } catch (err) {
+      this.isAdminConnected = false;
+      throw err;
     }
+  }
+  /**
+   * Create and connect a transactional producer for EOS retry routing.
+   * Each retry level consumer gets its own producer with a unique `transactionalId`
+   * so Kafka can fence stale producers on restart without affecting other levels.
+   */
+  async createRetryTxProducer(transactionalId) {
+    const p = this.kafka.producer({
+      kafkaJS: {
+        acks: -1,
+        idempotent: true,
+        transactionalId,
+        maxInFlightRequests: 1
+      }
+    });
+    await p.connect();
+    this.retryTxProducers.set(transactionalId, p);
+    return p;
+  }
+  async ensureTopic(topic2) {
+    if (!this.autoCreateTopicsEnabled || this.ensuredTopics.has(topic2)) return;
+    await this.ensureAdminConnected();
     await this.admin.createTopics({
       topics: [{ topic: topic2, numPartitions: this.numPartitions }]
     });
@@ -1182,6 +1497,12 @@ var KafkaClient = class {
         `Cannot use ${mode} on consumer group "${gid}" \u2014 it is already running with ${oppositeMode}. Use a different groupId for this consumer.`
       );
     }
+    if (existingMode === mode) {
+      const callerName = mode === "eachMessage" ? "startConsumer" : "startBatchConsumer";
+      throw new Error(
+        `${callerName}("${gid}") called twice \u2014 this group is already consuming. Call stopConsumer("${gid}") first or pass a different groupId.`
+      );
+    }
     const consumer = getOrCreateConsumer(
       gid,
       fromBeginning,
@@ -1191,7 +1512,8 @@ var KafkaClient = class {
     const schemaMap = buildSchemaMap(
       topics,
       this.schemaRegistry,
-      optionSchemas
+      optionSchemas,
+      this.logger
     );
     const topicNames = topics.map((t) => resolveTopicName(t));
     for (const t of topicNames) {
@@ -1201,6 +1523,9 @@ var KafkaClient = class {
       for (const t of topicNames) {
         await this.ensureTopic(`${t}.dlq`);
       }
+      if (!this.autoCreateTopicsEnabled) {
+        await this.validateDlqTopicsExist(topicNames);
+      }
     }
     await consumer.connect();
     await subscribeWithRetry(
@@ -1219,7 +1544,8 @@ var KafkaClient = class {
     return {
       schemaRegistry: this.schemaRegistry,
       strictSchemasEnabled: this.strictSchemasEnabled,
-      instrumentation: this.instrumentation
+      instrumentation: this.instrumentation,
+      logger: this.logger
     };
   }
   get consumerOpsDeps() {
@@ -1247,7 +1573,8 @@ var KafkaClient = class {
       onMessageLost: this.onMessageLost,
       ensureTopic: (t) => this.ensureTopic(t),
       getOrCreateConsumer: (gid, fb, ac) => getOrCreateConsumer(gid, fb, ac, this.consumerOpsDeps),
-      runningConsumers: this.runningConsumers
+      runningConsumers: this.runningConsumers,
+      createRetryTxProducer: (txId) => this.createRetryTxProducer(txId)
     };
   }
 };
@@ -1390,11 +1717,7 @@ var KafkaModule = class {
       global: options.isGlobal ?? false,
       module: KafkaModule,
       imports: [import_core2.DiscoveryModule],
-      providers: [
-        kafkaClientProvider,
-        KafkaModule.buildDestroyProvider(token),
-        KafkaExplorer
-      ],
+      providers: [kafkaClientProvider, KafkaExplorer],
       exports: [kafkaClientProvider]
     };
   }
@@ -1410,11 +1733,7 @@ var KafkaModule = class {
       global: asyncOptions.isGlobal ?? false,
       module: KafkaModule,
       imports: [...asyncOptions.imports || [], import_core2.DiscoveryModule],
-      providers: [
-        kafkaClientProvider,
-        KafkaModule.buildDestroyProvider(token),
-        KafkaExplorer
-      ],
+      providers: [kafkaClientProvider, KafkaExplorer],
       exports: [kafkaClientProvider]
     };
   }
@@ -1436,15 +1755,6 @@ var KafkaModule = class {
     await client.connectProducer();
     return client;
   }
-  static buildDestroyProvider(token) {
-    return {
-      provide: `${token}_DESTROY`,
-      useFactory: (client) => ({
-        onModuleDestroy: () => client.disconnect()
-      }),
-      inject: [token]
-    };
-  }
 };
 KafkaModule = __decorateClass([
   (0, import_common3.Module)({})