@drarzter/kafka-client 0.5.4 → 0.5.6

package/dist/index.js

@@ -55,10 +55,10 @@ __export(index_exports, {
 });
 module.exports = __toCommonJS(index_exports);
 
-// src/client/kafka.client.ts
+// src/client/kafka.client/index.ts
 var import_kafka_javascript = require("@confluentinc/kafka-javascript");
 
-// src/client/envelope.ts
+// src/client/message/envelope.ts
 var import_node_async_hooks = require("async_hooks");
 var import_node_crypto = require("crypto");
 var HEADER_EVENT_ID = "x-event-id";
@@ -118,6 +118,108 @@ function extractEnvelope(payload, headers, topic2, partition, offset) {
   };
 }
 
+// src/client/kafka.client/producer-ops.ts
+function resolveTopicName(topicOrDescriptor) {
+  if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
+  if (topicOrDescriptor && typeof topicOrDescriptor === "object" && "__topic" in topicOrDescriptor) {
+    return topicOrDescriptor.__topic;
+  }
+  return String(topicOrDescriptor);
+}
+function registerSchema(topicOrDesc, schemaRegistry) {
+  if (topicOrDesc?.__schema) {
+    const topic2 = resolveTopicName(topicOrDesc);
+    schemaRegistry.set(topic2, topicOrDesc.__schema);
+  }
+}
+async function validateMessage(topicOrDesc, message, deps) {
+  if (topicOrDesc?.__schema) {
+    return await topicOrDesc.__schema.parse(message);
+  }
+  if (deps.strictSchemasEnabled && typeof topicOrDesc === "string") {
+    const schema = deps.schemaRegistry.get(topicOrDesc);
+    if (schema) return await schema.parse(message);
+  }
+  return message;
+}
+async function buildSendPayload(topicOrDesc, messages, deps) {
+  registerSchema(topicOrDesc, deps.schemaRegistry);
+  const topic2 = resolveTopicName(topicOrDesc);
+  const builtMessages = await Promise.all(
+    messages.map(async (m) => {
+      const envelopeHeaders = buildEnvelopeHeaders({
+        correlationId: m.correlationId,
+        schemaVersion: m.schemaVersion,
+        eventId: m.eventId,
+        headers: m.headers
+      });
+      for (const inst of deps.instrumentation) {
+        inst.beforeSend?.(topic2, envelopeHeaders);
+      }
+      return {
+        value: JSON.stringify(
+          await validateMessage(topicOrDesc, m.value, deps)
+        ),
+        key: m.key ?? null,
+        headers: envelopeHeaders
+      };
+    })
+  );
+  return { topic: topic2, messages: builtMessages };
+}
+
+// src/client/kafka.client/consumer-ops.ts
+function getOrCreateConsumer(groupId, fromBeginning, autoCommit, deps) {
+  const { consumers, consumerCreationOptions, kafka, onRebalance, logger } = deps;
+  if (consumers.has(groupId)) {
+    const prev = consumerCreationOptions.get(groupId);
+    if (prev.fromBeginning !== fromBeginning || prev.autoCommit !== autoCommit) {
+      logger.warn(
+        `Consumer group "${groupId}" already exists with options (fromBeginning: ${prev.fromBeginning}, autoCommit: ${prev.autoCommit}) \u2014 new options (fromBeginning: ${fromBeginning}, autoCommit: ${autoCommit}) ignored. Use a different groupId to apply different options.`
+      );
+    }
+    return consumers.get(groupId);
+  }
+  consumerCreationOptions.set(groupId, { fromBeginning, autoCommit });
+  const config = {
+    kafkaJS: { groupId, fromBeginning, autoCommit }
+  };
+  if (onRebalance) {
+    const cb = onRebalance;
+    config["rebalance_cb"] = (err, assignment) => {
+      const type = err.code === -175 ? "assign" : "revoke";
+      try {
+        cb(
+          type,
+          assignment.map((p) => ({ topic: p.topic, partition: p.partition }))
+        );
+      } catch (e) {
+        logger.warn(`onRebalance callback threw: ${e.message}`);
+      }
+    };
+  }
+  const consumer = kafka.consumer(config);
+  consumers.set(groupId, consumer);
+  return consumer;
+}
+function buildSchemaMap(topics, schemaRegistry, optionSchemas) {
+  const schemaMap = /* @__PURE__ */ new Map();
+  for (const t of topics) {
+    if (t?.__schema) {
+      const name = resolveTopicName(t);
+      schemaMap.set(name, t.__schema);
+      schemaRegistry.set(name, t.__schema);
+    }
+  }
+  if (optionSchemas) {
+    for (const [k, v] of optionSchemas) {
+      schemaMap.set(k, v);
+      schemaRegistry.set(k, v);
+    }
+  }
+  return schemaMap;
+}
+
 // src/client/errors.ts
 var KafkaProcessingError = class extends Error {
   constructor(message, topic2, originalMessage, options) {
@@ -150,7 +252,7 @@ var KafkaRetryExhaustedError = class extends KafkaProcessingError {
   }
 };
 
-// src/client/consumer-pipeline.ts
+// src/client/consumer/pipeline.ts
 function toError(error) {
   return error instanceof Error ? error : new Error(String(error));
 }
@@ -189,9 +291,20 @@ async function validateWithSchema(message, raw, topic2, schemaMap, interceptors,
         originalHeaders: deps.originalHeaders
       });
     } else {
-      await deps.onMessageLost?.({ topic: topic2, error: validationError, attempt: 0, headers: deps.originalHeaders ?? {} });
+      await deps.onMessageLost?.({
+        topic: topic2,
+        error: validationError,
+        attempt: 0,
+        headers: deps.originalHeaders ?? {}
+      });
     }
-    const errorEnvelope = extractEnvelope(message, deps.originalHeaders ?? {}, topic2, -1, "");
+    const errorEnvelope = extractEnvelope(
+      message,
+      deps.originalHeaders ?? {},
+      topic2,
+      -1,
+      ""
+    );
     for (const interceptor of interceptors) {
       await interceptor.onError?.(errorEnvelope, validationError);
     }
@@ -243,7 +356,10 @@ async function sendToRetryTopic(originalTopic, rawMessages, attempt, maxRetries,
   };
   try {
     for (const raw of rawMessages) {
-      await deps.producer.send({ topic: retryTopic, messages: [{ value: raw, headers }] });
+      await deps.producer.send({
+        topic: retryTopic,
+        messages: [{ value: raw, headers }]
+      });
     }
     deps.logger.warn(
       `Message queued in retry topic ${retryTopic} (attempt ${attempt}/${maxRetries})`
@@ -255,106 +371,244 @@ async function sendToRetryTopic(originalTopic, rawMessages, attempt, maxRetries,
     );
   }
 }
+async function broadcastToInterceptors(envelopes, interceptors, cb) {
+  for (const env of envelopes) {
+    for (const interceptor of interceptors) {
+      await cb(interceptor, env);
+    }
+  }
+}
+async function runHandlerWithPipeline(fn, envelopes, interceptors, instrumentation) {
+  const cleanups = [];
+  try {
+    for (const env of envelopes) {
+      for (const inst of instrumentation) {
+        const cleanup = inst.beforeConsume?.(env);
+        if (typeof cleanup === "function") cleanups.push(cleanup);
+      }
+    }
+    for (const env of envelopes) {
+      for (const interceptor of interceptors) {
+        await interceptor.before?.(env);
+      }
+    }
+    await fn();
+    for (const env of envelopes) {
+      for (const interceptor of interceptors) {
+        await interceptor.after?.(env);
+      }
+    }
+    for (const cleanup of cleanups) cleanup();
+    return null;
+  } catch (error) {
+    const err = toError(error);
+    for (const env of envelopes) {
+      for (const inst of instrumentation) {
+        inst.onConsumeError?.(env, err);
+      }
+    }
+    for (const cleanup of cleanups) cleanup();
+    return err;
+  }
+}
+async function notifyInterceptorsOnError(envelopes, interceptors, error) {
+  await broadcastToInterceptors(
+    envelopes,
+    interceptors,
+    (i, env) => i.onError?.(env, error)
+  );
+}
 async function executeWithRetry(fn, ctx, deps) {
-  const { envelope, rawMessages, interceptors, dlq, retry, isBatch, retryTopics } = ctx;
+  const {
+    envelope,
+    rawMessages,
+    interceptors,
+    dlq,
+    retry,
+    isBatch,
+    retryTopics
+  } = ctx;
   const maxAttempts = retryTopics ? 1 : retry ? retry.maxRetries + 1 : 1;
   const backoffMs = retry?.backoffMs ?? 1e3;
   const maxBackoffMs = retry?.maxBackoffMs ?? 3e4;
   const envelopes = Array.isArray(envelope) ? envelope : [envelope];
   const topic2 = envelopes[0]?.topic ?? "unknown";
   for (let attempt = 1; attempt <= maxAttempts; attempt++) {
-    const cleanups = [];
-    try {
-      for (const env of envelopes) {
-        for (const inst of deps.instrumentation) {
-          const cleanup = inst.beforeConsume?.(env);
-          if (typeof cleanup === "function") cleanups.push(cleanup);
-        }
-      }
-      for (const env of envelopes) {
-        for (const interceptor of interceptors) {
-          await interceptor.before?.(env);
-        }
-      }
-      await fn();
-      for (const env of envelopes) {
-        for (const interceptor of interceptors) {
-          await interceptor.after?.(env);
-        }
-      }
-      for (const cleanup of cleanups) cleanup();
-      return;
-    } catch (error) {
-      const err = toError(error);
-      const isLastAttempt = attempt === maxAttempts;
-      for (const env of envelopes) {
-        for (const inst of deps.instrumentation) {
-          inst.onConsumeError?.(env, err);
-        }
-      }
-      for (const cleanup of cleanups) cleanup();
-      if (isLastAttempt && maxAttempts > 1) {
-        const exhaustedError = new KafkaRetryExhaustedError(
-          topic2,
-          envelopes.map((e) => e.payload),
-          maxAttempts,
-          { cause: err }
-        );
-        for (const env of envelopes) {
-          for (const interceptor of interceptors) {
-            await interceptor.onError?.(env, exhaustedError);
-          }
-        }
-      } else {
-        for (const env of envelopes) {
-          for (const interceptor of interceptors) {
-            await interceptor.onError?.(env, err);
-          }
-        }
-      }
-      deps.logger.error(
-        `Error processing ${isBatch ? "batch" : "message"} from topic ${topic2} (attempt ${attempt}/${maxAttempts}):`,
-        err.stack
+    const error = await runHandlerWithPipeline(
+      fn,
+      envelopes,
+      interceptors,
+      deps.instrumentation
+    );
+    if (!error) return;
+    const isLastAttempt = attempt === maxAttempts;
+    const reportedError = isLastAttempt && maxAttempts > 1 ? new KafkaRetryExhaustedError(
+      topic2,
+      envelopes.map((e) => e.payload),
+      maxAttempts,
+      { cause: error }
+    ) : error;
+    await notifyInterceptorsOnError(envelopes, interceptors, reportedError);
+    deps.logger.error(
+      `Error processing ${isBatch ? "batch" : "message"} from topic ${topic2} (attempt ${attempt}/${maxAttempts}):`,
+      error.stack
+    );
+    if (retryTopics && retry) {
+      const cap = Math.min(backoffMs, maxBackoffMs);
+      const delay = Math.floor(Math.random() * cap);
+      await sendToRetryTopic(
+        topic2,
+        rawMessages,
+        1,
+        retry.maxRetries,
+        delay,
+        envelopes[0]?.headers ?? {},
+        deps
       );
-      if (retryTopics && retry) {
-        const cap = Math.min(backoffMs, maxBackoffMs);
-        const delay = Math.floor(Math.random() * cap);
-        await sendToRetryTopic(
-          topic2,
-          rawMessages,
-          1,
-          retry.maxRetries,
-          delay,
-          envelopes[0]?.headers ?? {},
-          deps
-        );
-      } else if (isLastAttempt) {
-        if (dlq) {
-          const dlqMeta = {
-            error: err,
-            attempt,
-            originalHeaders: envelopes[0]?.headers
-          };
-          for (const raw of rawMessages) {
-            await sendToDlq(topic2, raw, deps, dlqMeta);
-          }
-        } else {
-          await deps.onMessageLost?.({
-            topic: topic2,
-            error: err,
-            attempt,
-            headers: envelopes[0]?.headers ?? {}
-          });
+    } else if (isLastAttempt) {
+      if (dlq) {
+        const dlqMeta = {
+          error,
+          attempt,
+          originalHeaders: envelopes[0]?.headers
+        };
+        for (const raw of rawMessages) {
+          await sendToDlq(topic2, raw, deps, dlqMeta);
         }
       } else {
-        const cap = Math.min(backoffMs * 2 ** (attempt - 1), maxBackoffMs);
-        await sleep(Math.random() * cap);
+        await deps.onMessageLost?.({
+          topic: topic2,
+          error,
+          attempt,
+          headers: envelopes[0]?.headers ?? {}
+        });
       }
+    } else {
+      const cap = Math.min(backoffMs * 2 ** (attempt - 1), maxBackoffMs);
+      await sleep(Math.random() * cap);
     }
   }
 }
 
-// src/client/subscribe-retry.ts
+// src/client/kafka.client/message-handler.ts
+async function parseSingleMessage(message, topic2, partition, schemaMap, interceptors, dlq, deps) {
+  if (!message.value) {
+    deps.logger.warn(`Received empty message from topic ${topic2}`);
+    return null;
+  }
+  const raw = message.value.toString();
+  const parsed = parseJsonMessage(raw, topic2, deps.logger);
+  if (parsed === null) return null;
+  const headers = decodeHeaders(message.headers);
+  const validated = await validateWithSchema(
+    parsed,
+    raw,
+    topic2,
+    schemaMap,
+    interceptors,
+    dlq,
+    { ...deps, originalHeaders: headers }
+  );
+  if (validated === null) return null;
+  return extractEnvelope(validated, headers, topic2, partition, message.offset);
+}
+async function handleEachMessage(payload, opts, deps) {
+  const { topic: topic2, partition, message } = payload;
+  const {
+    schemaMap,
+    handleMessage,
+    interceptors,
+    dlq,
+    retry,
+    retryTopics,
+    timeoutMs,
+    wrapWithTimeout
+  } = opts;
+  const envelope = await parseSingleMessage(
+    message,
+    topic2,
+    partition,
+    schemaMap,
+    interceptors,
+    dlq,
+    deps
+  );
+  if (envelope === null) return;
+  await executeWithRetry(
+    () => {
+      const fn = () => runWithEnvelopeContext(
+        {
+          correlationId: envelope.correlationId,
+          traceparent: envelope.traceparent
+        },
+        () => handleMessage(envelope)
+      );
+      return timeoutMs ? wrapWithTimeout(fn, timeoutMs, topic2) : fn();
+    },
+    {
+      envelope,
+      rawMessages: [message.value.toString()],
+      interceptors,
+      dlq,
+      retry,
+      retryTopics
+    },
+    deps
+  );
+}
+async function handleEachBatch(payload, opts, deps) {
+  const { batch, heartbeat, resolveOffset, commitOffsetsIfNecessary } = payload;
+  const {
+    schemaMap,
+    handleBatch,
+    interceptors,
+    dlq,
+    retry,
+    timeoutMs,
+    wrapWithTimeout
+  } = opts;
+  const envelopes = [];
+  const rawMessages = [];
+  for (const message of batch.messages) {
+    const envelope = await parseSingleMessage(
+      message,
+      batch.topic,
+      batch.partition,
+      schemaMap,
+      interceptors,
+      dlq,
+      deps
+    );
+    if (envelope === null) continue;
+    envelopes.push(envelope);
+    rawMessages.push(message.value.toString());
+  }
+  if (envelopes.length === 0) return;
+  const meta = {
+    partition: batch.partition,
+    highWatermark: batch.highWatermark,
+    heartbeat,
+    resolveOffset,
+    commitOffsetsIfNecessary
+  };
+  await executeWithRetry(
+    () => {
+      const fn = () => handleBatch(envelopes, meta);
+      return timeoutMs ? wrapWithTimeout(fn, timeoutMs, batch.topic) : fn();
+    },
+    {
+      envelope: envelopes,
+      rawMessages: batch.messages.filter((m) => m.value).map((m) => m.value.toString()),
+      interceptors,
+      dlq,
+      retry,
+      isBatch: true
+    },
+    deps
+  );
+}
+
+// src/client/consumer/subscribe-retry.ts
 async function subscribeWithRetry(consumer, topics, logger, retryOpts) {
   const maxAttempts = retryOpts?.retries ?? 5;
   const backoffMs = retryOpts?.backoffMs ?? 5e3;
@@ -373,7 +627,156 @@ async function subscribeWithRetry(consumer, topics, logger, retryOpts) {
   }
 }
 
-// src/client/kafka.client.ts
+// src/client/kafka.client/retry-topic.ts
+async function waitForPartitionAssignment(consumer, topics, logger, timeoutMs = 1e4) {
+  const topicSet = new Set(topics);
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    try {
+      const assigned = consumer.assignment();
+      if (assigned.some((a) => topicSet.has(a.topic))) return;
+    } catch {
+    }
+    await sleep(200);
+  }
+  logger.warn(
+    `Retry consumer did not receive partition assignments for [${topics.join(", ")}] within ${timeoutMs}ms`
+  );
+}
+async function startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap, deps) {
+  const {
+    logger,
+    producer,
+    instrumentation,
+    onMessageLost,
+    ensureTopic,
+    getOrCreateConsumer: getOrCreateConsumer2,
+    runningConsumers
+  } = deps;
+  const retryTopicNames = originalTopics.map((t) => `${t}.retry`);
+  const retryGroupId = `${originalGroupId}-retry`;
+  const backoffMs = retry.backoffMs ?? 1e3;
+  const maxBackoffMs = retry.maxBackoffMs ?? 3e4;
+  const pipelineDeps = { logger, producer, instrumentation, onMessageLost };
+  for (const rt of retryTopicNames) {
+    await ensureTopic(rt);
+  }
+  const consumer = getOrCreateConsumer2(retryGroupId, false, true);
+  await consumer.connect();
+  await subscribeWithRetry(consumer, retryTopicNames, logger);
+  await consumer.run({
+    eachMessage: async ({ topic: retryTopic, partition, message }) => {
+      if (!message.value) return;
+      const raw = message.value.toString();
+      const parsed = parseJsonMessage(raw, retryTopic, logger);
+      if (parsed === null) return;
+      const headers = decodeHeaders(message.headers);
+      const originalTopic = headers[RETRY_HEADER_ORIGINAL_TOPIC] ?? retryTopic.replace(/\.retry$/, "");
+      const currentAttempt = parseInt(
+        headers[RETRY_HEADER_ATTEMPT] ?? "1",
+        10
+      );
+      const maxRetries = parseInt(
+        headers[RETRY_HEADER_MAX_RETRIES] ?? String(retry.maxRetries),
+        10
+      );
+      const retryAfter = parseInt(
+        headers[RETRY_HEADER_AFTER] ?? "0",
+        10
+      );
+      const remaining = retryAfter - Date.now();
+      if (remaining > 0) {
+        consumer.pause([{ topic: retryTopic, partitions: [partition] }]);
+        await sleep(remaining);
+        consumer.resume([{ topic: retryTopic, partitions: [partition] }]);
+      }
+      const validated = await validateWithSchema(
+        parsed,
+        raw,
+        originalTopic,
+        schemaMap,
+        interceptors,
+        dlq,
+        { ...pipelineDeps, originalHeaders: headers }
+      );
+      if (validated === null) return;
+      const envelope = extractEnvelope(
+        validated,
+        headers,
+        originalTopic,
+        partition,
+        message.offset
+      );
+      const error = await runHandlerWithPipeline(
+        () => runWithEnvelopeContext(
+          {
+            correlationId: envelope.correlationId,
+            traceparent: envelope.traceparent
+          },
+          () => handleMessage(envelope)
+        ),
+        [envelope],
+        interceptors,
+        instrumentation
+      );
+      if (error) {
+        const nextAttempt = currentAttempt + 1;
+        const exhausted = currentAttempt >= maxRetries;
+        const reportedError = exhausted && maxRetries > 1 ? new KafkaRetryExhaustedError(
+          originalTopic,
+          [envelope.payload],
+          maxRetries,
+          { cause: error }
+        ) : error;
+        await notifyInterceptorsOnError(
+          [envelope],
+          interceptors,
+          reportedError
+        );
+        logger.error(
+          `Retry consumer error for ${originalTopic} (attempt ${currentAttempt}/${maxRetries}):`,
+          error.stack
+        );
+        if (!exhausted) {
+          const cap = Math.min(backoffMs * 2 ** currentAttempt, maxBackoffMs);
+          const delay = Math.floor(Math.random() * cap);
+          await sendToRetryTopic(
+            originalTopic,
+            [raw],
+            nextAttempt,
+            maxRetries,
+            delay,
+            headers,
+            pipelineDeps
+          );
+        } else if (dlq) {
+          await sendToDlq(originalTopic, raw, pipelineDeps, {
+            error,
+            // +1 to account for the main consumer's initial attempt before
+            // routing to the retry topic, making this consistent with the
+            // in-process retry path where attempt counts all tries.
+            attempt: currentAttempt + 1,
+            originalHeaders: headers
+          });
+        } else {
+          await onMessageLost?.({
+            topic: originalTopic,
+            error,
+            attempt: currentAttempt,
+            headers
+          });
+        }
+      }
+    }
+  });
+  runningConsumers.set(retryGroupId, "eachMessage");
+  await waitForPartitionAssignment(consumer, retryTopicNames, logger);
+  logger.log(
+    `Retry topic consumers started for: ${originalTopics.join(", ")} (group: ${retryGroupId})`
+  );
+}
+
+// src/client/kafka.client/index.ts
 var { Kafka: KafkaClass, logLevel: KafkaLogLevel } = import_kafka_javascript.KafkaJS;
 var KafkaClient = class {
   kafka;
@@ -389,8 +792,10 @@ var KafkaClient = class {
   defaultGroupId;
   schemaRegistry = /* @__PURE__ */ new Map();
   runningConsumers = /* @__PURE__ */ new Map();
+  consumerCreationOptions = /* @__PURE__ */ new Map();
   instrumentation;
   onMessageLost;
+  onRebalance;
   isAdminConnected = false;
   clientId;
   constructor(clientId, groupId, brokers, options) {
@@ -406,6 +811,7 @@ var KafkaClient = class {
     this.numPartitions = options?.numPartitions ?? 1;
     this.instrumentation = options?.instrumentation ?? [];
     this.onMessageLost = options?.onMessageLost;
+    this.onRebalance = options?.onRebalance;
     this.kafka = new KafkaClass({
       kafkaJS: {
         clientId: this.clientId,
@@ -421,7 +827,7 @@ var KafkaClient = class {
     this.admin = this.kafka.admin();
   }
   async sendMessage(topicOrDesc, message, options = {}) {
-    const payload = await this.buildSendPayload(topicOrDesc, [
+    const payload = await this.preparePayload(topicOrDesc, [
       {
         value: message,
         key: options.key,
@@ -431,19 +837,13 @@ var KafkaClient = class {
         eventId: options.eventId
       }
     ]);
-    await this.ensureTopic(payload.topic);
     await this.producer.send(payload);
-    for (const inst of this.instrumentation) {
-      inst.afterSend?.(payload.topic);
-    }
+    this.notifyAfterSend(payload.topic, payload.messages.length);
   }
   async sendBatch(topicOrDesc, messages) {
-    const payload = await this.buildSendPayload(topicOrDesc, messages);
-    await this.ensureTopic(payload.topic);
+    const payload = await this.preparePayload(topicOrDesc, messages);
     await this.producer.send(payload);
-    for (const inst of this.instrumentation) {
-      inst.afterSend?.(payload.topic);
-    }
+    this.notifyAfterSend(payload.topic, payload.messages.length);
   }
   /** Execute multiple sends atomically. Commits on success, aborts on error. */
   async transaction(fn) {
@@ -462,7 +862,7 @@ var KafkaClient = class {
     try {
       const ctx = {
         send: async (topicOrDesc, message, options = {}) => {
-          const payload = await this.buildSendPayload(topicOrDesc, [
+          const payload = await this.preparePayload(topicOrDesc, [
             {
               value: message,
               key: options.key,
@@ -472,13 +872,10 @@ var KafkaClient = class {
               eventId: options.eventId
             }
           ]);
-          await this.ensureTopic(payload.topic);
           await tx.send(payload);
         },
         sendBatch: async (topicOrDesc, messages) => {
-          const payload = await this.buildSendPayload(topicOrDesc, messages);
-          await this.ensureTopic(payload.topic);
-          await tx.send(payload);
+          await tx.send(await this.preparePayload(topicOrDesc, messages));
         }
       };
       await fn(ctx);
@@ -512,131 +909,76 @@ var KafkaClient = class {
       );
     }
     const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", options);
-    const deps = { logger: this.logger, producer: this.producer, instrumentation: this.instrumentation, onMessageLost: this.onMessageLost };
+    const deps = this.messageDeps;
+    const timeoutMs = options.handlerTimeoutMs;
     await consumer.run({
-      eachMessage: async ({ topic: topic2, partition, message }) => {
-        if (!message.value) {
-          this.logger.warn(`Received empty message from topic ${topic2}`);
-          return;
-        }
-        const raw = message.value.toString();
-        const parsed = parseJsonMessage(raw, topic2, this.logger);
-        if (parsed === null) return;
-        const headers = decodeHeaders(message.headers);
-        const validated = await validateWithSchema(
-          parsed,
-          raw,
-          topic2,
+      eachMessage: (payload) => handleEachMessage(
+        payload,
+        {
           schemaMap,
+          handleMessage,
           interceptors,
           dlq,
-          { ...deps, originalHeaders: headers }
-        );
-        if (validated === null) return;
-        const envelope = extractEnvelope(
-          validated,
-          headers,
-          topic2,
-          partition,
-          message.offset
-        );
-        await executeWithRetry(
-          () => runWithEnvelopeContext(
-            { correlationId: envelope.correlationId, traceparent: envelope.traceparent },
-            () => handleMessage(envelope)
-          ),
-          { envelope, rawMessages: [raw], interceptors, dlq, retry, retryTopics: options.retryTopics },
-          deps
-        );
-      }
+          retry,
+          retryTopics: options.retryTopics,
+          timeoutMs,
+          wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this)
+        },
+        deps
+      )
     });
     this.runningConsumers.set(gid, "eachMessage");
     if (options.retryTopics && retry) {
-      await this.startRetryTopicConsumers(
+      await startRetryTopicConsumers(
         topicNames,
         gid,
         handleMessage,
         retry,
         dlq,
         interceptors,
-        schemaMap
+        schemaMap,
+        this.retryTopicDeps
       );
     }
+    return { groupId: gid, stop: () => this.stopConsumer(gid) };
   }
   async startBatchConsumer(topics, handleBatch, options = {}) {
     const { consumer, schemaMap, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", options);
-    const deps = { logger: this.logger, producer: this.producer, instrumentation: this.instrumentation, onMessageLost: this.onMessageLost };
+    const deps = this.messageDeps;
+    const timeoutMs = options.handlerTimeoutMs;
     await consumer.run({
-      eachBatch: async ({
-        batch,
-        heartbeat,
-        resolveOffset,
-        commitOffsetsIfNecessary
-      }) => {
-        const envelopes = [];
-        const rawMessages = [];
-        for (const message of batch.messages) {
-          if (!message.value) {
-            this.logger.warn(
-              `Received empty message from topic ${batch.topic}`
-            );
-            continue;
-          }
-          const raw = message.value.toString();
-          const parsed = parseJsonMessage(raw, batch.topic, this.logger);
-          if (parsed === null) continue;
-          const headers = decodeHeaders(message.headers);
-          const validated = await validateWithSchema(
-            parsed,
-            raw,
-            batch.topic,
-            schemaMap,
-            interceptors,
-            dlq,
-            { ...deps, originalHeaders: headers }
-          );
-          if (validated === null) continue;
-          envelopes.push(
-            extractEnvelope(validated, headers, batch.topic, batch.partition, message.offset)
-          );
-          rawMessages.push(raw);
-        }
-        if (envelopes.length === 0) return;
-        const meta = {
-          partition: batch.partition,
-          highWatermark: batch.highWatermark,
-          heartbeat,
-          resolveOffset,
-          commitOffsetsIfNecessary
-        };
-        await executeWithRetry(
-          () => handleBatch(envelopes, meta),
-          {
-            envelope: envelopes,
-            rawMessages: batch.messages.filter((m) => m.value).map((m) => m.value.toString()),
-            interceptors,
-            dlq,
-            retry,
-            isBatch: true
-          },
-          deps
-        );
-      }
+      eachBatch: (payload) => handleEachBatch(
+        payload,
+        {
+          schemaMap,
+          handleBatch,
+          interceptors,
+          dlq,
+          retry,
+          timeoutMs,
+          wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this)
+        },
+        deps
+      )
     });
     this.runningConsumers.set(gid, "eachBatch");
+    return { groupId: gid, stop: () => this.stopConsumer(gid) };
   }
   // ── Consumer lifecycle ───────────────────────────────────────────
   async stopConsumer(groupId) {
     if (groupId !== void 0) {
       const consumer = this.consumers.get(groupId);
       if (!consumer) {
-        this.logger.warn(`stopConsumer: no active consumer for group "${groupId}"`);
+        this.logger.warn(
+          `stopConsumer: no active consumer for group "${groupId}"`
+        );
         return;
       }
       await consumer.disconnect().catch(() => {
       });
       this.consumers.delete(groupId);
       this.runningConsumers.delete(groupId);
+      this.consumerCreationOptions.delete(groupId);
       this.logger.log(`Consumer disconnected: group "${groupId}"`);
     } else {
       const tasks = Array.from(this.consumers.values()).map(
@@ -646,9 +988,36 @@ var KafkaClient = class {
       await Promise.allSettled(tasks);
       this.consumers.clear();
       this.runningConsumers.clear();
+      this.consumerCreationOptions.clear();
       this.logger.log("All consumers disconnected");
     }
   }
+  /**
+   * Query consumer group lag per partition.
+   * Lag = broker high-watermark − last committed offset.
+   * A committed offset of -1 (nothing committed yet) counts as full lag.
+   */
+  async getConsumerLag(groupId) {
+    const gid = groupId ?? this.defaultGroupId;
+    if (!this.isAdminConnected) {
+      await this.admin.connect();
+      this.isAdminConnected = true;
+    }
+    const committedByTopic = await this.admin.fetchOffsets({ groupId: gid });
+    const result = [];
+    for (const { topic: topic2, partitions } of committedByTopic) {
+      const brokerOffsets = await this.admin.fetchTopicOffsets(topic2);
+      for (const { partition, offset } of partitions) {
+        const broker = brokerOffsets.find((o) => o.partition === partition);
+        if (!broker) continue;
+        const committed = parseInt(offset, 10);
+        const high = parseInt(broker.high, 10);
+        const lag = committed === -1 ? high : Math.max(0, high - committed);
+        result.push({ topic: topic2, partition, lag });
+      }
+    }
+    return result;
+  }
   /** Check broker connectivity and return status, clientId, and available topics. */
   async checkStatus() {
     if (!this.isAdminConnected) {
@@ -678,183 +1047,42 @@ var KafkaClient = class {
     await Promise.allSettled(tasks);
     this.consumers.clear();
     this.runningConsumers.clear();
+    this.consumerCreationOptions.clear();
     this.logger.log("All connections closed");
   }
-  // ── Retry topic chain ────────────────────────────────────────────
-  /**
-   * Auto-start companion consumers on `<topic>.retry` for each original topic.
-   * Called by `startConsumer` when `retryTopics: true`.
-   *
-   * Flow per message:
-   *   1. Sleep until `x-retry-after` (scheduled by the main consumer or previous retry hop)
-   *   2. Call the original handler
-   *   3. On failure: if retries remain → re-send to `<originalTopic>.retry` with incremented attempt
-   *                  if exhausted       → DLQ or onMessageLost
-   */
-  async startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap) {
-    const retryTopicNames = originalTopics.map((t) => `${t}.retry`);
-    const retryGroupId = `${originalGroupId}-retry`;
-    const backoffMs = retry.backoffMs ?? 1e3;
-    const maxBackoffMs = retry.maxBackoffMs ?? 3e4;
-    const deps = {
-      logger: this.logger,
-      producer: this.producer,
-      instrumentation: this.instrumentation,
-      onMessageLost: this.onMessageLost
-    };
-    for (const rt of retryTopicNames) {
-      await this.ensureTopic(rt);
-    }
-    const consumer = this.getOrCreateConsumer(retryGroupId, false, true);
-    await consumer.connect();
-    await subscribeWithRetry(consumer, retryTopicNames, this.logger);
-    await consumer.run({
-      eachMessage: async ({ topic: retryTopic, partition, message }) => {
-        if (!message.value) return;
-        const raw = message.value.toString();
-        const parsed = parseJsonMessage(raw, retryTopic, this.logger);
-        if (parsed === null) return;
-        const headers = decodeHeaders(message.headers);
-        const originalTopic = headers[RETRY_HEADER_ORIGINAL_TOPIC] ?? retryTopic.replace(/\.retry$/, "");
-        const currentAttempt = parseInt(
-          headers[RETRY_HEADER_ATTEMPT] ?? "1",
-          10
-        );
-        const maxRetries = parseInt(
-          headers[RETRY_HEADER_MAX_RETRIES] ?? String(retry.maxRetries),
-          10
-        );
-        const retryAfter = parseInt(
-          headers[RETRY_HEADER_AFTER] ?? "0",
-          10
-        );
-        const remaining = retryAfter - Date.now();
-        if (remaining > 0) {
-          consumer.pause([{ topic: retryTopic, partitions: [partition] }]);
-          await sleep(remaining);
-          consumer.resume([{ topic: retryTopic, partitions: [partition] }]);
-        }
-        const validated = await validateWithSchema(
-          parsed,
-          raw,
-          originalTopic,
-          schemaMap,
-          interceptors,
-          dlq,
-          { ...deps, originalHeaders: headers }
-        );
-        if (validated === null) return;
-        const envelope = extractEnvelope(
-          validated,
-          headers,
-          originalTopic,
-          partition,
-          message.offset
-        );
-        try {
-          const cleanups = [];
-          for (const inst of this.instrumentation) {
-            const c = inst.beforeConsume?.(envelope);
-            if (typeof c === "function") cleanups.push(c);
-          }
-          for (const interceptor of interceptors) await interceptor.before?.(envelope);
-          await runWithEnvelopeContext(
-            { correlationId: envelope.correlationId, traceparent: envelope.traceparent },
-            () => handleMessage(envelope)
-          );
-          for (const interceptor of interceptors) await interceptor.after?.(envelope);
-          for (const cleanup of cleanups) cleanup();
-        } catch (error) {
-          const err = toError(error);
-          const nextAttempt = currentAttempt + 1;
-          const exhausted = currentAttempt >= maxRetries;
-          for (const inst of this.instrumentation) inst.onConsumeError?.(envelope, err);
-          const reportedError = exhausted && maxRetries > 1 ? new KafkaRetryExhaustedError(originalTopic, [envelope.payload], maxRetries, { cause: err }) : err;
-          for (const interceptor of interceptors) {
-            await interceptor.onError?.(envelope, reportedError);
-          }
-          this.logger.error(
-            `Retry consumer error for ${originalTopic} (attempt ${currentAttempt}/${maxRetries}):`,
-            err.stack
-          );
-          if (!exhausted) {
-            const cap = Math.min(backoffMs * 2 ** currentAttempt, maxBackoffMs);
-            const delay = Math.floor(Math.random() * cap);
-            await sendToRetryTopic(
-              originalTopic,
-              [raw],
-              nextAttempt,
-              maxRetries,
-              delay,
-              headers,
-              deps
-            );
-          } else if (dlq) {
-            await sendToDlq(originalTopic, raw, deps, {
-              error: err,
-              // +1 to account for the main consumer's initial attempt before
-              // routing to the retry topic, making this consistent with the
-              // in-process retry path where attempt counts all tries.
-              attempt: currentAttempt + 1,
-              originalHeaders: headers
-            });
-          } else {
-            await deps.onMessageLost?.({
-              topic: originalTopic,
-              error: err,
-              attempt: currentAttempt,
-              headers
-            });
-          }
-        }
-      }
-    });
-    this.runningConsumers.set(retryGroupId, "eachMessage");
-    await this.waitForPartitionAssignment(consumer, retryTopicNames);
-    this.logger.log(
-      `Retry topic consumers started for: ${originalTopics.join(", ")} (group: ${retryGroupId})`
+  // ── Private helpers ──────────────────────────────────────────────
+  async preparePayload(topicOrDesc, messages) {
+    const payload = await buildSendPayload(
+      topicOrDesc,
+      messages,
+      this.producerOpsDeps
     );
+    await this.ensureTopic(payload.topic);
+    return payload;
   }
-  // ── Private helpers ──────────────────────────────────────────────
-  /**
-   * Poll `consumer.assignment()` until the consumer has received at least one
-   * partition for the given topics, then return. Logs a warning and returns
-   * (rather than throwing) on timeout so that a slow broker does not break
-   * the caller — in the worst case a message sent immediately after would be
-   * missed, which is the same behaviour as before this guard was added.
-   */
-  async waitForPartitionAssignment(consumer, topics, timeoutMs = 1e4) {
-    const topicSet = new Set(topics);
-    const deadline = Date.now() + timeoutMs;
-    while (Date.now() < deadline) {
-      try {
-        const assigned = consumer.assignment();
-        if (assigned.some((a) => topicSet.has(a.topic))) return;
-      } catch {
+  // afterSend is called once per message — symmetric with beforeSend in buildSendPayload.
+  notifyAfterSend(topic2, count) {
+    for (let i = 0; i < count; i++) {
+      for (const inst of this.instrumentation) {
+        inst.afterSend?.(topic2);
       }
-      await sleep(200);
     }
-    this.logger.warn(
-      `Retry consumer did not receive partition assignments for [${topics.join(", ")}] within ${timeoutMs}ms`
-    );
   }
-  getOrCreateConsumer(groupId, fromBeginning, autoCommit) {
-    if (!this.consumers.has(groupId)) {
-      this.consumers.set(
-        groupId,
-        this.kafka.consumer({
-          kafkaJS: { groupId, fromBeginning, autoCommit }
-        })
+  /**
+   * Start a timer that logs a warning if `fn` hasn't resolved within `timeoutMs`.
+   * The handler itself is not cancelled — the warning is diagnostic only.
+   */
+  wrapWithTimeoutWarning(fn, timeoutMs, topic2) {
+    let timer;
+    const promise = fn().finally(() => {
+      if (timer !== void 0) clearTimeout(timer);
+    });
+    timer = setTimeout(() => {
+      this.logger.warn(
+        `Handler for topic "${topic2}" has not resolved after ${timeoutMs}ms \u2014 possible stuck handler`
       );
-    }
-    return this.consumers.get(groupId);
-  }
-  resolveTopicName(topicOrDescriptor) {
-    if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
-    if (topicOrDescriptor && typeof topicOrDescriptor === "object" && "__topic" in topicOrDescriptor) {
-      return topicOrDescriptor.__topic;
-    }
-    return String(topicOrDescriptor);
+    }, timeoutMs);
+    return promise;
   }
   async ensureTopic(topic2) {
     if (!this.autoCreateTopicsEnabled || this.ensuredTopics.has(topic2)) return;
@@ -867,52 +1095,6 @@ var KafkaClient = class {
     });
     this.ensuredTopics.add(topic2);
   }
-  /** Register schema from descriptor into global registry (side-effect). */
-  registerSchema(topicOrDesc) {
-    if (topicOrDesc?.__schema) {
-      const topic2 = this.resolveTopicName(topicOrDesc);
-      this.schemaRegistry.set(topic2, topicOrDesc.__schema);
-    }
-  }
-  /** Validate message against schema. Pure — no side-effects on registry. */
-  async validateMessage(topicOrDesc, message) {
-    if (topicOrDesc?.__schema) {
-      return await topicOrDesc.__schema.parse(message);
-    }
-    if (this.strictSchemasEnabled && typeof topicOrDesc === "string") {
-      const schema = this.schemaRegistry.get(topicOrDesc);
-      if (schema) return await schema.parse(message);
-    }
-    return message;
-  }
-  /**
-   * Build a kafkajs-ready send payload.
-   * Handles: topic resolution, schema registration, validation, JSON serialization,
-   * envelope header generation, and instrumentation hooks.
-   */
-  async buildSendPayload(topicOrDesc, messages) {
-    this.registerSchema(topicOrDesc);
-    const topic2 = this.resolveTopicName(topicOrDesc);
-    const builtMessages = await Promise.all(
-      messages.map(async (m) => {
-        const envelopeHeaders = buildEnvelopeHeaders({
-          correlationId: m.correlationId,
-          schemaVersion: m.schemaVersion,
-          eventId: m.eventId,
-          headers: m.headers
-        });
-        for (const inst of this.instrumentation) {
-          inst.beforeSend?.(topic2, envelopeHeaders);
-        }
-        return {
-          value: JSON.stringify(await this.validateMessage(topicOrDesc, m.value)),
-          key: m.key ?? null,
-          headers: envelopeHeaders
-        };
-      })
-    );
-    return { topic: topic2, messages: builtMessages };
-  }
   /** Shared consumer setup: groupId check, schema map, connect, subscribe. */
   async setupConsumer(topics, mode, options) {
     const {
@@ -931,11 +1113,18 @@ var KafkaClient = class {
         `Cannot use ${mode} on consumer group "${gid}" \u2014 it is already running with ${oppositeMode}. Use a different groupId for this consumer.`
       );
     }
-    const consumer = this.getOrCreateConsumer(gid, fromBeginning, options.autoCommit ?? true);
-    const schemaMap = this.buildSchemaMap(topics, optionSchemas);
-    const topicNames = topics.map(
-      (t) => this.resolveTopicName(t)
+    const consumer = getOrCreateConsumer(
+      gid,
+      fromBeginning,
+      options.autoCommit ?? true,
+      this.consumerOpsDeps
     );
+    const schemaMap = buildSchemaMap(
+      topics,
+      this.schemaRegistry,
+      optionSchemas
+    );
+    const topicNames = topics.map((t) => resolveTopicName(t));
     for (const t of topicNames) {
       await this.ensureTopic(t);
     }
@@ -945,32 +1134,56 @@ var KafkaClient = class {
       }
     }
     await consumer.connect();
-    await subscribeWithRetry(consumer, topicNames, this.logger, options.subscribeRetry);
+    await subscribeWithRetry(
+      consumer,
+      topicNames,
+      this.logger,
+      options.subscribeRetry
+    );
     this.logger.log(
       `${mode === "eachBatch" ? "Batch consumer" : "Consumer"} subscribed to topics: ${topicNames.join(", ")}`
     );
     return { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry };
   }
-  buildSchemaMap(topics, optionSchemas) {
-    const schemaMap = /* @__PURE__ */ new Map();
-    for (const t of topics) {
-      if (t?.__schema) {
-        const name = this.resolveTopicName(t);
-        schemaMap.set(name, t.__schema);
-        this.schemaRegistry.set(name, t.__schema);
-      }
-    }
-    if (optionSchemas) {
-      for (const [k, v] of optionSchemas) {
-        schemaMap.set(k, v);
-        this.schemaRegistry.set(k, v);
-      }
-    }
-    return schemaMap;
+  // ── Deps object getters ──────────────────────────────────────────
+  get producerOpsDeps() {
+    return {
+      schemaRegistry: this.schemaRegistry,
+      strictSchemasEnabled: this.strictSchemasEnabled,
+      instrumentation: this.instrumentation
+    };
+  }
+  get consumerOpsDeps() {
+    return {
+      consumers: this.consumers,
+      consumerCreationOptions: this.consumerCreationOptions,
+      kafka: this.kafka,
+      onRebalance: this.onRebalance,
+      logger: this.logger
+    };
+  }
+  get messageDeps() {
+    return {
+      logger: this.logger,
+      producer: this.producer,
+      instrumentation: this.instrumentation,
+      onMessageLost: this.onMessageLost
+    };
+  }
+  get retryTopicDeps() {
+    return {
+      logger: this.logger,
+      producer: this.producer,
+      instrumentation: this.instrumentation,
+      onMessageLost: this.onMessageLost,
+      ensureTopic: (t) => this.ensureTopic(t),
+      getOrCreateConsumer: (gid, fb, ac) => getOrCreateConsumer(gid, fb, ac, this.consumerOpsDeps),
+      runningConsumers: this.runningConsumers
+    };
   }
 };
 
-// src/client/topic.ts
+// src/client/message/topic.ts
 function topic(name) {
   const fn = () => ({
     __topic: name,
@@ -1100,35 +1313,17 @@ var KafkaModule = class {
     const token = getKafkaClientToken(options.name);
     const kafkaClientProvider = {
       provide: token,
-      useFactory: async () => {
-        const client = new KafkaClient(
-          options.clientId,
-          options.groupId,
-          options.brokers,
-          {
-            autoCreateTopics: options.autoCreateTopics,
-            strictSchemas: options.strictSchemas,
-            numPartitions: options.numPartitions,
-            instrumentation: options.instrumentation,
-            logger: new import_common3.Logger(`KafkaClient:${options.clientId}`)
-          }
-        );
-        await client.connectProducer();
-        return client;
-      }
-    };
-    const destroyProvider = {
-      provide: `${token}_DESTROY`,
-      useFactory: (client) => ({
-        onModuleDestroy: () => client.disconnect()
-      }),
-      inject: [token]
+      useFactory: () => KafkaModule.buildClient(options)
     };
     return {
       global: options.isGlobal ?? false,
       module: KafkaModule,
       imports: [import_core2.DiscoveryModule],
-      providers: [kafkaClientProvider, destroyProvider, KafkaExplorer],
+      providers: [
+        kafkaClientProvider,
+        KafkaModule.buildDestroyProvider(token),
+        KafkaExplorer
+      ],
       exports: [kafkaClientProvider]
     };
   }
@@ -1137,40 +1332,48 @@ var KafkaModule = class {
     const token = getKafkaClientToken(asyncOptions.name);
     const kafkaClientProvider = {
       provide: token,
-      useFactory: async (...args) => {
-        const options = await asyncOptions.useFactory(...args);
-        const client = new KafkaClient(
-          options.clientId,
-          options.groupId,
-          options.brokers,
-          {
-            autoCreateTopics: options.autoCreateTopics,
-            strictSchemas: options.strictSchemas,
-            numPartitions: options.numPartitions,
-            instrumentation: options.instrumentation,
-            logger: new import_common3.Logger(`KafkaClient:${options.clientId}`)
-          }
-        );
-        await client.connectProducer();
-        return client;
-      },
+      useFactory: async (...args) => KafkaModule.buildClient(await asyncOptions.useFactory(...args)),
       inject: asyncOptions.inject || []
     };
-    const destroyProvider = {
-      provide: `${token}_DESTROY`,
-      useFactory: (client) => ({
-        onModuleDestroy: () => client.disconnect()
-      }),
-      inject: [token]
-    };
     return {
       global: asyncOptions.isGlobal ?? false,
       module: KafkaModule,
       imports: [...asyncOptions.imports || [], import_core2.DiscoveryModule],
-      providers: [kafkaClientProvider, destroyProvider, KafkaExplorer],
+      providers: [
+        kafkaClientProvider,
+        KafkaModule.buildDestroyProvider(token),
+        KafkaExplorer
+      ],
       exports: [kafkaClientProvider]
     };
   }
+  static async buildClient(options) {
+    const client = new KafkaClient(
+      options.clientId,
+      options.groupId,
+      options.brokers,
+      {
+        autoCreateTopics: options.autoCreateTopics,
+        strictSchemas: options.strictSchemas,
+        numPartitions: options.numPartitions,
+        instrumentation: options.instrumentation,
+        onMessageLost: options.onMessageLost,
+        onRebalance: options.onRebalance,
+        logger: new import_common3.Logger(`KafkaClient:${options.clientId}`)
+      }
+    );
+    await client.connectProducer();
+    return client;
+  }
+  static buildDestroyProvider(token) {
+    return {
+      provide: `${token}_DESTROY`,
+      useFactory: (client) => ({
+        onModuleDestroy: () => client.disconnect()
+      }),
+      inject: [token]
+    };
+  }
 };
 KafkaModule = __decorateClass([
   (0, import_common3.Module)({})