@drarzter/kafka-client 0.5.5 → 0.5.7

package/dist/core.js

@@ -38,10 +38,10 @@ __export(core_exports, {
 });
 module.exports = __toCommonJS(core_exports);
 
-// src/client/kafka.client.ts
+// src/client/kafka.client/index.ts
 var import_kafka_javascript = require("@confluentinc/kafka-javascript");
 
-// src/client/envelope.ts
+// src/client/message/envelope.ts
 var import_node_async_hooks = require("async_hooks");
 var import_node_crypto = require("crypto");
 var HEADER_EVENT_ID = "x-event-id";
@@ -133,7 +133,123 @@ var KafkaRetryExhaustedError = class extends KafkaProcessingError {
   }
 };
 
-// src/client/consumer-pipeline.ts
+// src/client/kafka.client/producer-ops.ts
+function resolveTopicName(topicOrDescriptor) {
+  if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
+  if (topicOrDescriptor && typeof topicOrDescriptor === "object" && "__topic" in topicOrDescriptor) {
+    return topicOrDescriptor.__topic;
+  }
+  return String(topicOrDescriptor);
+}
+function registerSchema(topicOrDesc, schemaRegistry) {
+  if (topicOrDesc?.__schema) {
+    const topic2 = resolveTopicName(topicOrDesc);
+    schemaRegistry.set(topic2, topicOrDesc.__schema);
+  }
+}
+async function validateMessage(topicOrDesc, message, deps) {
+  const topicName = resolveTopicName(topicOrDesc);
+  if (topicOrDesc?.__schema) {
+    try {
+      return await topicOrDesc.__schema.parse(message);
+    } catch (error) {
+      throw new KafkaValidationError(topicName, message, {
+        cause: error instanceof Error ? error : new Error(String(error))
+      });
+    }
+  }
+  if (deps.strictSchemasEnabled && typeof topicOrDesc === "string") {
+    const schema = deps.schemaRegistry.get(topicOrDesc);
+    if (schema) {
+      try {
+        return await schema.parse(message);
+      } catch (error) {
+        throw new KafkaValidationError(topicName, message, {
+          cause: error instanceof Error ? error : new Error(String(error))
+        });
+      }
+    }
+  }
+  return message;
+}
+async function buildSendPayload(topicOrDesc, messages, deps) {
+  const topic2 = resolveTopicName(topicOrDesc);
+  const builtMessages = await Promise.all(
+    messages.map(async (m) => {
+      const envelopeHeaders = buildEnvelopeHeaders({
+        correlationId: m.correlationId,
+        schemaVersion: m.schemaVersion,
+        eventId: m.eventId,
+        headers: m.headers
+      });
+      for (const inst of deps.instrumentation) {
+        inst.beforeSend?.(topic2, envelopeHeaders);
+      }
+      return {
+        value: JSON.stringify(
+          await validateMessage(topicOrDesc, m.value, deps)
+        ),
+        key: m.key ?? null,
+        headers: envelopeHeaders
+      };
+    })
+  );
+  return { topic: topic2, messages: builtMessages };
+}
+
+// src/client/kafka.client/consumer-ops.ts
+function getOrCreateConsumer(groupId, fromBeginning, autoCommit, deps) {
+  const { consumers, consumerCreationOptions, kafka, onRebalance, logger } = deps;
+  if (consumers.has(groupId)) {
+    const prev = consumerCreationOptions.get(groupId);
+    if (prev.fromBeginning !== fromBeginning || prev.autoCommit !== autoCommit) {
+      logger.warn(
+        `Consumer group "${groupId}" already exists with options (fromBeginning: ${prev.fromBeginning}, autoCommit: ${prev.autoCommit}) \u2014 new options (fromBeginning: ${fromBeginning}, autoCommit: ${autoCommit}) ignored. Use a different groupId to apply different options.`
+      );
+    }
+    return consumers.get(groupId);
+  }
+  consumerCreationOptions.set(groupId, { fromBeginning, autoCommit });
+  const config = {
+    kafkaJS: { groupId, fromBeginning, autoCommit }
+  };
+  if (onRebalance) {
+    const cb = onRebalance;
+    config["rebalance_cb"] = (err, assignment) => {
+      const type = err.code === -175 ? "assign" : "revoke";
+      try {
+        cb(
+          type,
+          assignment.map((p) => ({ topic: p.topic, partition: p.partition }))
+        );
+      } catch (e) {
+        logger.warn(`onRebalance callback threw: ${e.message}`);
+      }
+    };
+  }
+  const consumer = kafka.consumer(config);
+  consumers.set(groupId, consumer);
+  return consumer;
+}
+function buildSchemaMap(topics, schemaRegistry, optionSchemas) {
+  const schemaMap = /* @__PURE__ */ new Map();
+  for (const t of topics) {
+    if (t?.__schema) {
+      const name = resolveTopicName(t);
+      schemaMap.set(name, t.__schema);
+      schemaRegistry.set(name, t.__schema);
+    }
+  }
+  if (optionSchemas) {
+    for (const [k, v] of optionSchemas) {
+      schemaMap.set(k, v);
+      schemaRegistry.set(k, v);
+    }
+  }
+  return schemaMap;
+}
+
+// src/client/consumer/pipeline.ts
 function toError(error) {
   return error instanceof Error ? error : new Error(String(error));
 }
@@ -220,7 +336,7 @@ var RETRY_HEADER_AFTER = "x-retry-after";
 var RETRY_HEADER_MAX_RETRIES = "x-retry-max-retries";
 var RETRY_HEADER_ORIGINAL_TOPIC = "x-retry-original-topic";
 async function sendToRetryTopic(originalTopic, rawMessages, attempt, maxRetries, delayMs, originalHeaders, deps) {
-  const retryTopic = `${originalTopic}.retry`;
+  const retryTopic = `${originalTopic}.retry.${attempt}`;
   const {
     [RETRY_HEADER_ATTEMPT]: _a,
     [RETRY_HEADER_AFTER]: _b,
@@ -252,6 +368,53 @@ async function sendToRetryTopic(originalTopic, rawMessages, attempt, maxRetries,
     );
   }
 }
+async function broadcastToInterceptors(envelopes, interceptors, cb) {
+  for (const env of envelopes) {
+    for (const interceptor of interceptors) {
+      await cb(interceptor, env);
+    }
+  }
+}
+async function runHandlerWithPipeline(fn, envelopes, interceptors, instrumentation) {
+  const cleanups = [];
+  try {
+    for (const env of envelopes) {
+      for (const inst of instrumentation) {
+        const cleanup = inst.beforeConsume?.(env);
+        if (typeof cleanup === "function") cleanups.push(cleanup);
+      }
+    }
+    for (const env of envelopes) {
+      for (const interceptor of interceptors) {
+        await interceptor.before?.(env);
+      }
+    }
+    await fn();
+    for (const env of envelopes) {
+      for (const interceptor of interceptors) {
+        await interceptor.after?.(env);
+      }
+    }
+    for (const cleanup of cleanups) cleanup();
+    return null;
+  } catch (error) {
+    const err = toError(error);
+    for (const env of envelopes) {
+      for (const inst of instrumentation) {
+        inst.onConsumeError?.(env, err);
+      }
+    }
+    for (const cleanup of cleanups) cleanup();
+    return err;
+  }
+}
+async function notifyInterceptorsOnError(envelopes, interceptors, error) {
+  await broadcastToInterceptors(
+    envelopes,
+    interceptors,
+    (i, env) => i.onError?.(env, error)
+  );
+}
 async function executeWithRetry(fn, ctx, deps) {
   const {
     envelope,
@@ -268,98 +431,181 @@ async function executeWithRetry(fn, ctx, deps) {
   const envelopes = Array.isArray(envelope) ? envelope : [envelope];
   const topic2 = envelopes[0]?.topic ?? "unknown";
   for (let attempt = 1; attempt <= maxAttempts; attempt++) {
-    const cleanups = [];
-    try {
-      for (const env of envelopes) {
-        for (const inst of deps.instrumentation) {
-          const cleanup = inst.beforeConsume?.(env);
-          if (typeof cleanup === "function") cleanups.push(cleanup);
-        }
-      }
-      for (const env of envelopes) {
-        for (const interceptor of interceptors) {
-          await interceptor.before?.(env);
-        }
-      }
-      await fn();
-      for (const env of envelopes) {
-        for (const interceptor of interceptors) {
-          await interceptor.after?.(env);
-        }
-      }
-      for (const cleanup of cleanups) cleanup();
-      return;
-    } catch (error) {
-      const err = toError(error);
-      const isLastAttempt = attempt === maxAttempts;
-      for (const env of envelopes) {
-        for (const inst of deps.instrumentation) {
-          inst.onConsumeError?.(env, err);
-        }
-      }
-      for (const cleanup of cleanups) cleanup();
-      if (isLastAttempt && maxAttempts > 1) {
-        const exhaustedError = new KafkaRetryExhaustedError(
-          topic2,
-          envelopes.map((e) => e.payload),
-          maxAttempts,
-          { cause: err }
-        );
-        for (const env of envelopes) {
-          for (const interceptor of interceptors) {
-            await interceptor.onError?.(env, exhaustedError);
-          }
-        }
-      } else {
-        for (const env of envelopes) {
-          for (const interceptor of interceptors) {
-            await interceptor.onError?.(env, err);
-          }
-        }
-      }
-      deps.logger.error(
-        `Error processing ${isBatch ? "batch" : "message"} from topic ${topic2} (attempt ${attempt}/${maxAttempts}):`,
-        err.stack
+    const error = await runHandlerWithPipeline(
+      fn,
+      envelopes,
+      interceptors,
+      deps.instrumentation
+    );
+    if (!error) return;
+    const isLastAttempt = attempt === maxAttempts;
+    const reportedError = isLastAttempt && maxAttempts > 1 ? new KafkaRetryExhaustedError(
+      topic2,
+      envelopes.map((e) => e.payload),
+      maxAttempts,
+      { cause: error }
+    ) : error;
+    await notifyInterceptorsOnError(envelopes, interceptors, reportedError);
+    deps.logger.error(
+      `Error processing ${isBatch ? "batch" : "message"} from topic ${topic2} (attempt ${attempt}/${maxAttempts}):`,
+      error.stack
+    );
+    if (retryTopics && retry) {
+      const cap = Math.min(backoffMs, maxBackoffMs);
+      const delay = Math.floor(Math.random() * cap);
+      await sendToRetryTopic(
+        topic2,
+        rawMessages,
+        1,
+        retry.maxRetries,
+        delay,
+        envelopes[0]?.headers ?? {},
+        deps
       );
-      if (retryTopics && retry) {
-        const cap = Math.min(backoffMs, maxBackoffMs);
-        const delay = Math.floor(Math.random() * cap);
-        await sendToRetryTopic(
-          topic2,
-          rawMessages,
-          1,
-          retry.maxRetries,
-          delay,
-          envelopes[0]?.headers ?? {},
-          deps
-        );
-      } else if (isLastAttempt) {
-        if (dlq) {
-          const dlqMeta = {
-            error: err,
-            attempt,
-            originalHeaders: envelopes[0]?.headers
-          };
-          for (const raw of rawMessages) {
-            await sendToDlq(topic2, raw, deps, dlqMeta);
-          }
-        } else {
-          await deps.onMessageLost?.({
-            topic: topic2,
-            error: err,
-            attempt,
-            headers: envelopes[0]?.headers ?? {}
-          });
+    } else if (isLastAttempt) {
+      if (dlq) {
+        const dlqMeta = {
+          error,
+          attempt,
+          originalHeaders: envelopes[0]?.headers
+        };
+        for (const raw of rawMessages) {
+          await sendToDlq(topic2, raw, deps, dlqMeta);
         }
       } else {
-        const cap = Math.min(backoffMs * 2 ** (attempt - 1), maxBackoffMs);
-        await sleep(Math.random() * cap);
+        await deps.onMessageLost?.({
+          topic: topic2,
+          error,
+          attempt,
+          headers: envelopes[0]?.headers ?? {}
+        });
       }
+    } else {
+      const cap = Math.min(backoffMs * 2 ** (attempt - 1), maxBackoffMs);
+      await sleep(Math.random() * cap);
     }
   }
 }
 
-// src/client/subscribe-retry.ts
+// src/client/kafka.client/message-handler.ts
+async function parseSingleMessage(message, topic2, partition, schemaMap, interceptors, dlq, deps) {
+  if (!message.value) {
+    deps.logger.warn(`Received empty message from topic ${topic2}`);
+    return null;
+  }
+  const raw = message.value.toString();
+  const parsed = parseJsonMessage(raw, topic2, deps.logger);
+  if (parsed === null) return null;
+  const headers = decodeHeaders(message.headers);
+  const validated = await validateWithSchema(
+    parsed,
+    raw,
+    topic2,
+    schemaMap,
+    interceptors,
+    dlq,
+    { ...deps, originalHeaders: headers }
+  );
+  if (validated === null) return null;
+  return extractEnvelope(validated, headers, topic2, partition, message.offset);
+}
+async function handleEachMessage(payload, opts, deps) {
+  const { topic: topic2, partition, message } = payload;
+  const {
+    schemaMap,
+    handleMessage,
+    interceptors,
+    dlq,
+    retry,
+    retryTopics,
+    timeoutMs,
+    wrapWithTimeout
+  } = opts;
+  const envelope = await parseSingleMessage(
+    message,
+    topic2,
+    partition,
+    schemaMap,
+    interceptors,
+    dlq,
+    deps
+  );
+  if (envelope === null) return;
+  await executeWithRetry(
+    () => {
+      const fn = () => runWithEnvelopeContext(
+        {
+          correlationId: envelope.correlationId,
+          traceparent: envelope.traceparent
+        },
+        () => handleMessage(envelope)
+      );
+      return timeoutMs ? wrapWithTimeout(fn, timeoutMs, topic2) : fn();
+    },
+    {
+      envelope,
+      rawMessages: [message.value.toString()],
+      interceptors,
+      dlq,
+      retry,
+      retryTopics
+    },
+    deps
+  );
+}
+async function handleEachBatch(payload, opts, deps) {
+  const { batch, heartbeat, resolveOffset, commitOffsetsIfNecessary } = payload;
+  const {
+    schemaMap,
+    handleBatch,
+    interceptors,
+    dlq,
+    retry,
+    timeoutMs,
+    wrapWithTimeout
+  } = opts;
+  const envelopes = [];
+  const rawMessages = [];
+  for (const message of batch.messages) {
+    const envelope = await parseSingleMessage(
+      message,
+      batch.topic,
+      batch.partition,
+      schemaMap,
+      interceptors,
+      dlq,
+      deps
+    );
+    if (envelope === null) continue;
+    envelopes.push(envelope);
+    rawMessages.push(message.value.toString());
+  }
+  if (envelopes.length === 0) return;
+  const meta = {
+    partition: batch.partition,
+    highWatermark: batch.highWatermark,
+    heartbeat,
+    resolveOffset,
+    commitOffsetsIfNecessary
+  };
+  await executeWithRetry(
+    () => {
+      const fn = () => handleBatch(envelopes, meta);
+      return timeoutMs ? wrapWithTimeout(fn, timeoutMs, batch.topic) : fn();
+    },
+    {
+      envelope: envelopes,
+      rawMessages: batch.messages.filter((m) => m.value).map((m) => m.value.toString()),
+      interceptors,
+      dlq,
+      retry,
+      isBatch: true
+    },
+    deps
+  );
+}
+
+// src/client/consumer/subscribe-retry.ts
 async function subscribeWithRetry(consumer, topics, logger, retryOpts) {
   const maxAttempts = retryOpts?.retries ?? 5;
   const backoffMs = retryOpts?.backoffMs ?? 5e3;
@@ -378,7 +624,183 @@ async function subscribeWithRetry(consumer, topics, logger, retryOpts) {
   }
 }
 
-// src/client/kafka.client.ts
+// src/client/kafka.client/retry-topic.ts
+async function waitForPartitionAssignment(consumer, topics, logger, timeoutMs = 1e4) {
+  const topicSet = new Set(topics);
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    try {
+      const assigned = consumer.assignment();
+      if (assigned.some((a) => topicSet.has(a.topic))) return;
+    } catch {
+    }
+    await sleep(200);
+  }
+  logger.warn(
+    `Retry consumer did not receive partition assignments for [${topics.join(", ")}] within ${timeoutMs}ms`
+  );
+}
+async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopics, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs) {
+  const {
+    logger,
+    producer,
+    instrumentation,
+    onMessageLost,
+    ensureTopic,
+    getOrCreateConsumer: getOrCreateConsumer2,
+    runningConsumers
+  } = deps;
+  const backoffMs = retry.backoffMs ?? 1e3;
+  const maxBackoffMs = retry.maxBackoffMs ?? 3e4;
+  const pipelineDeps = { logger, producer, instrumentation, onMessageLost };
+  for (const lt of levelTopics) {
+    await ensureTopic(lt);
+  }
+  const consumer = getOrCreateConsumer2(levelGroupId, false, false);
+  await consumer.connect();
+  await subscribeWithRetry(consumer, levelTopics, logger);
+  await consumer.run({
+    eachMessage: async ({ topic: levelTopic, partition, message }) => {
+      const nextOffset = {
+        topic: levelTopic,
+        partition,
+        offset: (parseInt(message.offset, 10) + 1).toString()
+      };
+      if (!message.value) {
+        await consumer.commitOffsets([nextOffset]);
+        return;
+      }
+      const headers = decodeHeaders(message.headers);
+      const retryAfter = parseInt(
+        headers[RETRY_HEADER_AFTER] ?? "0",
+        10
+      );
+      const remaining = retryAfter - Date.now();
+      if (remaining > 0) {
+        consumer.pause([{ topic: levelTopic, partitions: [partition] }]);
+        await sleep(remaining);
+        consumer.resume([{ topic: levelTopic, partitions: [partition] }]);
+      }
+      const raw = message.value.toString();
+      const parsed = parseJsonMessage(raw, levelTopic, logger);
+      if (parsed === null) {
+        await consumer.commitOffsets([nextOffset]);
+        return;
+      }
+      const currentMaxRetries = parseInt(
+        headers[RETRY_HEADER_MAX_RETRIES] ?? String(retry.maxRetries),
+        10
+      );
+      const originalTopic = headers[RETRY_HEADER_ORIGINAL_TOPIC] ?? levelTopic.replace(/\.retry\.\d+$/, "");
+      const validated = await validateWithSchema(
+        parsed,
+        raw,
+        originalTopic,
+        schemaMap,
+        interceptors,
+        dlq,
+        { ...pipelineDeps, originalHeaders: headers }
+      );
+      if (validated === null) {
+        await consumer.commitOffsets([nextOffset]);
+        return;
+      }
+      const envelope = extractEnvelope(
+        validated,
+        headers,
+        originalTopic,
+        partition,
+        message.offset
+      );
+      const error = await runHandlerWithPipeline(
+        () => runWithEnvelopeContext(
+          {
+            correlationId: envelope.correlationId,
+            traceparent: envelope.traceparent
+          },
+          () => handleMessage(envelope)
+        ),
+        [envelope],
+        interceptors,
+        instrumentation
+      );
+      if (!error) {
+        await consumer.commitOffsets([nextOffset]);
+        return;
+      }
+      const exhausted = level >= currentMaxRetries;
+      const reportedError = exhausted && currentMaxRetries > 1 ? new KafkaRetryExhaustedError(
+        originalTopic,
+        [envelope.payload],
+        currentMaxRetries,
+        { cause: error }
+      ) : error;
+      await notifyInterceptorsOnError([envelope], interceptors, reportedError);
+      logger.error(
+        `Retry consumer error for ${originalTopic} (level ${level}/${currentMaxRetries}):`,
+        error.stack
+      );
+      if (!exhausted) {
+        const nextLevel = level + 1;
+        const cap = Math.min(backoffMs * 2 ** level, maxBackoffMs);
+        const delay = Math.floor(Math.random() * cap);
+        await sendToRetryTopic(
+          originalTopic,
+          [raw],
+          nextLevel,
+          currentMaxRetries,
+          delay,
+          headers,
+          pipelineDeps
+        );
+      } else if (dlq) {
+        await sendToDlq(originalTopic, raw, pipelineDeps, {
+          error,
+          // +1 to account for the main consumer's initial attempt before routing.
+          attempt: level + 1,
+          originalHeaders: headers
+        });
+      } else {
+        await onMessageLost?.({
+          topic: originalTopic,
+          error,
+          attempt: level,
+          headers
+        });
+      }
+      await consumer.commitOffsets([nextOffset]);
+    }
+  });
+  runningConsumers.set(levelGroupId, "eachMessage");
+  await waitForPartitionAssignment(consumer, levelTopics, logger, assignmentTimeoutMs);
+  logger.log(
+    `Retry level ${level}/${retry.maxRetries} consumer started for: ${originalTopics.join(", ")} (group: ${levelGroupId})`
+  );
+}
+async function startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs) {
+  const levelGroupIds = [];
+  for (let level = 1; level <= retry.maxRetries; level++) {
+    const levelTopics = originalTopics.map((t) => `${t}.retry.${level}`);
+    const levelGroupId = `${originalGroupId}-retry.${level}`;
+    await startLevelConsumer(
+      level,
+      levelTopics,
+      levelGroupId,
+      originalTopics,
+      handleMessage,
+      retry,
+      dlq,
+      interceptors,
+      schemaMap,
+      deps,
+      assignmentTimeoutMs
+    );
+    levelGroupIds.push(levelGroupId);
+  }
+  return levelGroupIds;
+}
+
+// src/client/kafka.client/index.ts
 var { Kafka: KafkaClass, logLevel: KafkaLogLevel } = import_kafka_javascript.KafkaJS;
 var KafkaClient = class {
   kafka;
@@ -394,6 +816,9 @@ var KafkaClient = class {
   defaultGroupId;
   schemaRegistry = /* @__PURE__ */ new Map();
   runningConsumers = /* @__PURE__ */ new Map();
+  consumerCreationOptions = /* @__PURE__ */ new Map();
+  /** Maps each main consumer groupId to its companion retry level groupIds. */
+  companionGroupIds = /* @__PURE__ */ new Map();
   instrumentation;
   onMessageLost;
   onRebalance;
@@ -428,7 +853,7 @@ var KafkaClient = class {
     this.admin = this.kafka.admin();
   }
   async sendMessage(topicOrDesc, message, options = {}) {
-    const payload = await this.buildSendPayload(topicOrDesc, [
+    const payload = await this.preparePayload(topicOrDesc, [
       {
         value: message,
         key: options.key,
@@ -438,19 +863,13 @@ var KafkaClient = class {
         eventId: options.eventId
       }
     ]);
-    await this.ensureTopic(payload.topic);
     await this.producer.send(payload);
-    for (const inst of this.instrumentation) {
-      inst.afterSend?.(payload.topic);
-    }
+    this.notifyAfterSend(payload.topic, payload.messages.length);
   }
   async sendBatch(topicOrDesc, messages) {
-    const payload = await this.buildSendPayload(topicOrDesc, messages);
-    await this.ensureTopic(payload.topic);
+    const payload = await this.preparePayload(topicOrDesc, messages);
     await this.producer.send(payload);
-    for (const inst of this.instrumentation) {
-      inst.afterSend?.(payload.topic);
-    }
+    this.notifyAfterSend(payload.topic, payload.messages.length);
   }
   /** Execute multiple sends atomically. Commits on success, aborts on error. */
   async transaction(fn) {
@@ -469,7 +888,7 @@ var KafkaClient = class {
     try {
       const ctx = {
         send: async (topicOrDesc, message, options = {}) => {
-          const payload = await this.buildSendPayload(topicOrDesc, [
+          const payload = await this.preparePayload(topicOrDesc, [
             {
               value: message,
               key: options.key,
@@ -479,13 +898,10 @@ var KafkaClient = class {
               eventId: options.eventId
             }
           ]);
-          await this.ensureTopic(payload.topic);
           await tx.send(payload);
         },
         sendBatch: async (topicOrDesc, messages) => {
-          const payload = await this.buildSendPayload(topicOrDesc, messages);
-          await this.ensureTopic(payload.topic);
-          await tx.send(payload);
+          await tx.send(await this.preparePayload(topicOrDesc, messages));
         }
       };
       await fn(ctx);
@@ -519,151 +935,59 @@ var KafkaClient = class {
       );
     }
     const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", options);
-    const deps = {
-      logger: this.logger,
-      producer: this.producer,
-      instrumentation: this.instrumentation,
-      onMessageLost: this.onMessageLost
-    };
+    const deps = this.messageDeps;
     const timeoutMs = options.handlerTimeoutMs;
     await consumer.run({
-      eachMessage: async ({ topic: topic2, partition, message }) => {
-        if (!message.value) {
-          this.logger.warn(`Received empty message from topic ${topic2}`);
-          return;
-        }
-        const raw = message.value.toString();
-        const parsed = parseJsonMessage(raw, topic2, this.logger);
-        if (parsed === null) return;
-        const headers = decodeHeaders(message.headers);
-        const validated = await validateWithSchema(
-          parsed,
-          raw,
-          topic2,
+      eachMessage: (payload) => handleEachMessage(
+        payload,
+        {
           schemaMap,
+          handleMessage,
           interceptors,
           dlq,
-          { ...deps, originalHeaders: headers }
-        );
-        if (validated === null) return;
-        const envelope = extractEnvelope(
-          validated,
-          headers,
-          topic2,
-          partition,
-          message.offset
-        );
-        await executeWithRetry(
-          () => {
-            const fn = () => runWithEnvelopeContext(
-              {
-                correlationId: envelope.correlationId,
-                traceparent: envelope.traceparent
-              },
-              () => handleMessage(envelope)
-            );
-            return timeoutMs ? this.wrapWithTimeoutWarning(fn, timeoutMs, topic2) : fn();
-          },
-          {
-            envelope,
-            rawMessages: [raw],
-            interceptors,
-            dlq,
-            retry,
-            retryTopics: options.retryTopics
-          },
-          deps
-        );
-      }
+          retry,
+          retryTopics: options.retryTopics,
+          timeoutMs,
+          wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this)
+        },
+        deps
+      )
     });
     this.runningConsumers.set(gid, "eachMessage");
     if (options.retryTopics && retry) {
-      await this.startRetryTopicConsumers(
+      const companions = await startRetryTopicConsumers(
         topicNames,
         gid,
         handleMessage,
         retry,
         dlq,
         interceptors,
-        schemaMap
+        schemaMap,
+        this.retryTopicDeps,
+        options.retryTopicAssignmentTimeoutMs
       );
+      this.companionGroupIds.set(gid, companions);
     }
     return { groupId: gid, stop: () => this.stopConsumer(gid) };
   }
   async startBatchConsumer(topics, handleBatch, options = {}) {
     const { consumer, schemaMap, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", options);
-    const deps = {
-      logger: this.logger,
-      producer: this.producer,
-      instrumentation: this.instrumentation,
-      onMessageLost: this.onMessageLost
-    };
+    const deps = this.messageDeps;
     const timeoutMs = options.handlerTimeoutMs;
     await consumer.run({
-      eachBatch: async ({
-        batch,
-        heartbeat,
-        resolveOffset,
-        commitOffsetsIfNecessary
-      }) => {
-        const envelopes = [];
-        const rawMessages = [];
-        for (const message of batch.messages) {
-          if (!message.value) {
-            this.logger.warn(
-              `Received empty message from topic ${batch.topic}`
-            );
-            continue;
-          }
-          const raw = message.value.toString();
-          const parsed = parseJsonMessage(raw, batch.topic, this.logger);
-          if (parsed === null) continue;
-          const headers = decodeHeaders(message.headers);
-          const validated = await validateWithSchema(
-            parsed,
-            raw,
-            batch.topic,
-            schemaMap,
-            interceptors,
-            dlq,
-            { ...deps, originalHeaders: headers }
-          );
-          if (validated === null) continue;
-          envelopes.push(
-            extractEnvelope(
-              validated,
-              headers,
-              batch.topic,
-              batch.partition,
-              message.offset
-            )
-          );
-          rawMessages.push(raw);
-        }
-        if (envelopes.length === 0) return;
-        const meta = {
-          partition: batch.partition,
-          highWatermark: batch.highWatermark,
-          heartbeat,
-          resolveOffset,
-          commitOffsetsIfNecessary
-        };
-        await executeWithRetry(
-          () => {
-            const fn = () => handleBatch(envelopes, meta);
-            return timeoutMs ? this.wrapWithTimeoutWarning(fn, timeoutMs, batch.topic) : fn();
-          },
-          {
-            envelope: envelopes,
-            rawMessages: batch.messages.filter((m) => m.value).map((m) => m.value.toString()),
-            interceptors,
-            dlq,
-            retry,
-            isBatch: true
-          },
-          deps
-        );
-      }
+      eachBatch: (payload) => handleEachBatch(
+        payload,
+        {
+          schemaMap,
+          handleBatch,
+          interceptors,
+          dlq,
+          retry,
+          timeoutMs,
+          wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this)
+        },
+        deps
+      )
     });
     this.runningConsumers.set(gid, "eachBatch");
     return { groupId: gid, stop: () => this.stopConsumer(gid) };
@@ -682,7 +1006,21 @@ var KafkaClient = class {
       });
       this.consumers.delete(groupId);
       this.runningConsumers.delete(groupId);
+      this.consumerCreationOptions.delete(groupId);
       this.logger.log(`Consumer disconnected: group "${groupId}"`);
+      const companions = this.companionGroupIds.get(groupId) ?? [];
+      for (const cGroupId of companions) {
+        const cConsumer = this.consumers.get(cGroupId);
+        if (cConsumer) {
+          await cConsumer.disconnect().catch(() => {
+          });
+          this.consumers.delete(cGroupId);
+          this.runningConsumers.delete(cGroupId);
+          this.consumerCreationOptions.delete(cGroupId);
+          this.logger.log(`Retry consumer disconnected: group "${cGroupId}"`);
+        }
+      }
+      this.companionGroupIds.delete(groupId);
     } else {
       const tasks = Array.from(this.consumers.values()).map(
         (c) => c.disconnect().catch(() => {
@@ -691,6 +1029,8 @@ var KafkaClient = class {
       await Promise.allSettled(tasks);
       this.consumers.clear();
       this.runningConsumers.clear();
+      this.consumerCreationOptions.clear();
+      this.companionGroupIds.clear();
       this.logger.log("All consumers disconnected");
     }
   }
@@ -720,14 +1060,22 @@ var KafkaClient = class {
     }
     return result;
   }
-  /** Check broker connectivity and return status, clientId, and available topics. */
+  /** Check broker connectivity. Never throws — returns a discriminated union. */
   async checkStatus() {
-    if (!this.isAdminConnected) {
-      await this.admin.connect();
-      this.isAdminConnected = true;
+    try {
+      if (!this.isAdminConnected) {
+        await this.admin.connect();
+        this.isAdminConnected = true;
+      }
+      const topics = await this.admin.listTopics();
+      return { status: "up", clientId: this.clientId, topics };
+    } catch (error) {
+      return {
+        status: "down",
+        clientId: this.clientId,
+        error: error instanceof Error ? error.message : String(error)
+      };
     }
-    const topics = await this.admin.listTopics();
-    return { status: "up", clientId: this.clientId, topics };
   }
   getClientId() {
     return this.clientId;
@@ -749,204 +1097,28 @@ var KafkaClient = class {
     await Promise.allSettled(tasks);
     this.consumers.clear();
     this.runningConsumers.clear();
+    this.consumerCreationOptions.clear();
+    this.companionGroupIds.clear();
     this.logger.log("All connections closed");
   }
-  // ── Retry topic chain ────────────────────────────────────────────
-  /**
-   * Auto-start companion consumers on `<topic>.retry` for each original topic.
-   * Called by `startConsumer` when `retryTopics: true`.
-   *
-   * Flow per message:
-   *   1. Sleep until `x-retry-after` (scheduled by the main consumer or previous retry hop)
-   *   2. Call the original handler
-   *   3. On failure: if retries remain → re-send to `<originalTopic>.retry` with incremented attempt
-   *                  if exhausted       → DLQ or onMessageLost
-   */
-  async startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap) {
-    const retryTopicNames = originalTopics.map((t) => `${t}.retry`);
-    const retryGroupId = `${originalGroupId}-retry`;
-    const backoffMs = retry.backoffMs ?? 1e3;
-    const maxBackoffMs = retry.maxBackoffMs ?? 3e4;
-    const deps = {
-      logger: this.logger,
-      producer: this.producer,
-      instrumentation: this.instrumentation,
-      onMessageLost: this.onMessageLost
-    };
-    for (const rt of retryTopicNames) {
-      await this.ensureTopic(rt);
-    }
-    const consumer = this.getOrCreateConsumer(retryGroupId, false, true);
-    await consumer.connect();
-    await subscribeWithRetry(consumer, retryTopicNames, this.logger);
-    await consumer.run({
-      eachMessage: async ({ topic: retryTopic, partition, message }) => {
-        if (!message.value) return;
-        const raw = message.value.toString();
-        const parsed = parseJsonMessage(raw, retryTopic, this.logger);
-        if (parsed === null) return;
-        const headers = decodeHeaders(message.headers);
-        const originalTopic = headers[RETRY_HEADER_ORIGINAL_TOPIC] ?? retryTopic.replace(/\.retry$/, "");
-        const currentAttempt = parseInt(
-          headers[RETRY_HEADER_ATTEMPT] ?? "1",
-          10
-        );
-        const maxRetries = parseInt(
-          headers[RETRY_HEADER_MAX_RETRIES] ?? String(retry.maxRetries),
-          10
-        );
-        const retryAfter = parseInt(
-          headers[RETRY_HEADER_AFTER] ?? "0",
-          10
-        );
-        const remaining = retryAfter - Date.now();
-        if (remaining > 0) {
-          consumer.pause([{ topic: retryTopic, partitions: [partition] }]);
-          await sleep(remaining);
-          consumer.resume([{ topic: retryTopic, partitions: [partition] }]);
-        }
-        const validated = await validateWithSchema(
-          parsed,
-          raw,
-          originalTopic,
-          schemaMap,
-          interceptors,
-          dlq,
-          { ...deps, originalHeaders: headers }
-        );
-        if (validated === null) return;
-        const envelope = extractEnvelope(
-          validated,
-          headers,
-          originalTopic,
-          partition,
-          message.offset
-        );
-        try {
-          const cleanups = [];
-          for (const inst of this.instrumentation) {
-            const c = inst.beforeConsume?.(envelope);
-            if (typeof c === "function") cleanups.push(c);
-          }
-          for (const interceptor of interceptors)
-            await interceptor.before?.(envelope);
-          await runWithEnvelopeContext(
-            {
-              correlationId: envelope.correlationId,
-              traceparent: envelope.traceparent
-            },
-            () => handleMessage(envelope)
-          );
-          for (const interceptor of interceptors)
-            await interceptor.after?.(envelope);
-          for (const cleanup of cleanups) cleanup();
-        } catch (error) {
-          const err = toError(error);
-          const nextAttempt = currentAttempt + 1;
-          const exhausted = currentAttempt >= maxRetries;
-          for (const inst of this.instrumentation)
-            inst.onConsumeError?.(envelope, err);
-          const reportedError = exhausted && maxRetries > 1 ? new KafkaRetryExhaustedError(
-            originalTopic,
-            [envelope.payload],
-            maxRetries,
-            { cause: err }
-          ) : err;
-          for (const interceptor of interceptors) {
-            await interceptor.onError?.(envelope, reportedError);
-          }
-          this.logger.error(
-            `Retry consumer error for ${originalTopic} (attempt ${currentAttempt}/${maxRetries}):`,
-            err.stack
-          );
-          if (!exhausted) {
-            const cap = Math.min(backoffMs * 2 ** currentAttempt, maxBackoffMs);
-            const delay = Math.floor(Math.random() * cap);
-            await sendToRetryTopic(
-              originalTopic,
-              [raw],
-              nextAttempt,
-              maxRetries,
-              delay,
-              headers,
-              deps
-            );
-          } else if (dlq) {
-            await sendToDlq(originalTopic, raw, deps, {
-              error: err,
-              // +1 to account for the main consumer's initial attempt before
-              // routing to the retry topic, making this consistent with the
-              // in-process retry path where attempt counts all tries.
-              attempt: currentAttempt + 1,
-              originalHeaders: headers
-            });
-          } else {
-            await deps.onMessageLost?.({
-              topic: originalTopic,
-              error: err,
-              attempt: currentAttempt,
-              headers
-            });
-          }
-        }
-      }
-    });
-    this.runningConsumers.set(retryGroupId, "eachMessage");
-    await this.waitForPartitionAssignment(consumer, retryTopicNames);
-    this.logger.log(
-      `Retry topic consumers started for: ${originalTopics.join(", ")} (group: ${retryGroupId})`
-    );
-  }
   // ── Private helpers ──────────────────────────────────────────────
-  /**
-   * Poll `consumer.assignment()` until the consumer has received at least one
-   * partition for the given topics, then return. Logs a warning and returns
-   * (rather than throwing) on timeout so that a slow broker does not break
-   * the caller — in the worst case a message sent immediately after would be
-   * missed, which is the same behaviour as before this guard was added.
-   */
-  async waitForPartitionAssignment(consumer, topics, timeoutMs = 1e4) {
-    const topicSet = new Set(topics);
-    const deadline = Date.now() + timeoutMs;
-    while (Date.now() < deadline) {
-      try {
-        const assigned = consumer.assignment();
-        if (assigned.some((a) => topicSet.has(a.topic))) return;
-      } catch {
-      }
-      await sleep(200);
-    }
-    this.logger.warn(
-      `Retry consumer did not receive partition assignments for [${topics.join(", ")}] within ${timeoutMs}ms`
+  async preparePayload(topicOrDesc, messages) {
+    registerSchema(topicOrDesc, this.schemaRegistry);
+    const payload = await buildSendPayload(
+      topicOrDesc,
+      messages,
+      this.producerOpsDeps
     );
+    await this.ensureTopic(payload.topic);
+    return payload;
   }
-  getOrCreateConsumer(groupId, fromBeginning, autoCommit) {
-    if (!this.consumers.has(groupId)) {
-      const config = {
-        kafkaJS: { groupId, fromBeginning, autoCommit }
-      };
-      if (this.onRebalance) {
-        const onRebalance = this.onRebalance;
-        config["rebalance_cb"] = (err, assignment) => {
-          const type = err.code === -175 ? "assign" : "revoke";
-          try {
-            onRebalance(
-              type,
-              assignment.map((p) => ({
-                topic: p.topic,
-                partition: p.partition
-              }))
-            );
-          } catch (e) {
-            this.logger.warn(
-              `onRebalance callback threw: ${e.message}`
-            );
-          }
-        };
+  // afterSend is called once per message — symmetric with beforeSend in buildSendPayload.
+  notifyAfterSend(topic2, count) {
+    for (let i = 0; i < count; i++) {
+      for (const inst of this.instrumentation) {
+        inst.afterSend?.(topic2);
       }
-      this.consumers.set(groupId, this.kafka.consumer(config));
     }
-    return this.consumers.get(groupId);
   }
   /**
    * Start a timer that logs a warning if `fn` hasn't resolved within `timeoutMs`.
@@ -964,13 +1136,6 @@ var KafkaClient = class {
     }, timeoutMs);
     return promise;
   }
-  resolveTopicName(topicOrDescriptor) {
-    if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
-    if (topicOrDescriptor && typeof topicOrDescriptor === "object" && "__topic" in topicOrDescriptor) {
-      return topicOrDescriptor.__topic;
-    }
-    return String(topicOrDescriptor);
-  }
   async ensureTopic(topic2) {
     if (!this.autoCreateTopicsEnabled || this.ensuredTopics.has(topic2)) return;
     if (!this.isAdminConnected) {
@@ -982,54 +1147,6 @@ var KafkaClient = class {
     });
     this.ensuredTopics.add(topic2);
   }
-  /** Register schema from descriptor into global registry (side-effect). */
-  registerSchema(topicOrDesc) {
-    if (topicOrDesc?.__schema) {
-      const topic2 = this.resolveTopicName(topicOrDesc);
-      this.schemaRegistry.set(topic2, topicOrDesc.__schema);
-    }
-  }
-  /** Validate message against schema. Pure — no side-effects on registry. */
-  async validateMessage(topicOrDesc, message) {
-    if (topicOrDesc?.__schema) {
-      return await topicOrDesc.__schema.parse(message);
-    }
-    if (this.strictSchemasEnabled && typeof topicOrDesc === "string") {
-      const schema = this.schemaRegistry.get(topicOrDesc);
-      if (schema) return await schema.parse(message);
-    }
-    return message;
-  }
-  /**
-   * Build a kafkajs-ready send payload.
-   * Handles: topic resolution, schema registration, validation, JSON serialization,
-   * envelope header generation, and instrumentation hooks.
-   */
-  async buildSendPayload(topicOrDesc, messages) {
-    this.registerSchema(topicOrDesc);
-    const topic2 = this.resolveTopicName(topicOrDesc);
-    const builtMessages = await Promise.all(
-      messages.map(async (m) => {
-        const envelopeHeaders = buildEnvelopeHeaders({
-          correlationId: m.correlationId,
-          schemaVersion: m.schemaVersion,
-          eventId: m.eventId,
-          headers: m.headers
-        });
-        for (const inst of this.instrumentation) {
-          inst.beforeSend?.(topic2, envelopeHeaders);
-        }
-        return {
-          value: JSON.stringify(
-            await this.validateMessage(topicOrDesc, m.value)
-          ),
-          key: m.key ?? null,
-          headers: envelopeHeaders
-        };
-      })
-    );
-    return { topic: topic2, messages: builtMessages };
-  }
   /** Shared consumer setup: groupId check, schema map, connect, subscribe. */
   async setupConsumer(topics, mode, options) {
     const {
@@ -1048,15 +1165,18 @@ var KafkaClient = class {
         `Cannot use ${mode} on consumer group "${gid}" \u2014 it is already running with ${oppositeMode}. Use a different groupId for this consumer.`
       );
     }
-    const consumer = this.getOrCreateConsumer(
+    const consumer = getOrCreateConsumer(
       gid,
       fromBeginning,
-      options.autoCommit ?? true
+      options.autoCommit ?? true,
+      this.consumerOpsDeps
     );
-    const schemaMap = this.buildSchemaMap(topics, optionSchemas);
-    const topicNames = topics.map(
-      (t) => this.resolveTopicName(t)
+    const schemaMap = buildSchemaMap(
+      topics,
+      this.schemaRegistry,
+      optionSchemas
     );
+    const topicNames = topics.map((t) => resolveTopicName(t));
     for (const t of topicNames) {
       await this.ensureTopic(t);
     }
@@ -1077,37 +1197,58 @@ var KafkaClient = class {
     );
     return { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry };
   }
-  buildSchemaMap(topics, optionSchemas) {
-    const schemaMap = /* @__PURE__ */ new Map();
-    for (const t of topics) {
-      if (t?.__schema) {
-        const name = this.resolveTopicName(t);
-        schemaMap.set(name, t.__schema);
-        this.schemaRegistry.set(name, t.__schema);
-      }
-    }
-    if (optionSchemas) {
-      for (const [k, v] of optionSchemas) {
-        schemaMap.set(k, v);
-        this.schemaRegistry.set(k, v);
-      }
-    }
-    return schemaMap;
+  // ── Deps object getters ──────────────────────────────────────────
+  get producerOpsDeps() {
+    return {
+      schemaRegistry: this.schemaRegistry,
+      strictSchemasEnabled: this.strictSchemasEnabled,
+      instrumentation: this.instrumentation
+    };
+  }
+  get consumerOpsDeps() {
+    return {
+      consumers: this.consumers,
+      consumerCreationOptions: this.consumerCreationOptions,
+      kafka: this.kafka,
+      onRebalance: this.onRebalance,
+      logger: this.logger
+    };
+  }
+  get messageDeps() {
+    return {
+      logger: this.logger,
+      producer: this.producer,
+      instrumentation: this.instrumentation,
+      onMessageLost: this.onMessageLost
+    };
+  }
+  get retryTopicDeps() {
+    return {
+      logger: this.logger,
+      producer: this.producer,
+      instrumentation: this.instrumentation,
+      onMessageLost: this.onMessageLost,
+      ensureTopic: (t) => this.ensureTopic(t),
+      getOrCreateConsumer: (gid, fb, ac) => getOrCreateConsumer(gid, fb, ac, this.consumerOpsDeps),
+      runningConsumers: this.runningConsumers
+    };
   }
 };
 
-// src/client/topic.ts
+// src/client/message/topic.ts
 function topic(name) {
-  const fn = () => ({
-    __topic: name,
-    __type: void 0
-  });
-  fn.schema = (schema) => ({
-    __topic: name,
-    __type: void 0,
-    __schema: schema
-  });
-  return fn;
+  return {
+    /** Provide an explicit message type without a runtime schema. */
+    type: () => ({
+      __topic: name,
+      __type: void 0
+    }),
+    schema: (schema) => ({
+      __topic: name,
+      __type: void 0,
+      __schema: schema
+    })
+  };
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {