@drarzter/kafka-client 0.10.0 → 0.11.0

package/dist/core.js

@@ -20,6 +20,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/core.ts
 var core_exports = {};
 __export(core_exports, {
+  ConfluentTransport: () => ConfluentTransport,
   HEADER_CORRELATION_ID: () => HEADER_CORRELATION_ID,
   HEADER_DELAYED_TARGET: () => HEADER_DELAYED_TARGET,
   HEADER_DELAYED_UNTIL: () => HEADER_DELAYED_UNTIL,
@@ -30,6 +31,7 @@ __export(core_exports, {
   HEADER_TRACEPARENT: () => HEADER_TRACEPARENT,
   InMemoryDedupStore: () => InMemoryDedupStore,
   InMemoryOutboxStore: () => InMemoryOutboxStore,
+  JsonSerde: () => JsonSerde,
   KafkaClient: () => KafkaClient,
   KafkaProcessingError: () => KafkaProcessingError,
   KafkaRetryExhaustedError: () => KafkaRetryExhaustedError,
@@ -260,6 +262,18 @@ var ConfluentTransport = class {
   }
 };
 
+// src/client/message/serde.ts
+var JsonSerde = class {
+  /** JSON-stringify the validated payload. Returns a UTF-8 string. */
+  serialize(value) {
+    return JSON.stringify(value);
+  }
+  /** JSON-parse UTF-8 wire bytes into an object. */
+  deserialize(data) {
+    return JSON.parse(data.toString("utf8"));
+  }
+};
+
 // src/client/kafka.client/infra/dedup.store.ts
 var InMemoryDedupStore = class {
   constructor(states) {
@@ -384,6 +398,9 @@ var KafkaRetryExhaustedError = class extends KafkaProcessingError {
 };
 
 // src/client/kafka.client/producer/ops.ts
+function resolveSerde(topicOrDesc, clientSerde) {
+  return topicOrDesc?.__serde ?? clientSerde;
+}
 function resolveTopicName(topicOrDescriptor) {
   if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
   if (topicOrDescriptor && typeof topicOrDescriptor === "object" && "__topic" in topicOrDescriptor) {
@@ -430,6 +447,7 @@ async function validateMessage(topicOrDesc, message, deps, ctx) {
 }
 async function buildSendPayload(topicOrDesc, messages, deps, compression) {
   const topic2 = resolveTopicName(topicOrDesc);
+  const serde = resolveSerde(topicOrDesc, deps.serde);
   const builtMessages = await Promise.all(
     messages.map(async (m) => {
       const envelopeHeaders = buildEnvelopeHeaders({
@@ -449,10 +467,13 @@ async function buildSendPayload(topicOrDesc, messages, deps, compression) {
         headers: envelopeHeaders,
         version: m.schemaVersion ?? 1
       };
+      const validated = await validateMessage(topicOrDesc, m.value, deps, sendCtx);
       return {
-        value: JSON.stringify(
-          await validateMessage(topicOrDesc, m.value, deps, sendCtx)
-        ),
+        value: await serde.serialize(validated, {
+          topic: topic2,
+          headers: envelopeHeaders,
+          isKey: false
+        }),
         // Explicit key wins; otherwise fall back to the descriptor's .key()
         // extractor (runs on the original, pre-validation payload).
         key: m.key ?? topicOrDesc?.__key?.(m.value) ?? null,
@@ -537,6 +558,15 @@ function buildSchemaMap(topics, schemaRegistry, optionSchemas, logger) {
   }
   return schemaMap;
 }
+function buildSerdeMap(topics) {
+  let serdeMap;
+  for (const t of topics) {
+    if (t?.__serde) {
+      (serdeMap ??= /* @__PURE__ */ new Map()).set(resolveTopicName(t), t.__serde);
+    }
+  }
+  return serdeMap;
+}
 
 // src/client/kafka.client/admin/ops.ts
 var AdminOps = class {
@@ -822,17 +852,6 @@ var AdminOps = class {
 function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
-function parseJsonMessage(raw, topic2, logger) {
-  try {
-    return JSON.parse(raw);
-  } catch (error) {
-    logger.error(
-      `Failed to parse message from topic ${topic2}:`,
-      toError(error).stack
-    );
-    return null;
-  }
-}
 async function validateWithSchema(message, raw, topic2, schemaMap, interceptors, dlq, deps) {
   const schema = schemaMap.get(topic2);
   if (!schema) return message;
@@ -1225,15 +1244,15 @@ async function replayDlqTopic(topic2, deps, options = {}) {
           const originalHeaders = Object.fromEntries(
             Object.entries(headers).filter(([k]) => !deps.dlqHeaderKeys.has(k))
           );
-          const value = message.value.toString();
-          const shouldProcess = !options.filter || options.filter(headers, value);
+          const bytes = message.value;
+          const shouldProcess = !options.filter || options.filter(headers, bytes.toString("utf8"));
           if (!targetTopic || !shouldProcess) {
             skipped++;
           } else if (options.dryRun) {
             deps.logger.log(`[DLQ replay dry-run] Would replay to "${targetTopic}"`);
             replayed++;
           } else {
-            await deps.send(targetTopic, [{ value, headers: originalHeaders }]);
+            await deps.send(targetTopic, [{ value: bytes, headers: originalHeaders }]);
             replayed++;
           }
           const allDone = Array.from(highWatermarks.entries()).every(
@@ -2107,7 +2126,7 @@ async function waitForPartitionAssignment(consumer, topics, logger, timeoutMs =
     `Retry consumer did not receive partition assignments for [${topics.join(", ")}] within ${timeoutMs}ms`
   );
 }
-async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopics, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs) {
+async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopics, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs, serdeMap) {
   const {
     logger,
     producer,
@@ -2153,20 +2172,35 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
         await sleep(remaining);
         consumer.resume([{ topic: levelTopic, partitions: [partition] }]);
       }
-      const raw = message.value.toString();
-      const parsed = parseJsonMessage(raw, levelTopic, logger);
-      if (parsed === null) {
-        await consumer.commitOffsets([nextOffset]);
-        return;
-      }
+      const rawBytes = message.value;
       const currentMaxRetries = parseInt(
         headers[RETRY_HEADER_MAX_RETRIES] ?? String(retry.maxRetries),
         10
       );
       const originalTopic = headers[RETRY_HEADER_ORIGINAL_TOPIC] ?? levelTopic.replace(/\.retry\.\d+$/, "");
+      const serde = serdeMap?.get(originalTopic) ?? deps.serde;
+      let parsed;
+      try {
+        parsed = await serde.deserialize(rawBytes, {
+          topic: originalTopic,
+          headers,
+          isKey: false
+        });
+      } catch (err) {
+        logger.error(
+          `Failed to deserialize retry message from topic ${levelTopic}:`,
+          toError(err).stack
+        );
+        await consumer.commitOffsets([nextOffset]);
+        return;
+      }
+      if (parsed === null) {
+        await consumer.commitOffsets([nextOffset]);
+        return;
+      }
       const validated = await validateWithSchema(
         parsed,
-        raw,
+        rawBytes,
         originalTopic,
         schemaMap,
         interceptors,
@@ -2220,7 +2254,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
         const delay = Math.floor(Math.random() * cap);
         const { topic: rtTopic, messages: rtMsgs } = buildRetryTopicPayload(
           originalTopic,
-          [raw],
+          [rawBytes],
           nextLevel,
           currentMaxRetries,
           delay,
@@ -2262,7 +2296,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
       } else if (dlq) {
         const { topic: dTopic, messages: dMsgs } = buildDlqPayload(
           originalTopic,
-          raw,
+          rawBytes,
           {
             error,
             // +1 to account for the main consumer's initial attempt before routing.
@@ -2324,7 +2358,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
     `Retry level ${level}/${retry.maxRetries} consumer started for: ${originalTopics.join(", ")} (group: ${levelGroupId})`
   );
 }
-async function startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs) {
+async function startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs, serdeMap) {
   const levelGroupIds = new Array(retry.maxRetries);
   await Promise.all(
     Array.from({ length: retry.maxRetries }, async (_, i) => {
@@ -2342,7 +2376,8 @@ async function startRetryTopicConsumers(originalTopics, originalGroupId, handleM
         interceptors,
         schemaMap,
         deps,
-        assignmentTimeoutMs
+        assignmentTimeoutMs,
+        serdeMap
       );
       levelGroupIds[i] = levelGroupId;
     })
@@ -2426,6 +2461,7 @@ async function setupConsumer(ctx, topics, mode, options) {
     optionSchemas,
     ctx.logger
   );
+  const serdeMap = buildSerdeMap(stringTopics);
   const topicNames = stringTopics.map((t) => resolveTopicName(t));
   const subscribeTopics = [...topicNames, ...regexTopics];
   await ensureConsumerTopics(ctx, topicNames, dlq, options.deduplication);
@@ -2443,7 +2479,7 @@ async function setupConsumer(ctx, topics, mode, options) {
   ctx.logger.log(
     `${mode === "eachBatch" ? "Batch consumer" : "Consumer"} subscribed to topics: ${displayTopics}`
   );
-  return { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, hasRegex, readyPromise };
+  return { consumer, schemaMap, serdeMap, topicNames, gid, dlq, interceptors, retry, hasRegex, readyPromise };
 }
 function resolveDeduplicationContext(ctx, groupId, options) {
   if (!options) return void 0;
@@ -2455,6 +2491,7 @@ function messageDepsFor(ctx, gid, options) {
   return {
     logger: ctx.logger,
     producer: ctx.producer,
+    serde: ctx.serde,
     instrumentation: ctx.instrumentation,
     onMessageLost: options?.onMessageLost ?? ctx.onMessageLost,
     onTtlExpired: ctx.onTtlExpired,
@@ -2472,6 +2509,7 @@ function buildRetryTopicDeps(ctx) {
   return {
     logger: ctx.logger,
     producer: ctx.producer,
+    serde: ctx.serde,
     instrumentation: ctx.instrumentation,
     onMessageLost: ctx.onMessageLost,
     onRetry: ctx.metrics.notifyRetry.bind(ctx.metrics),
@@ -2489,7 +2527,7 @@ async function makeEosMainContext(ctx, gid, consumer, options) {
   return { txProducer, consumer };
 }
 async function launchRetryChain(ctx, gid, topicNames, handleMessage, opts) {
-  const { retry, dlq, interceptors, schemaMap, assignmentTimeoutMs } = opts;
+  const { retry, dlq, interceptors, schemaMap, serdeMap, assignmentTimeoutMs } = opts;
   if (!ctx.autoCreateTopicsEnabled) {
     await ctx.adminOps.validateRetryTopicsExist(topicNames, retry.maxRetries);
   }
@@ -2513,7 +2551,8 @@ async function launchRetryChain(ctx, gid, topicNames, handleMessage, opts) {
         ctx.companionGroupIds.get(gid).push(levelGroupId);
       }
     },
-    assignmentTimeoutMs
+    assignmentTimeoutMs,
+    serdeMap
   );
 }
 
@@ -2571,18 +2610,29 @@ async function applyDeduplication(envelope, raw, dedup, dlq, deps) {
   }
   return false;
 }
-async function parseSingleMessage(message, topic2, partition, schemaMap, interceptors, dlq, deps) {
+async function parseSingleMessage(message, topic2, partition, schemaMap, interceptors, dlq, deps, serdeMap) {
   if (!message.value) {
     deps.logger.warn(`Received empty message from topic ${topic2}`);
     return null;
   }
-  const raw = message.value.toString();
-  const parsed = parseJsonMessage(raw, topic2, deps.logger);
-  if (parsed === null) return null;
+  const bytes = message.value;
   const headers = decodeHeaders(message.headers);
+  const serde = serdeMap?.get(topic2) ?? deps.serde;
+  let parsed;
+  try {
+    parsed = await serde.deserialize(bytes, { topic: topic2, headers, isKey: false });
+  } catch (error) {
+    deps.logger.error(
+      `Failed to deserialize message from topic ${topic2}:`,
+      toError(error).stack
+    );
+    return null;
+  }
+  if (parsed === null) return null;
   const validated = await validateWithSchema(
     parsed,
-    raw,
+    // Forward the ORIGINAL bytes to DLQ on validation failure (binary-safe).
+    bytes,
     topic2,
     schemaMap,
     interceptors,
@@ -2596,6 +2646,7 @@ async function handleEachMessage(payload, opts, deps) {
   const { topic: topic2, partition, message } = payload;
   const {
     schemaMap,
+    serdeMap,
     handleMessage,
     interceptors,
     dlq,
@@ -2604,6 +2655,7 @@ async function handleEachMessage(payload, opts, deps) {
     timeoutMs,
     wrapWithTimeout
   } = opts;
+  const rawBytes = message.value;
   const eos = opts.eosMainContext;
   const nextOffsetStr = (parseInt(message.offset, 10) + 1).toString();
   const commitOffset = eos ? async () => {
@@ -2648,7 +2700,8 @@ async function handleEachMessage(payload, opts, deps) {
     schemaMap,
     interceptors,
     dlq,
-    deps
+    deps,
+    serdeMap
   );
   if (envelope === null) {
     await commitOffset?.();
@@ -2657,7 +2710,7 @@ async function handleEachMessage(payload, opts, deps) {
   if (opts.deduplication) {
     const isDuplicate = await applyDeduplication(
       envelope,
-      message.value.toString(),
+      rawBytes,
       opts.deduplication,
       dlq,
       deps
@@ -2674,7 +2727,7 @@ async function handleEachMessage(payload, opts, deps) {
         `[KafkaClient] TTL expired on ${topic2}: age ${ageMs}ms > ${opts.messageTtlMs}ms`
       );
       if (dlq) {
-        await sendToDlq(topic2, message.value.toString(), deps, {
+        await sendToDlq(topic2, rawBytes, deps, {
           error: new Error(`Message TTL expired: age ${ageMs}ms`),
           attempt: 0,
           originalHeaders: envelope.headers
@@ -2706,7 +2759,7 @@ async function handleEachMessage(payload, opts, deps) {
     },
     {
       envelope,
-      rawMessages: [message.value.toString()],
+      rawMessages: [rawBytes],
       interceptors,
       dlq,
       retry,
@@ -2719,6 +2772,7 @@ async function handleEachBatch(payload, opts, deps) {
   const { batch, heartbeat, resolveOffset, commitOffsetsIfNecessary } = payload;
   const {
     schemaMap,
+    serdeMap,
     handleBatch,
     interceptors,
     dlq,
@@ -2774,6 +2828,7 @@ async function handleEachBatch(payload, opts, deps) {
   const envelopes = [];
   const rawMessages = [];
   for (const message of batch.messages) {
+    const rawBytes = message.value;
     const envelope = await parseSingleMessage(
       message,
       batch.topic,
@@ -2781,14 +2836,14 @@ async function handleEachBatch(payload, opts, deps) {
       schemaMap,
       interceptors,
       dlq,
-      deps
+      deps,
+      serdeMap
     );
     if (envelope === null) continue;
     if (opts.deduplication) {
-      const raw = message.value.toString();
       const isDuplicate = await applyDeduplication(
         envelope,
-        raw,
+        rawBytes,
         opts.deduplication,
         dlq,
         deps
@@ -2802,7 +2857,7 @@ async function handleEachBatch(payload, opts, deps) {
           `[KafkaClient] TTL expired on ${batch.topic}: age ${ageMs}ms > ${opts.messageTtlMs}ms`
         );
         if (dlq) {
-          await sendToDlq(batch.topic, message.value.toString(), deps, {
+          await sendToDlq(batch.topic, rawBytes, deps, {
             error: new Error(`Message TTL expired: age ${ageMs}ms`),
             attempt: 0,
             originalHeaders: envelope.headers
@@ -2821,7 +2876,7 @@ async function handleEachBatch(payload, opts, deps) {
       }
     }
     envelopes.push(envelope);
-    rawMessages.push(message.value.toString());
+    rawMessages.push(rawBytes);
   }
   if (envelopes.length === 0) {
     await commitBatchOffset?.();
@@ -2984,7 +3039,7 @@ function resumeTopicAllPartitions(ctx, gid, topic2) {
 async function startConsumerImpl(ctx, topics, handleMessage, options = {}) {
   validateTopicConsumerOpts(topics, options);
   const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
-  const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachMessage", setupOptions);
+  const { consumer, schemaMap, serdeMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachMessage", setupOptions);
   if (options.circuitBreaker)
     ctx.circuitBreaker.setConfig(gid, options.circuitBreaker);
   const deps = messageDepsFor(ctx, gid, options);
@@ -2995,6 +3050,7 @@ async function startConsumerImpl(ctx, topics, handleMessage, options = {}) {
         payload,
         {
           schemaMap,
+          serdeMap,
           handleMessage,
           interceptors,
           dlq,
@@ -3022,6 +3078,7 @@ async function startConsumerImpl(ctx, topics, handleMessage, options = {}) {
       dlq,
       interceptors,
       schemaMap,
+      serdeMap,
       assignmentTimeoutMs: options.retryTopicAssignmentTimeoutMs
     });
   }
@@ -3035,7 +3092,7 @@ async function startBatchConsumerImpl(ctx, topics, handleBatch, options = {}) {
     );
   }
   const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
-  const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachBatch", setupOptions);
+  const { consumer, schemaMap, serdeMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachBatch", setupOptions);
   if (options.circuitBreaker)
     ctx.circuitBreaker.setConfig(gid, options.circuitBreaker);
   const deps = messageDepsFor(ctx, gid, options);
@@ -3046,6 +3103,7 @@ async function startBatchConsumerImpl(ctx, topics, handleBatch, options = {}) {
         payload,
         {
           schemaMap,
+          serdeMap,
           handleBatch,
           interceptors,
           dlq,
@@ -3083,6 +3141,7 @@ async function startBatchConsumerImpl(ctx, topics, handleBatch, options = {}) {
       dlq,
       interceptors,
       schemaMap,
+      serdeMap,
       assignmentTimeoutMs: options.retryTopicAssignmentTimeoutMs
     });
   }
@@ -3095,7 +3154,7 @@ async function startTransactionalConsumerImpl(ctx, topics, handler, options = {}
     );
   }
   const setupOptions = { ...options, autoCommit: false };
-  const { consumer, schemaMap, gid, readyPromise } = await setupConsumer(
+  const { consumer, schemaMap, serdeMap, gid, readyPromise } = await setupConsumer(
     ctx,
     topics,
     "eachMessage",
@@ -3112,7 +3171,8 @@ async function startTransactionalConsumerImpl(ctx, topics, handler, options = {}
         schemaMap,
         options.interceptors ?? [],
         false,
-        deps
+        deps,
+        serdeMap
       );
       const nextOffset = String(Number.parseInt(message.offset, 10) + 1);
       if (envelope === null) {
@@ -3330,7 +3390,9 @@ async function startDelayedRelayImpl(ctx, topics, options) {
           topic: target,
           messages: [
             {
-              value: message.value.toString(),
+              // Forward the ORIGINAL wire bytes unchanged — no re-serialization,
+              // so binary payloads (Avro/Protobuf) are relayed losslessly.
+              value: message.value,
               key: message.key ? message.key.toString() : null,
               headers: forwardHeaders
             }
@@ -3657,6 +3719,7 @@ var KafkaClient = class {
     const consumers = /* @__PURE__ */ new Map();
     const consumerCreationOptions = /* @__PURE__ */ new Map();
     const schemaRegistry = /* @__PURE__ */ new Map();
+    const serde = options?.serde ?? new JsonSerde();
     const adminOps = new AdminOps({
       admin: transport.admin(),
       logger,
@@ -3683,6 +3746,7 @@ var KafkaClient = class {
       autoCreateTopicsEnabled: options?.autoCreateTopics ?? false,
       strictSchemasEnabled: options?.strictSchemas ?? true,
       numPartitions: options?.numPartitions ?? 1,
+      serde,
       txId: options?.transactionalId ?? `${clientId}-tx`,
       clockRecoveryTopics: options?.clockRecovery?.topics ?? [],
       clockRecoveryTimeoutMs: options?.clockRecovery?.timeoutMs ?? 3e4,
@@ -3717,6 +3781,7 @@ var KafkaClient = class {
         strictSchemasEnabled: options?.strictSchemas ?? true,
         instrumentation: options?.instrumentation ?? [],
         logger,
+        serde,
         nextLamportClock: () => 0
         // patched below
       },
@@ -3735,6 +3800,7 @@ var KafkaClient = class {
       strictSchemasEnabled: options?.strictSchemas ?? true,
       instrumentation: options?.instrumentation ?? [],
       logger,
+      serde,
       nextLamportClock: () => ++ctx._lamportClock
     };
     ctx.retryTopicDeps = buildRetryTopicDeps(ctx);
@@ -4003,10 +4069,8 @@ function topic(name) {
 function keyable(desc) {
   return {
     ...desc,
-    key: (extractor) => ({
-      ...desc,
-      __key: extractor
-    })
+    key: (extractor) => keyable({ ...desc, __key: extractor }),
+    serde: (serde) => keyable({ ...desc, __serde: serde })
   };
 }
 
@@ -4051,6 +4115,12 @@ var SchemaRegistryClient = class {
   fetchFn;
   cacheTtlMs;
   latestCache = /* @__PURE__ */ new Map();
+  /**
+   * `id → schema` cache. Schema ids are immutable in a Confluent-compatible
+   * registry (a given id always maps to the same schema string), so entries
+   * are cached for the lifetime of the client with no TTL.
+   */
+  byIdCache = /* @__PURE__ */ new Map();
   headers() {
     const h = {
       "Content-Type": "application/vnd.schemaregistry.v1+json"
@@ -4092,6 +4162,25 @@ var SchemaRegistryClient = class {
     });
     return value;
   }
+  /**
+   * Fetch a schema by its globally unique registry id (`GET /schemas/ids/{id}`).
+   *
+   * Used by the Avro/Protobuf serdes on the deserialize path: the writer schema
+   * id is read from the Confluent wire-format prefix, then resolved here. Results
+   * are cached forever (schema ids are immutable), so a given id triggers exactly
+   * one registry round-trip regardless of how many messages reference it.
+   */
+  async getSchemaById(id) {
+    const cached = this.byIdCache.get(id);
+    if (cached) return cached;
+    const raw = await this.request(
+      "GET",
+      `/schemas/ids/${id}`
+    );
+    const value = { id, schema: raw.schema, schemaType: raw.schemaType };
+    this.byIdCache.set(id, value);
+    return value;
+  }
   /** Fetch a specific schema version of a subject. */
   async getSchemaVersion(subject, version) {
     const raw = await this.request(
@@ -4834,6 +4923,7 @@ function mergeConsumerOptions(...layers) {
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  ConfluentTransport,
   HEADER_CORRELATION_ID,
   HEADER_DELAYED_TARGET,
   HEADER_DELAYED_UNTIL,
@@ -4844,6 +4934,7 @@ function mergeConsumerOptions(...layers) {
   HEADER_TRACEPARENT,
   InMemoryDedupStore,
   InMemoryOutboxStore,
+  JsonSerde,
   KafkaClient,
   KafkaProcessingError,
   KafkaRetryExhaustedError,