@drarzter/kafka-client 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/README.md +70 -2
  2. package/dist/{chunk-CMO7SMVK.mjs → chunk-OR7TPAAE.mjs} +110 -164
  3. package/dist/chunk-OR7TPAAE.mjs.map +1 -0
  4. package/dist/chunk-PQVBRDNV.mjs +149 -0
  5. package/dist/chunk-PQVBRDNV.mjs.map +1 -0
  6. package/dist/cli/index.js +115 -51
  7. package/dist/cli/index.js.map +1 -1
  8. package/dist/cli/index.mjs +2 -1
  9. package/dist/cli/index.mjs.map +1 -1
  10. package/dist/client/kafka.client/consumer/features/delayed.d.ts.map +1 -1
  11. package/dist/client/kafka.client/consumer/features/dlq-replay.d.ts +1 -1
  12. package/dist/client/kafka.client/consumer/features/dlq-replay.d.ts.map +1 -1
  13. package/dist/client/kafka.client/consumer/features/snapshot.d.ts.map +1 -1
  14. package/dist/client/kafka.client/consumer/handler.d.ts +16 -2
  15. package/dist/client/kafka.client/consumer/handler.d.ts.map +1 -1
  16. package/dist/client/kafka.client/consumer/ops.d.ts +13 -0
  17. package/dist/client/kafka.client/consumer/ops.d.ts.map +1 -1
  18. package/dist/client/kafka.client/consumer/pipeline.d.ts +14 -13
  19. package/dist/client/kafka.client/consumer/pipeline.d.ts.map +1 -1
  20. package/dist/client/kafka.client/consumer/retry-topic.d.ts +4 -1
  21. package/dist/client/kafka.client/consumer/retry-topic.d.ts.map +1 -1
  22. package/dist/client/kafka.client/consumer/setup.d.ts +3 -0
  23. package/dist/client/kafka.client/consumer/setup.d.ts.map +1 -1
  24. package/dist/client/kafka.client/consumer/start.d.ts.map +1 -1
  25. package/dist/client/kafka.client/context.d.ts +3 -0
  26. package/dist/client/kafka.client/context.d.ts.map +1 -1
  27. package/dist/client/kafka.client/index.d.ts.map +1 -1
  28. package/dist/client/kafka.client/producer/ops.d.ts +12 -3
  29. package/dist/client/kafka.client/producer/ops.d.ts.map +1 -1
  30. package/dist/client/kafka.client/producer/send.d.ts +1 -1
  31. package/dist/client/message/schema-registry.d.ts +23 -4
  32. package/dist/client/message/schema-registry.d.ts.map +1 -1
  33. package/dist/client/message/serde.d.ts +68 -0
  34. package/dist/client/message/serde.d.ts.map +1 -0
  35. package/dist/client/message/topic.d.ts +25 -4
  36. package/dist/client/message/topic.d.ts.map +1 -1
  37. package/dist/client/transport/transport.interface.d.ts +6 -1
  38. package/dist/client/transport/transport.interface.d.ts.map +1 -1
  39. package/dist/client/types/config.types.d.ts +17 -0
  40. package/dist/client/types/config.types.d.ts.map +1 -1
  41. package/dist/core.d.ts +3 -0
  42. package/dist/core.d.ts.map +1 -1
  43. package/dist/core.js +146 -55
  44. package/dist/core.js.map +1 -1
  45. package/dist/core.mjs +9 -3
  46. package/dist/index.js +146 -55
  47. package/dist/index.js.map +1 -1
  48. package/dist/index.mjs +9 -3
  49. package/dist/index.mjs.map +1 -1
  50. package/dist/serde.d.ts +157 -0
  51. package/dist/serde.d.ts.map +1 -0
  52. package/dist/serde.js +308 -0
  53. package/dist/serde.js.map +1 -0
  54. package/dist/serde.mjs +158 -0
  55. package/dist/serde.mjs.map +1 -0
  56. package/package.json +20 -1
  57. package/dist/chunk-CMO7SMVK.mjs.map +0 -1
package/dist/core.mjs CHANGED
@@ -1,4 +1,5 @@
1
1
  import {
2
+ ConfluentTransport,
2
3
  HEADER_CORRELATION_ID,
3
4
  HEADER_DELAYED_TARGET,
4
5
  HEADER_DELAYED_UNTIL,
@@ -13,7 +14,6 @@ import {
13
14
  KafkaProcessingError,
14
15
  KafkaRetryExhaustedError,
15
16
  KafkaValidationError,
16
- SchemaRegistryClient,
17
17
  awsMskIamProvider,
18
18
  buildEnvelopeHeaders,
19
19
  consumerOptionsFromEnv,
@@ -24,7 +24,6 @@ import {
24
24
  getEnvelopeContext,
25
25
  kafkaClientConfigFromEnv,
26
26
  mergeConsumerOptions,
27
- registrySchema,
28
27
  resolveSecurityOptions,
29
28
  runWithEnvelopeContext,
30
29
  startOutboxRelay,
@@ -33,9 +32,15 @@ import {
33
32
  toMskIamPolicy,
34
33
  topic,
35
34
  versionedSchema
36
- } from "./chunk-CMO7SMVK.mjs";
35
+ } from "./chunk-OR7TPAAE.mjs";
36
+ import {
37
+ JsonSerde,
38
+ SchemaRegistryClient,
39
+ registrySchema
40
+ } from "./chunk-PQVBRDNV.mjs";
37
41
  import "./chunk-EQQGB2QZ.mjs";
38
42
  export {
43
+ ConfluentTransport,
39
44
  HEADER_CORRELATION_ID,
40
45
  HEADER_DELAYED_TARGET,
41
46
  HEADER_DELAYED_UNTIL,
@@ -46,6 +51,7 @@ export {
46
51
  HEADER_TRACEPARENT,
47
52
  InMemoryDedupStore,
48
53
  InMemoryOutboxStore,
54
+ JsonSerde,
49
55
  KafkaClient,
50
56
  KafkaProcessingError,
51
57
  KafkaRetryExhaustedError,
package/dist/index.js CHANGED
@@ -29,6 +29,7 @@ var __decorateParam = (index, decorator) => (target, key) => decorator(target, k
29
29
  // src/index.ts
30
30
  var index_exports = {};
31
31
  __export(index_exports, {
32
+ ConfluentTransport: () => ConfluentTransport,
32
33
  HEADER_CORRELATION_ID: () => HEADER_CORRELATION_ID,
33
34
  HEADER_DELAYED_TARGET: () => HEADER_DELAYED_TARGET,
34
35
  HEADER_DELAYED_UNTIL: () => HEADER_DELAYED_UNTIL,
@@ -40,6 +41,7 @@ __export(index_exports, {
40
41
  InMemoryDedupStore: () => InMemoryDedupStore,
41
42
  InMemoryOutboxStore: () => InMemoryOutboxStore,
42
43
  InjectKafkaClient: () => InjectKafkaClient,
44
+ JsonSerde: () => JsonSerde,
43
45
  KAFKA_CLIENT: () => KAFKA_CLIENT,
44
46
  KAFKA_SUBSCRIBER_METADATA: () => KAFKA_SUBSCRIBER_METADATA,
45
47
  KafkaClient: () => KafkaClient,
@@ -277,6 +279,18 @@ var ConfluentTransport = class {
277
279
  }
278
280
  };
279
281
 
282
+ // src/client/message/serde.ts
283
+ var JsonSerde = class {
284
+ /** JSON-stringify the validated payload. Returns a UTF-8 string. */
285
+ serialize(value) {
286
+ return JSON.stringify(value);
287
+ }
288
+ /** JSON-parse UTF-8 wire bytes into an object. */
289
+ deserialize(data) {
290
+ return JSON.parse(data.toString("utf8"));
291
+ }
292
+ };
293
+
280
294
  // src/client/kafka.client/infra/dedup.store.ts
281
295
  var InMemoryDedupStore = class {
282
296
  constructor(states) {
@@ -401,6 +415,9 @@ var KafkaRetryExhaustedError = class extends KafkaProcessingError {
401
415
  };
402
416
 
403
417
  // src/client/kafka.client/producer/ops.ts
418
+ function resolveSerde(topicOrDesc, clientSerde) {
419
+ return topicOrDesc?.__serde ?? clientSerde;
420
+ }
404
421
  function resolveTopicName(topicOrDescriptor) {
405
422
  if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
406
423
  if (topicOrDescriptor && typeof topicOrDescriptor === "object" && "__topic" in topicOrDescriptor) {
@@ -447,6 +464,7 @@ async function validateMessage(topicOrDesc, message, deps, ctx) {
447
464
  }
448
465
  async function buildSendPayload(topicOrDesc, messages, deps, compression) {
449
466
  const topic2 = resolveTopicName(topicOrDesc);
467
+ const serde = resolveSerde(topicOrDesc, deps.serde);
450
468
  const builtMessages = await Promise.all(
451
469
  messages.map(async (m) => {
452
470
  const envelopeHeaders = buildEnvelopeHeaders({
@@ -466,10 +484,13 @@ async function buildSendPayload(topicOrDesc, messages, deps, compression) {
466
484
  headers: envelopeHeaders,
467
485
  version: m.schemaVersion ?? 1
468
486
  };
487
+ const validated = await validateMessage(topicOrDesc, m.value, deps, sendCtx);
469
488
  return {
470
- value: JSON.stringify(
471
- await validateMessage(topicOrDesc, m.value, deps, sendCtx)
472
- ),
489
+ value: await serde.serialize(validated, {
490
+ topic: topic2,
491
+ headers: envelopeHeaders,
492
+ isKey: false
493
+ }),
473
494
  // Explicit key wins; otherwise fall back to the descriptor's .key()
474
495
  // extractor (runs on the original, pre-validation payload).
475
496
  key: m.key ?? topicOrDesc?.__key?.(m.value) ?? null,
@@ -554,6 +575,15 @@ function buildSchemaMap(topics, schemaRegistry, optionSchemas, logger) {
554
575
  }
555
576
  return schemaMap;
556
577
  }
578
+ function buildSerdeMap(topics) {
579
+ let serdeMap;
580
+ for (const t of topics) {
581
+ if (t?.__serde) {
582
+ (serdeMap ??= /* @__PURE__ */ new Map()).set(resolveTopicName(t), t.__serde);
583
+ }
584
+ }
585
+ return serdeMap;
586
+ }
557
587
 
558
588
  // src/client/kafka.client/admin/ops.ts
559
589
  var AdminOps = class {
@@ -839,17 +869,6 @@ var AdminOps = class {
839
869
  function sleep(ms) {
840
870
  return new Promise((resolve) => setTimeout(resolve, ms));
841
871
  }
842
- function parseJsonMessage(raw, topic2, logger) {
843
- try {
844
- return JSON.parse(raw);
845
- } catch (error) {
846
- logger.error(
847
- `Failed to parse message from topic ${topic2}:`,
848
- toError(error).stack
849
- );
850
- return null;
851
- }
852
- }
853
872
  async function validateWithSchema(message, raw, topic2, schemaMap, interceptors, dlq, deps) {
854
873
  const schema = schemaMap.get(topic2);
855
874
  if (!schema) return message;
@@ -1242,15 +1261,15 @@ async function replayDlqTopic(topic2, deps, options = {}) {
1242
1261
  const originalHeaders = Object.fromEntries(
1243
1262
  Object.entries(headers).filter(([k]) => !deps.dlqHeaderKeys.has(k))
1244
1263
  );
1245
- const value = message.value.toString();
1246
- const shouldProcess = !options.filter || options.filter(headers, value);
1264
+ const bytes = message.value;
1265
+ const shouldProcess = !options.filter || options.filter(headers, bytes.toString("utf8"));
1247
1266
  if (!targetTopic || !shouldProcess) {
1248
1267
  skipped++;
1249
1268
  } else if (options.dryRun) {
1250
1269
  deps.logger.log(`[DLQ replay dry-run] Would replay to "${targetTopic}"`);
1251
1270
  replayed++;
1252
1271
  } else {
1253
- await deps.send(targetTopic, [{ value, headers: originalHeaders }]);
1272
+ await deps.send(targetTopic, [{ value: bytes, headers: originalHeaders }]);
1254
1273
  replayed++;
1255
1274
  }
1256
1275
  const allDone = Array.from(highWatermarks.entries()).every(
@@ -2124,7 +2143,7 @@ async function waitForPartitionAssignment(consumer, topics, logger, timeoutMs =
2124
2143
  `Retry consumer did not receive partition assignments for [${topics.join(", ")}] within ${timeoutMs}ms`
2125
2144
  );
2126
2145
  }
2127
- async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopics, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs) {
2146
+ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopics, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs, serdeMap) {
2128
2147
  const {
2129
2148
  logger,
2130
2149
  producer,
@@ -2170,20 +2189,35 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
2170
2189
  await sleep(remaining);
2171
2190
  consumer.resume([{ topic: levelTopic, partitions: [partition] }]);
2172
2191
  }
2173
- const raw = message.value.toString();
2174
- const parsed = parseJsonMessage(raw, levelTopic, logger);
2175
- if (parsed === null) {
2176
- await consumer.commitOffsets([nextOffset]);
2177
- return;
2178
- }
2192
+ const rawBytes = message.value;
2179
2193
  const currentMaxRetries = parseInt(
2180
2194
  headers[RETRY_HEADER_MAX_RETRIES] ?? String(retry.maxRetries),
2181
2195
  10
2182
2196
  );
2183
2197
  const originalTopic = headers[RETRY_HEADER_ORIGINAL_TOPIC] ?? levelTopic.replace(/\.retry\.\d+$/, "");
2198
+ const serde = serdeMap?.get(originalTopic) ?? deps.serde;
2199
+ let parsed;
2200
+ try {
2201
+ parsed = await serde.deserialize(rawBytes, {
2202
+ topic: originalTopic,
2203
+ headers,
2204
+ isKey: false
2205
+ });
2206
+ } catch (err) {
2207
+ logger.error(
2208
+ `Failed to deserialize retry message from topic ${levelTopic}:`,
2209
+ toError(err).stack
2210
+ );
2211
+ await consumer.commitOffsets([nextOffset]);
2212
+ return;
2213
+ }
2214
+ if (parsed === null) {
2215
+ await consumer.commitOffsets([nextOffset]);
2216
+ return;
2217
+ }
2184
2218
  const validated = await validateWithSchema(
2185
2219
  parsed,
2186
- raw,
2220
+ rawBytes,
2187
2221
  originalTopic,
2188
2222
  schemaMap,
2189
2223
  interceptors,
@@ -2237,7 +2271,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
2237
2271
  const delay = Math.floor(Math.random() * cap);
2238
2272
  const { topic: rtTopic, messages: rtMsgs } = buildRetryTopicPayload(
2239
2273
  originalTopic,
2240
- [raw],
2274
+ [rawBytes],
2241
2275
  nextLevel,
2242
2276
  currentMaxRetries,
2243
2277
  delay,
@@ -2279,7 +2313,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
2279
2313
  } else if (dlq) {
2280
2314
  const { topic: dTopic, messages: dMsgs } = buildDlqPayload(
2281
2315
  originalTopic,
2282
- raw,
2316
+ rawBytes,
2283
2317
  {
2284
2318
  error,
2285
2319
  // +1 to account for the main consumer's initial attempt before routing.
@@ -2341,7 +2375,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
2341
2375
  `Retry level ${level}/${retry.maxRetries} consumer started for: ${originalTopics.join(", ")} (group: ${levelGroupId})`
2342
2376
  );
2343
2377
  }
2344
- async function startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs) {
2378
+ async function startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs, serdeMap) {
2345
2379
  const levelGroupIds = new Array(retry.maxRetries);
2346
2380
  await Promise.all(
2347
2381
  Array.from({ length: retry.maxRetries }, async (_, i) => {
@@ -2359,7 +2393,8 @@ async function startRetryTopicConsumers(originalTopics, originalGroupId, handleM
2359
2393
  interceptors,
2360
2394
  schemaMap,
2361
2395
  deps,
2362
- assignmentTimeoutMs
2396
+ assignmentTimeoutMs,
2397
+ serdeMap
2363
2398
  );
2364
2399
  levelGroupIds[i] = levelGroupId;
2365
2400
  })
@@ -2443,6 +2478,7 @@ async function setupConsumer(ctx, topics, mode, options) {
2443
2478
  optionSchemas,
2444
2479
  ctx.logger
2445
2480
  );
2481
+ const serdeMap = buildSerdeMap(stringTopics);
2446
2482
  const topicNames = stringTopics.map((t) => resolveTopicName(t));
2447
2483
  const subscribeTopics = [...topicNames, ...regexTopics];
2448
2484
  await ensureConsumerTopics(ctx, topicNames, dlq, options.deduplication);
@@ -2460,7 +2496,7 @@ async function setupConsumer(ctx, topics, mode, options) {
2460
2496
  ctx.logger.log(
2461
2497
  `${mode === "eachBatch" ? "Batch consumer" : "Consumer"} subscribed to topics: ${displayTopics}`
2462
2498
  );
2463
- return { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, hasRegex, readyPromise };
2499
+ return { consumer, schemaMap, serdeMap, topicNames, gid, dlq, interceptors, retry, hasRegex, readyPromise };
2464
2500
  }
2465
2501
  function resolveDeduplicationContext(ctx, groupId, options) {
2466
2502
  if (!options) return void 0;
@@ -2472,6 +2508,7 @@ function messageDepsFor(ctx, gid, options) {
2472
2508
  return {
2473
2509
  logger: ctx.logger,
2474
2510
  producer: ctx.producer,
2511
+ serde: ctx.serde,
2475
2512
  instrumentation: ctx.instrumentation,
2476
2513
  onMessageLost: options?.onMessageLost ?? ctx.onMessageLost,
2477
2514
  onTtlExpired: ctx.onTtlExpired,
@@ -2489,6 +2526,7 @@ function buildRetryTopicDeps(ctx) {
2489
2526
  return {
2490
2527
  logger: ctx.logger,
2491
2528
  producer: ctx.producer,
2529
+ serde: ctx.serde,
2492
2530
  instrumentation: ctx.instrumentation,
2493
2531
  onMessageLost: ctx.onMessageLost,
2494
2532
  onRetry: ctx.metrics.notifyRetry.bind(ctx.metrics),
@@ -2506,7 +2544,7 @@ async function makeEosMainContext(ctx, gid, consumer, options) {
2506
2544
  return { txProducer, consumer };
2507
2545
  }
2508
2546
  async function launchRetryChain(ctx, gid, topicNames, handleMessage, opts) {
2509
- const { retry, dlq, interceptors, schemaMap, assignmentTimeoutMs } = opts;
2547
+ const { retry, dlq, interceptors, schemaMap, serdeMap, assignmentTimeoutMs } = opts;
2510
2548
  if (!ctx.autoCreateTopicsEnabled) {
2511
2549
  await ctx.adminOps.validateRetryTopicsExist(topicNames, retry.maxRetries);
2512
2550
  }
@@ -2530,7 +2568,8 @@ async function launchRetryChain(ctx, gid, topicNames, handleMessage, opts) {
2530
2568
  ctx.companionGroupIds.get(gid).push(levelGroupId);
2531
2569
  }
2532
2570
  },
2533
- assignmentTimeoutMs
2571
+ assignmentTimeoutMs,
2572
+ serdeMap
2534
2573
  );
2535
2574
  }
2536
2575
 
@@ -2588,18 +2627,29 @@ async function applyDeduplication(envelope, raw, dedup, dlq, deps) {
2588
2627
  }
2589
2628
  return false;
2590
2629
  }
2591
- async function parseSingleMessage(message, topic2, partition, schemaMap, interceptors, dlq, deps) {
2630
+ async function parseSingleMessage(message, topic2, partition, schemaMap, interceptors, dlq, deps, serdeMap) {
2592
2631
  if (!message.value) {
2593
2632
  deps.logger.warn(`Received empty message from topic ${topic2}`);
2594
2633
  return null;
2595
2634
  }
2596
- const raw = message.value.toString();
2597
- const parsed = parseJsonMessage(raw, topic2, deps.logger);
2598
- if (parsed === null) return null;
2635
+ const bytes = message.value;
2599
2636
  const headers = decodeHeaders(message.headers);
2637
+ const serde = serdeMap?.get(topic2) ?? deps.serde;
2638
+ let parsed;
2639
+ try {
2640
+ parsed = await serde.deserialize(bytes, { topic: topic2, headers, isKey: false });
2641
+ } catch (error) {
2642
+ deps.logger.error(
2643
+ `Failed to deserialize message from topic ${topic2}:`,
2644
+ toError(error).stack
2645
+ );
2646
+ return null;
2647
+ }
2648
+ if (parsed === null) return null;
2600
2649
  const validated = await validateWithSchema(
2601
2650
  parsed,
2602
- raw,
2651
+ // Forward the ORIGINAL bytes to DLQ on validation failure (binary-safe).
2652
+ bytes,
2603
2653
  topic2,
2604
2654
  schemaMap,
2605
2655
  interceptors,
@@ -2613,6 +2663,7 @@ async function handleEachMessage(payload, opts, deps) {
2613
2663
  const { topic: topic2, partition, message } = payload;
2614
2664
  const {
2615
2665
  schemaMap,
2666
+ serdeMap,
2616
2667
  handleMessage,
2617
2668
  interceptors,
2618
2669
  dlq,
@@ -2621,6 +2672,7 @@ async function handleEachMessage(payload, opts, deps) {
2621
2672
  timeoutMs,
2622
2673
  wrapWithTimeout
2623
2674
  } = opts;
2675
+ const rawBytes = message.value;
2624
2676
  const eos = opts.eosMainContext;
2625
2677
  const nextOffsetStr = (parseInt(message.offset, 10) + 1).toString();
2626
2678
  const commitOffset = eos ? async () => {
@@ -2665,7 +2717,8 @@ async function handleEachMessage(payload, opts, deps) {
2665
2717
  schemaMap,
2666
2718
  interceptors,
2667
2719
  dlq,
2668
- deps
2720
+ deps,
2721
+ serdeMap
2669
2722
  );
2670
2723
  if (envelope === null) {
2671
2724
  await commitOffset?.();
@@ -2674,7 +2727,7 @@ async function handleEachMessage(payload, opts, deps) {
2674
2727
  if (opts.deduplication) {
2675
2728
  const isDuplicate = await applyDeduplication(
2676
2729
  envelope,
2677
- message.value.toString(),
2730
+ rawBytes,
2678
2731
  opts.deduplication,
2679
2732
  dlq,
2680
2733
  deps
@@ -2691,7 +2744,7 @@ async function handleEachMessage(payload, opts, deps) {
2691
2744
  `[KafkaClient] TTL expired on ${topic2}: age ${ageMs}ms > ${opts.messageTtlMs}ms`
2692
2745
  );
2693
2746
  if (dlq) {
2694
- await sendToDlq(topic2, message.value.toString(), deps, {
2747
+ await sendToDlq(topic2, rawBytes, deps, {
2695
2748
  error: new Error(`Message TTL expired: age ${ageMs}ms`),
2696
2749
  attempt: 0,
2697
2750
  originalHeaders: envelope.headers
@@ -2723,7 +2776,7 @@ async function handleEachMessage(payload, opts, deps) {
2723
2776
  },
2724
2777
  {
2725
2778
  envelope,
2726
- rawMessages: [message.value.toString()],
2779
+ rawMessages: [rawBytes],
2727
2780
  interceptors,
2728
2781
  dlq,
2729
2782
  retry,
@@ -2736,6 +2789,7 @@ async function handleEachBatch(payload, opts, deps) {
2736
2789
  const { batch, heartbeat, resolveOffset, commitOffsetsIfNecessary } = payload;
2737
2790
  const {
2738
2791
  schemaMap,
2792
+ serdeMap,
2739
2793
  handleBatch,
2740
2794
  interceptors,
2741
2795
  dlq,
@@ -2791,6 +2845,7 @@ async function handleEachBatch(payload, opts, deps) {
2791
2845
  const envelopes = [];
2792
2846
  const rawMessages = [];
2793
2847
  for (const message of batch.messages) {
2848
+ const rawBytes = message.value;
2794
2849
  const envelope = await parseSingleMessage(
2795
2850
  message,
2796
2851
  batch.topic,
@@ -2798,14 +2853,14 @@ async function handleEachBatch(payload, opts, deps) {
2798
2853
  schemaMap,
2799
2854
  interceptors,
2800
2855
  dlq,
2801
- deps
2856
+ deps,
2857
+ serdeMap
2802
2858
  );
2803
2859
  if (envelope === null) continue;
2804
2860
  if (opts.deduplication) {
2805
- const raw = message.value.toString();
2806
2861
  const isDuplicate = await applyDeduplication(
2807
2862
  envelope,
2808
- raw,
2863
+ rawBytes,
2809
2864
  opts.deduplication,
2810
2865
  dlq,
2811
2866
  deps
@@ -2819,7 +2874,7 @@ async function handleEachBatch(payload, opts, deps) {
2819
2874
  `[KafkaClient] TTL expired on ${batch.topic}: age ${ageMs}ms > ${opts.messageTtlMs}ms`
2820
2875
  );
2821
2876
  if (dlq) {
2822
- await sendToDlq(batch.topic, message.value.toString(), deps, {
2877
+ await sendToDlq(batch.topic, rawBytes, deps, {
2823
2878
  error: new Error(`Message TTL expired: age ${ageMs}ms`),
2824
2879
  attempt: 0,
2825
2880
  originalHeaders: envelope.headers
@@ -2838,7 +2893,7 @@ async function handleEachBatch(payload, opts, deps) {
2838
2893
  }
2839
2894
  }
2840
2895
  envelopes.push(envelope);
2841
- rawMessages.push(message.value.toString());
2896
+ rawMessages.push(rawBytes);
2842
2897
  }
2843
2898
  if (envelopes.length === 0) {
2844
2899
  await commitBatchOffset?.();
@@ -3001,7 +3056,7 @@ function resumeTopicAllPartitions(ctx, gid, topic2) {
3001
3056
  async function startConsumerImpl(ctx, topics, handleMessage, options = {}) {
3002
3057
  validateTopicConsumerOpts(topics, options);
3003
3058
  const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
3004
- const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachMessage", setupOptions);
3059
+ const { consumer, schemaMap, serdeMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachMessage", setupOptions);
3005
3060
  if (options.circuitBreaker)
3006
3061
  ctx.circuitBreaker.setConfig(gid, options.circuitBreaker);
3007
3062
  const deps = messageDepsFor(ctx, gid, options);
@@ -3012,6 +3067,7 @@ async function startConsumerImpl(ctx, topics, handleMessage, options = {}) {
3012
3067
  payload,
3013
3068
  {
3014
3069
  schemaMap,
3070
+ serdeMap,
3015
3071
  handleMessage,
3016
3072
  interceptors,
3017
3073
  dlq,
@@ -3039,6 +3095,7 @@ async function startConsumerImpl(ctx, topics, handleMessage, options = {}) {
3039
3095
  dlq,
3040
3096
  interceptors,
3041
3097
  schemaMap,
3098
+ serdeMap,
3042
3099
  assignmentTimeoutMs: options.retryTopicAssignmentTimeoutMs
3043
3100
  });
3044
3101
  }
@@ -3052,7 +3109,7 @@ async function startBatchConsumerImpl(ctx, topics, handleBatch, options = {}) {
3052
3109
  );
3053
3110
  }
3054
3111
  const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
3055
- const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachBatch", setupOptions);
3112
+ const { consumer, schemaMap, serdeMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachBatch", setupOptions);
3056
3113
  if (options.circuitBreaker)
3057
3114
  ctx.circuitBreaker.setConfig(gid, options.circuitBreaker);
3058
3115
  const deps = messageDepsFor(ctx, gid, options);
@@ -3063,6 +3120,7 @@ async function startBatchConsumerImpl(ctx, topics, handleBatch, options = {}) {
3063
3120
  payload,
3064
3121
  {
3065
3122
  schemaMap,
3123
+ serdeMap,
3066
3124
  handleBatch,
3067
3125
  interceptors,
3068
3126
  dlq,
@@ -3100,6 +3158,7 @@ async function startBatchConsumerImpl(ctx, topics, handleBatch, options = {}) {
3100
3158
  dlq,
3101
3159
  interceptors,
3102
3160
  schemaMap,
3161
+ serdeMap,
3103
3162
  assignmentTimeoutMs: options.retryTopicAssignmentTimeoutMs
3104
3163
  });
3105
3164
  }
@@ -3112,7 +3171,7 @@ async function startTransactionalConsumerImpl(ctx, topics, handler, options = {}
3112
3171
  );
3113
3172
  }
3114
3173
  const setupOptions = { ...options, autoCommit: false };
3115
- const { consumer, schemaMap, gid, readyPromise } = await setupConsumer(
3174
+ const { consumer, schemaMap, serdeMap, gid, readyPromise } = await setupConsumer(
3116
3175
  ctx,
3117
3176
  topics,
3118
3177
  "eachMessage",
@@ -3129,7 +3188,8 @@ async function startTransactionalConsumerImpl(ctx, topics, handler, options = {}
3129
3188
  schemaMap,
3130
3189
  options.interceptors ?? [],
3131
3190
  false,
3132
- deps
3191
+ deps,
3192
+ serdeMap
3133
3193
  );
3134
3194
  const nextOffset = String(Number.parseInt(message.offset, 10) + 1);
3135
3195
  if (envelope === null) {
@@ -3347,7 +3407,9 @@ async function startDelayedRelayImpl(ctx, topics, options) {
3347
3407
  topic: target,
3348
3408
  messages: [
3349
3409
  {
3350
- value: message.value.toString(),
3410
+ // Forward the ORIGINAL wire bytes unchanged — no re-serialization,
3411
+ // so binary payloads (Avro/Protobuf) are relayed losslessly.
3412
+ value: message.value,
3351
3413
  key: message.key ? message.key.toString() : null,
3352
3414
  headers: forwardHeaders
3353
3415
  }
@@ -3674,6 +3736,7 @@ var KafkaClient = class {
3674
3736
  const consumers = /* @__PURE__ */ new Map();
3675
3737
  const consumerCreationOptions = /* @__PURE__ */ new Map();
3676
3738
  const schemaRegistry = /* @__PURE__ */ new Map();
3739
+ const serde = options?.serde ?? new JsonSerde();
3677
3740
  const adminOps = new AdminOps({
3678
3741
  admin: transport.admin(),
3679
3742
  logger,
@@ -3700,6 +3763,7 @@ var KafkaClient = class {
3700
3763
  autoCreateTopicsEnabled: options?.autoCreateTopics ?? false,
3701
3764
  strictSchemasEnabled: options?.strictSchemas ?? true,
3702
3765
  numPartitions: options?.numPartitions ?? 1,
3766
+ serde,
3703
3767
  txId: options?.transactionalId ?? `${clientId}-tx`,
3704
3768
  clockRecoveryTopics: options?.clockRecovery?.topics ?? [],
3705
3769
  clockRecoveryTimeoutMs: options?.clockRecovery?.timeoutMs ?? 3e4,
@@ -3734,6 +3798,7 @@ var KafkaClient = class {
3734
3798
  strictSchemasEnabled: options?.strictSchemas ?? true,
3735
3799
  instrumentation: options?.instrumentation ?? [],
3736
3800
  logger,
3801
+ serde,
3737
3802
  nextLamportClock: () => 0
3738
3803
  // patched below
3739
3804
  },
@@ -3752,6 +3817,7 @@ var KafkaClient = class {
3752
3817
  strictSchemasEnabled: options?.strictSchemas ?? true,
3753
3818
  instrumentation: options?.instrumentation ?? [],
3754
3819
  logger,
3820
+ serde,
3755
3821
  nextLamportClock: () => ++ctx._lamportClock
3756
3822
  };
3757
3823
  ctx.retryTopicDeps = buildRetryTopicDeps(ctx);
@@ -4020,10 +4086,8 @@ function topic(name) {
4020
4086
  function keyable(desc) {
4021
4087
  return {
4022
4088
  ...desc,
4023
- key: (extractor) => ({
4024
- ...desc,
4025
- __key: extractor
4026
- })
4089
+ key: (extractor) => keyable({ ...desc, __key: extractor }),
4090
+ serde: (serde) => keyable({ ...desc, __serde: serde })
4027
4091
  };
4028
4092
  }
4029
4093
 
@@ -4068,6 +4132,12 @@ var SchemaRegistryClient = class {
4068
4132
  fetchFn;
4069
4133
  cacheTtlMs;
4070
4134
  latestCache = /* @__PURE__ */ new Map();
4135
+ /**
4136
+ * `id → schema` cache. Schema ids are immutable in a Confluent-compatible
4137
+ * registry (a given id always maps to the same schema string), so entries
4138
+ * are cached for the lifetime of the client with no TTL.
4139
+ */
4140
+ byIdCache = /* @__PURE__ */ new Map();
4071
4141
  headers() {
4072
4142
  const h = {
4073
4143
  "Content-Type": "application/vnd.schemaregistry.v1+json"
@@ -4109,6 +4179,25 @@ var SchemaRegistryClient = class {
4109
4179
  });
4110
4180
  return value;
4111
4181
  }
4182
+ /**
4183
+ * Fetch a schema by its globally unique registry id (`GET /schemas/ids/{id}`).
4184
+ *
4185
+ * Used by the Avro/Protobuf serdes on the deserialize path: the writer schema
4186
+ * id is read from the Confluent wire-format prefix, then resolved here. Results
4187
+ * are cached forever (schema ids are immutable), so a given id triggers exactly
4188
+ * one registry round-trip regardless of how many messages reference it.
4189
+ */
4190
+ async getSchemaById(id) {
4191
+ const cached = this.byIdCache.get(id);
4192
+ if (cached) return cached;
4193
+ const raw = await this.request(
4194
+ "GET",
4195
+ `/schemas/ids/${id}`
4196
+ );
4197
+ const value = { id, schema: raw.schema, schemaType: raw.schemaType };
4198
+ this.byIdCache.set(id, value);
4199
+ return value;
4200
+ }
4112
4201
  /** Fetch a specific schema version of a subject. */
4113
4202
  async getSchemaVersion(subject, version) {
4114
4203
  const raw = await this.request(
@@ -5050,6 +5139,7 @@ KafkaHealthIndicator = __decorateClass([
5050
5139
  ], KafkaHealthIndicator);
5051
5140
  // Annotate the CommonJS export names for ESM import in node:
5052
5141
  0 && (module.exports = {
5142
+ ConfluentTransport,
5053
5143
  HEADER_CORRELATION_ID,
5054
5144
  HEADER_DELAYED_TARGET,
5055
5145
  HEADER_DELAYED_UNTIL,
@@ -5061,6 +5151,7 @@ KafkaHealthIndicator = __decorateClass([
5061
5151
  InMemoryDedupStore,
5062
5152
  InMemoryOutboxStore,
5063
5153
  InjectKafkaClient,
5154
+ JsonSerde,
5064
5155
  KAFKA_CLIENT,
5065
5156
  KAFKA_SUBSCRIBER_METADATA,
5066
5157
  KafkaClient,