@drarzter/kafka-client 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/README.md +70 -2
  2. package/dist/{chunk-CMO7SMVK.mjs → chunk-OR7TPAAE.mjs} +110 -164
  3. package/dist/chunk-OR7TPAAE.mjs.map +1 -0
  4. package/dist/chunk-PQVBRDNV.mjs +149 -0
  5. package/dist/chunk-PQVBRDNV.mjs.map +1 -0
  6. package/dist/cli/index.js +115 -51
  7. package/dist/cli/index.js.map +1 -1
  8. package/dist/cli/index.mjs +2 -1
  9. package/dist/cli/index.mjs.map +1 -1
  10. package/dist/client/kafka.client/consumer/features/delayed.d.ts.map +1 -1
  11. package/dist/client/kafka.client/consumer/features/dlq-replay.d.ts +1 -1
  12. package/dist/client/kafka.client/consumer/features/dlq-replay.d.ts.map +1 -1
  13. package/dist/client/kafka.client/consumer/features/snapshot.d.ts.map +1 -1
  14. package/dist/client/kafka.client/consumer/handler.d.ts +16 -2
  15. package/dist/client/kafka.client/consumer/handler.d.ts.map +1 -1
  16. package/dist/client/kafka.client/consumer/ops.d.ts +13 -0
  17. package/dist/client/kafka.client/consumer/ops.d.ts.map +1 -1
  18. package/dist/client/kafka.client/consumer/pipeline.d.ts +14 -13
  19. package/dist/client/kafka.client/consumer/pipeline.d.ts.map +1 -1
  20. package/dist/client/kafka.client/consumer/retry-topic.d.ts +4 -1
  21. package/dist/client/kafka.client/consumer/retry-topic.d.ts.map +1 -1
  22. package/dist/client/kafka.client/consumer/setup.d.ts +3 -0
  23. package/dist/client/kafka.client/consumer/setup.d.ts.map +1 -1
  24. package/dist/client/kafka.client/consumer/start.d.ts.map +1 -1
  25. package/dist/client/kafka.client/context.d.ts +3 -0
  26. package/dist/client/kafka.client/context.d.ts.map +1 -1
  27. package/dist/client/kafka.client/index.d.ts.map +1 -1
  28. package/dist/client/kafka.client/producer/ops.d.ts +12 -3
  29. package/dist/client/kafka.client/producer/ops.d.ts.map +1 -1
  30. package/dist/client/kafka.client/producer/send.d.ts +1 -1
  31. package/dist/client/message/schema-registry.d.ts +23 -4
  32. package/dist/client/message/schema-registry.d.ts.map +1 -1
  33. package/dist/client/message/serde.d.ts +68 -0
  34. package/dist/client/message/serde.d.ts.map +1 -0
  35. package/dist/client/message/topic.d.ts +25 -4
  36. package/dist/client/message/topic.d.ts.map +1 -1
  37. package/dist/client/transport/transport.interface.d.ts +6 -1
  38. package/dist/client/transport/transport.interface.d.ts.map +1 -1
  39. package/dist/client/types/config.types.d.ts +17 -0
  40. package/dist/client/types/config.types.d.ts.map +1 -1
  41. package/dist/core.d.ts +3 -0
  42. package/dist/core.d.ts.map +1 -1
  43. package/dist/core.js +146 -55
  44. package/dist/core.js.map +1 -1
  45. package/dist/core.mjs +9 -3
  46. package/dist/index.js +146 -55
  47. package/dist/index.js.map +1 -1
  48. package/dist/index.mjs +9 -3
  49. package/dist/index.mjs.map +1 -1
  50. package/dist/serde.d.ts +157 -0
  51. package/dist/serde.d.ts.map +1 -0
  52. package/dist/serde.js +308 -0
  53. package/dist/serde.js.map +1 -0
  54. package/dist/serde.mjs +158 -0
  55. package/dist/serde.mjs.map +1 -0
  56. package/package.json +20 -1
  57. package/dist/chunk-CMO7SMVK.mjs.map +0 -1
@@ -1,3 +1,7 @@
1
+ import {
2
+ JsonSerde
3
+ } from "./chunk-PQVBRDNV.mjs";
4
+
1
5
  // src/client/transport/confluent.transport.ts
2
6
  import { KafkaJS } from "@confluentinc/kafka-javascript";
3
7
  var { Kafka: KafkaClass, logLevel: KafkaLogLevel, PartitionAssigners } = KafkaJS;
@@ -325,6 +329,9 @@ var KafkaRetryExhaustedError = class extends KafkaProcessingError {
325
329
  };
326
330
 
327
331
  // src/client/kafka.client/producer/ops.ts
332
+ function resolveSerde(topicOrDesc, clientSerde) {
333
+ return topicOrDesc?.__serde ?? clientSerde;
334
+ }
328
335
  function resolveTopicName(topicOrDescriptor) {
329
336
  if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
330
337
  if (topicOrDescriptor && typeof topicOrDescriptor === "object" && "__topic" in topicOrDescriptor) {
@@ -371,6 +378,7 @@ async function validateMessage(topicOrDesc, message, deps, ctx) {
371
378
  }
372
379
  async function buildSendPayload(topicOrDesc, messages, deps, compression) {
373
380
  const topic2 = resolveTopicName(topicOrDesc);
381
+ const serde = resolveSerde(topicOrDesc, deps.serde);
374
382
  const builtMessages = await Promise.all(
375
383
  messages.map(async (m) => {
376
384
  const envelopeHeaders = buildEnvelopeHeaders({
@@ -390,10 +398,13 @@ async function buildSendPayload(topicOrDesc, messages, deps, compression) {
390
398
  headers: envelopeHeaders,
391
399
  version: m.schemaVersion ?? 1
392
400
  };
401
+ const validated = await validateMessage(topicOrDesc, m.value, deps, sendCtx);
393
402
  return {
394
- value: JSON.stringify(
395
- await validateMessage(topicOrDesc, m.value, deps, sendCtx)
396
- ),
403
+ value: await serde.serialize(validated, {
404
+ topic: topic2,
405
+ headers: envelopeHeaders,
406
+ isKey: false
407
+ }),
397
408
  // Explicit key wins; otherwise fall back to the descriptor's .key()
398
409
  // extractor (runs on the original, pre-validation payload).
399
410
  key: m.key ?? topicOrDesc?.__key?.(m.value) ?? null,
@@ -478,6 +489,15 @@ function buildSchemaMap(topics, schemaRegistry, optionSchemas, logger) {
478
489
  }
479
490
  return schemaMap;
480
491
  }
492
+ function buildSerdeMap(topics) {
493
+ let serdeMap;
494
+ for (const t of topics) {
495
+ if (t?.__serde) {
496
+ (serdeMap ??= /* @__PURE__ */ new Map()).set(resolveTopicName(t), t.__serde);
497
+ }
498
+ }
499
+ return serdeMap;
500
+ }
481
501
 
482
502
  // src/client/kafka.client/admin/ops.ts
483
503
  var AdminOps = class {
@@ -763,17 +783,6 @@ var AdminOps = class {
763
783
  function sleep(ms) {
764
784
  return new Promise((resolve) => setTimeout(resolve, ms));
765
785
  }
766
- function parseJsonMessage(raw, topic2, logger) {
767
- try {
768
- return JSON.parse(raw);
769
- } catch (error) {
770
- logger.error(
771
- `Failed to parse message from topic ${topic2}:`,
772
- toError(error).stack
773
- );
774
- return null;
775
- }
776
- }
777
786
  async function validateWithSchema(message, raw, topic2, schemaMap, interceptors, dlq, deps) {
778
787
  const schema = schemaMap.get(topic2);
779
788
  if (!schema) return message;
@@ -1166,15 +1175,15 @@ async function replayDlqTopic(topic2, deps, options = {}) {
1166
1175
  const originalHeaders = Object.fromEntries(
1167
1176
  Object.entries(headers).filter(([k]) => !deps.dlqHeaderKeys.has(k))
1168
1177
  );
1169
- const value = message.value.toString();
1170
- const shouldProcess = !options.filter || options.filter(headers, value);
1178
+ const bytes = message.value;
1179
+ const shouldProcess = !options.filter || options.filter(headers, bytes.toString("utf8"));
1171
1180
  if (!targetTopic || !shouldProcess) {
1172
1181
  skipped++;
1173
1182
  } else if (options.dryRun) {
1174
1183
  deps.logger.log(`[DLQ replay dry-run] Would replay to "${targetTopic}"`);
1175
1184
  replayed++;
1176
1185
  } else {
1177
- await deps.send(targetTopic, [{ value, headers: originalHeaders }]);
1186
+ await deps.send(targetTopic, [{ value: bytes, headers: originalHeaders }]);
1178
1187
  replayed++;
1179
1188
  }
1180
1189
  const allDone = Array.from(highWatermarks.entries()).every(
@@ -2048,7 +2057,7 @@ async function waitForPartitionAssignment(consumer, topics, logger, timeoutMs =
2048
2057
  `Retry consumer did not receive partition assignments for [${topics.join(", ")}] within ${timeoutMs}ms`
2049
2058
  );
2050
2059
  }
2051
- async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopics, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs) {
2060
+ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopics, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs, serdeMap) {
2052
2061
  const {
2053
2062
  logger,
2054
2063
  producer,
@@ -2094,20 +2103,35 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
2094
2103
  await sleep(remaining);
2095
2104
  consumer.resume([{ topic: levelTopic, partitions: [partition] }]);
2096
2105
  }
2097
- const raw = message.value.toString();
2098
- const parsed = parseJsonMessage(raw, levelTopic, logger);
2099
- if (parsed === null) {
2100
- await consumer.commitOffsets([nextOffset]);
2101
- return;
2102
- }
2106
+ const rawBytes = message.value;
2103
2107
  const currentMaxRetries = parseInt(
2104
2108
  headers[RETRY_HEADER_MAX_RETRIES] ?? String(retry.maxRetries),
2105
2109
  10
2106
2110
  );
2107
2111
  const originalTopic = headers[RETRY_HEADER_ORIGINAL_TOPIC] ?? levelTopic.replace(/\.retry\.\d+$/, "");
2112
+ const serde = serdeMap?.get(originalTopic) ?? deps.serde;
2113
+ let parsed;
2114
+ try {
2115
+ parsed = await serde.deserialize(rawBytes, {
2116
+ topic: originalTopic,
2117
+ headers,
2118
+ isKey: false
2119
+ });
2120
+ } catch (err) {
2121
+ logger.error(
2122
+ `Failed to deserialize retry message from topic ${levelTopic}:`,
2123
+ toError(err).stack
2124
+ );
2125
+ await consumer.commitOffsets([nextOffset]);
2126
+ return;
2127
+ }
2128
+ if (parsed === null) {
2129
+ await consumer.commitOffsets([nextOffset]);
2130
+ return;
2131
+ }
2108
2132
  const validated = await validateWithSchema(
2109
2133
  parsed,
2110
- raw,
2134
+ rawBytes,
2111
2135
  originalTopic,
2112
2136
  schemaMap,
2113
2137
  interceptors,
@@ -2161,7 +2185,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
2161
2185
  const delay = Math.floor(Math.random() * cap);
2162
2186
  const { topic: rtTopic, messages: rtMsgs } = buildRetryTopicPayload(
2163
2187
  originalTopic,
2164
- [raw],
2188
+ [rawBytes],
2165
2189
  nextLevel,
2166
2190
  currentMaxRetries,
2167
2191
  delay,
@@ -2203,7 +2227,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
2203
2227
  } else if (dlq) {
2204
2228
  const { topic: dTopic, messages: dMsgs } = buildDlqPayload(
2205
2229
  originalTopic,
2206
- raw,
2230
+ rawBytes,
2207
2231
  {
2208
2232
  error,
2209
2233
  // +1 to account for the main consumer's initial attempt before routing.
@@ -2265,7 +2289,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
2265
2289
  `Retry level ${level}/${retry.maxRetries} consumer started for: ${originalTopics.join(", ")} (group: ${levelGroupId})`
2266
2290
  );
2267
2291
  }
2268
- async function startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs) {
2292
+ async function startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs, serdeMap) {
2269
2293
  const levelGroupIds = new Array(retry.maxRetries);
2270
2294
  await Promise.all(
2271
2295
  Array.from({ length: retry.maxRetries }, async (_, i) => {
@@ -2283,7 +2307,8 @@ async function startRetryTopicConsumers(originalTopics, originalGroupId, handleM
2283
2307
  interceptors,
2284
2308
  schemaMap,
2285
2309
  deps,
2286
- assignmentTimeoutMs
2310
+ assignmentTimeoutMs,
2311
+ serdeMap
2287
2312
  );
2288
2313
  levelGroupIds[i] = levelGroupId;
2289
2314
  })
@@ -2367,6 +2392,7 @@ async function setupConsumer(ctx, topics, mode, options) {
2367
2392
  optionSchemas,
2368
2393
  ctx.logger
2369
2394
  );
2395
+ const serdeMap = buildSerdeMap(stringTopics);
2370
2396
  const topicNames = stringTopics.map((t) => resolveTopicName(t));
2371
2397
  const subscribeTopics = [...topicNames, ...regexTopics];
2372
2398
  await ensureConsumerTopics(ctx, topicNames, dlq, options.deduplication);
@@ -2384,7 +2410,7 @@ async function setupConsumer(ctx, topics, mode, options) {
2384
2410
  ctx.logger.log(
2385
2411
  `${mode === "eachBatch" ? "Batch consumer" : "Consumer"} subscribed to topics: ${displayTopics}`
2386
2412
  );
2387
- return { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, hasRegex, readyPromise };
2413
+ return { consumer, schemaMap, serdeMap, topicNames, gid, dlq, interceptors, retry, hasRegex, readyPromise };
2388
2414
  }
2389
2415
  function resolveDeduplicationContext(ctx, groupId, options) {
2390
2416
  if (!options) return void 0;
@@ -2396,6 +2422,7 @@ function messageDepsFor(ctx, gid, options) {
2396
2422
  return {
2397
2423
  logger: ctx.logger,
2398
2424
  producer: ctx.producer,
2425
+ serde: ctx.serde,
2399
2426
  instrumentation: ctx.instrumentation,
2400
2427
  onMessageLost: options?.onMessageLost ?? ctx.onMessageLost,
2401
2428
  onTtlExpired: ctx.onTtlExpired,
@@ -2413,6 +2440,7 @@ function buildRetryTopicDeps(ctx) {
2413
2440
  return {
2414
2441
  logger: ctx.logger,
2415
2442
  producer: ctx.producer,
2443
+ serde: ctx.serde,
2416
2444
  instrumentation: ctx.instrumentation,
2417
2445
  onMessageLost: ctx.onMessageLost,
2418
2446
  onRetry: ctx.metrics.notifyRetry.bind(ctx.metrics),
@@ -2430,7 +2458,7 @@ async function makeEosMainContext(ctx, gid, consumer, options) {
2430
2458
  return { txProducer, consumer };
2431
2459
  }
2432
2460
  async function launchRetryChain(ctx, gid, topicNames, handleMessage, opts) {
2433
- const { retry, dlq, interceptors, schemaMap, assignmentTimeoutMs } = opts;
2461
+ const { retry, dlq, interceptors, schemaMap, serdeMap, assignmentTimeoutMs } = opts;
2434
2462
  if (!ctx.autoCreateTopicsEnabled) {
2435
2463
  await ctx.adminOps.validateRetryTopicsExist(topicNames, retry.maxRetries);
2436
2464
  }
@@ -2454,7 +2482,8 @@ async function launchRetryChain(ctx, gid, topicNames, handleMessage, opts) {
2454
2482
  ctx.companionGroupIds.get(gid).push(levelGroupId);
2455
2483
  }
2456
2484
  },
2457
- assignmentTimeoutMs
2485
+ assignmentTimeoutMs,
2486
+ serdeMap
2458
2487
  );
2459
2488
  }
2460
2489
 
@@ -2512,18 +2541,29 @@ async function applyDeduplication(envelope, raw, dedup, dlq, deps) {
2512
2541
  }
2513
2542
  return false;
2514
2543
  }
2515
- async function parseSingleMessage(message, topic2, partition, schemaMap, interceptors, dlq, deps) {
2544
+ async function parseSingleMessage(message, topic2, partition, schemaMap, interceptors, dlq, deps, serdeMap) {
2516
2545
  if (!message.value) {
2517
2546
  deps.logger.warn(`Received empty message from topic ${topic2}`);
2518
2547
  return null;
2519
2548
  }
2520
- const raw = message.value.toString();
2521
- const parsed = parseJsonMessage(raw, topic2, deps.logger);
2522
- if (parsed === null) return null;
2549
+ const bytes = message.value;
2523
2550
  const headers = decodeHeaders(message.headers);
2551
+ const serde = serdeMap?.get(topic2) ?? deps.serde;
2552
+ let parsed;
2553
+ try {
2554
+ parsed = await serde.deserialize(bytes, { topic: topic2, headers, isKey: false });
2555
+ } catch (error) {
2556
+ deps.logger.error(
2557
+ `Failed to deserialize message from topic ${topic2}:`,
2558
+ toError(error).stack
2559
+ );
2560
+ return null;
2561
+ }
2562
+ if (parsed === null) return null;
2524
2563
  const validated = await validateWithSchema(
2525
2564
  parsed,
2526
- raw,
2565
+ // Forward the ORIGINAL bytes to DLQ on validation failure (binary-safe).
2566
+ bytes,
2527
2567
  topic2,
2528
2568
  schemaMap,
2529
2569
  interceptors,
@@ -2537,6 +2577,7 @@ async function handleEachMessage(payload, opts, deps) {
2537
2577
  const { topic: topic2, partition, message } = payload;
2538
2578
  const {
2539
2579
  schemaMap,
2580
+ serdeMap,
2540
2581
  handleMessage,
2541
2582
  interceptors,
2542
2583
  dlq,
@@ -2545,6 +2586,7 @@ async function handleEachMessage(payload, opts, deps) {
2545
2586
  timeoutMs,
2546
2587
  wrapWithTimeout
2547
2588
  } = opts;
2589
+ const rawBytes = message.value;
2548
2590
  const eos = opts.eosMainContext;
2549
2591
  const nextOffsetStr = (parseInt(message.offset, 10) + 1).toString();
2550
2592
  const commitOffset = eos ? async () => {
@@ -2589,7 +2631,8 @@ async function handleEachMessage(payload, opts, deps) {
2589
2631
  schemaMap,
2590
2632
  interceptors,
2591
2633
  dlq,
2592
- deps
2634
+ deps,
2635
+ serdeMap
2593
2636
  );
2594
2637
  if (envelope === null) {
2595
2638
  await commitOffset?.();
@@ -2598,7 +2641,7 @@ async function handleEachMessage(payload, opts, deps) {
2598
2641
  if (opts.deduplication) {
2599
2642
  const isDuplicate = await applyDeduplication(
2600
2643
  envelope,
2601
- message.value.toString(),
2644
+ rawBytes,
2602
2645
  opts.deduplication,
2603
2646
  dlq,
2604
2647
  deps
@@ -2615,7 +2658,7 @@ async function handleEachMessage(payload, opts, deps) {
2615
2658
  `[KafkaClient] TTL expired on ${topic2}: age ${ageMs}ms > ${opts.messageTtlMs}ms`
2616
2659
  );
2617
2660
  if (dlq) {
2618
- await sendToDlq(topic2, message.value.toString(), deps, {
2661
+ await sendToDlq(topic2, rawBytes, deps, {
2619
2662
  error: new Error(`Message TTL expired: age ${ageMs}ms`),
2620
2663
  attempt: 0,
2621
2664
  originalHeaders: envelope.headers
@@ -2647,7 +2690,7 @@ async function handleEachMessage(payload, opts, deps) {
2647
2690
  },
2648
2691
  {
2649
2692
  envelope,
2650
- rawMessages: [message.value.toString()],
2693
+ rawMessages: [rawBytes],
2651
2694
  interceptors,
2652
2695
  dlq,
2653
2696
  retry,
@@ -2660,6 +2703,7 @@ async function handleEachBatch(payload, opts, deps) {
2660
2703
  const { batch, heartbeat, resolveOffset, commitOffsetsIfNecessary } = payload;
2661
2704
  const {
2662
2705
  schemaMap,
2706
+ serdeMap,
2663
2707
  handleBatch,
2664
2708
  interceptors,
2665
2709
  dlq,
@@ -2715,6 +2759,7 @@ async function handleEachBatch(payload, opts, deps) {
2715
2759
  const envelopes = [];
2716
2760
  const rawMessages = [];
2717
2761
  for (const message of batch.messages) {
2762
+ const rawBytes = message.value;
2718
2763
  const envelope = await parseSingleMessage(
2719
2764
  message,
2720
2765
  batch.topic,
@@ -2722,14 +2767,14 @@ async function handleEachBatch(payload, opts, deps) {
2722
2767
  schemaMap,
2723
2768
  interceptors,
2724
2769
  dlq,
2725
- deps
2770
+ deps,
2771
+ serdeMap
2726
2772
  );
2727
2773
  if (envelope === null) continue;
2728
2774
  if (opts.deduplication) {
2729
- const raw = message.value.toString();
2730
2775
  const isDuplicate = await applyDeduplication(
2731
2776
  envelope,
2732
- raw,
2777
+ rawBytes,
2733
2778
  opts.deduplication,
2734
2779
  dlq,
2735
2780
  deps
@@ -2743,7 +2788,7 @@ async function handleEachBatch(payload, opts, deps) {
2743
2788
  `[KafkaClient] TTL expired on ${batch.topic}: age ${ageMs}ms > ${opts.messageTtlMs}ms`
2744
2789
  );
2745
2790
  if (dlq) {
2746
- await sendToDlq(batch.topic, message.value.toString(), deps, {
2791
+ await sendToDlq(batch.topic, rawBytes, deps, {
2747
2792
  error: new Error(`Message TTL expired: age ${ageMs}ms`),
2748
2793
  attempt: 0,
2749
2794
  originalHeaders: envelope.headers
@@ -2762,7 +2807,7 @@ async function handleEachBatch(payload, opts, deps) {
2762
2807
  }
2763
2808
  }
2764
2809
  envelopes.push(envelope);
2765
- rawMessages.push(message.value.toString());
2810
+ rawMessages.push(rawBytes);
2766
2811
  }
2767
2812
  if (envelopes.length === 0) {
2768
2813
  await commitBatchOffset?.();
@@ -2925,7 +2970,7 @@ function resumeTopicAllPartitions(ctx, gid, topic2) {
2925
2970
  async function startConsumerImpl(ctx, topics, handleMessage, options = {}) {
2926
2971
  validateTopicConsumerOpts(topics, options);
2927
2972
  const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
2928
- const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachMessage", setupOptions);
2973
+ const { consumer, schemaMap, serdeMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachMessage", setupOptions);
2929
2974
  if (options.circuitBreaker)
2930
2975
  ctx.circuitBreaker.setConfig(gid, options.circuitBreaker);
2931
2976
  const deps = messageDepsFor(ctx, gid, options);
@@ -2936,6 +2981,7 @@ async function startConsumerImpl(ctx, topics, handleMessage, options = {}) {
2936
2981
  payload,
2937
2982
  {
2938
2983
  schemaMap,
2984
+ serdeMap,
2939
2985
  handleMessage,
2940
2986
  interceptors,
2941
2987
  dlq,
@@ -2963,6 +3009,7 @@ async function startConsumerImpl(ctx, topics, handleMessage, options = {}) {
2963
3009
  dlq,
2964
3010
  interceptors,
2965
3011
  schemaMap,
3012
+ serdeMap,
2966
3013
  assignmentTimeoutMs: options.retryTopicAssignmentTimeoutMs
2967
3014
  });
2968
3015
  }
@@ -2976,7 +3023,7 @@ async function startBatchConsumerImpl(ctx, topics, handleBatch, options = {}) {
2976
3023
  );
2977
3024
  }
2978
3025
  const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
2979
- const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachBatch", setupOptions);
3026
+ const { consumer, schemaMap, serdeMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachBatch", setupOptions);
2980
3027
  if (options.circuitBreaker)
2981
3028
  ctx.circuitBreaker.setConfig(gid, options.circuitBreaker);
2982
3029
  const deps = messageDepsFor(ctx, gid, options);
@@ -2987,6 +3034,7 @@ async function startBatchConsumerImpl(ctx, topics, handleBatch, options = {}) {
2987
3034
  payload,
2988
3035
  {
2989
3036
  schemaMap,
3037
+ serdeMap,
2990
3038
  handleBatch,
2991
3039
  interceptors,
2992
3040
  dlq,
@@ -3024,6 +3072,7 @@ async function startBatchConsumerImpl(ctx, topics, handleBatch, options = {}) {
3024
3072
  dlq,
3025
3073
  interceptors,
3026
3074
  schemaMap,
3075
+ serdeMap,
3027
3076
  assignmentTimeoutMs: options.retryTopicAssignmentTimeoutMs
3028
3077
  });
3029
3078
  }
@@ -3036,7 +3085,7 @@ async function startTransactionalConsumerImpl(ctx, topics, handler, options = {}
3036
3085
  );
3037
3086
  }
3038
3087
  const setupOptions = { ...options, autoCommit: false };
3039
- const { consumer, schemaMap, gid, readyPromise } = await setupConsumer(
3088
+ const { consumer, schemaMap, serdeMap, gid, readyPromise } = await setupConsumer(
3040
3089
  ctx,
3041
3090
  topics,
3042
3091
  "eachMessage",
@@ -3053,7 +3102,8 @@ async function startTransactionalConsumerImpl(ctx, topics, handler, options = {}
3053
3102
  schemaMap,
3054
3103
  options.interceptors ?? [],
3055
3104
  false,
3056
- deps
3105
+ deps,
3106
+ serdeMap
3057
3107
  );
3058
3108
  const nextOffset = String(Number.parseInt(message.offset, 10) + 1);
3059
3109
  if (envelope === null) {
@@ -3271,7 +3321,9 @@ async function startDelayedRelayImpl(ctx, topics, options) {
3271
3321
  topic: target,
3272
3322
  messages: [
3273
3323
  {
3274
- value: message.value.toString(),
3324
+ // Forward the ORIGINAL wire bytes unchanged — no re-serialization,
3325
+ // so binary payloads (Avro/Protobuf) are relayed losslessly.
3326
+ value: message.value,
3275
3327
  key: message.key ? message.key.toString() : null,
3276
3328
  headers: forwardHeaders
3277
3329
  }
@@ -3598,6 +3650,7 @@ var KafkaClient = class {
3598
3650
  const consumers = /* @__PURE__ */ new Map();
3599
3651
  const consumerCreationOptions = /* @__PURE__ */ new Map();
3600
3652
  const schemaRegistry = /* @__PURE__ */ new Map();
3653
+ const serde = options?.serde ?? new JsonSerde();
3601
3654
  const adminOps = new AdminOps({
3602
3655
  admin: transport.admin(),
3603
3656
  logger,
@@ -3624,6 +3677,7 @@ var KafkaClient = class {
3624
3677
  autoCreateTopicsEnabled: options?.autoCreateTopics ?? false,
3625
3678
  strictSchemasEnabled: options?.strictSchemas ?? true,
3626
3679
  numPartitions: options?.numPartitions ?? 1,
3680
+ serde,
3627
3681
  txId: options?.transactionalId ?? `${clientId}-tx`,
3628
3682
  clockRecoveryTopics: options?.clockRecovery?.topics ?? [],
3629
3683
  clockRecoveryTimeoutMs: options?.clockRecovery?.timeoutMs ?? 3e4,
@@ -3658,6 +3712,7 @@ var KafkaClient = class {
3658
3712
  strictSchemasEnabled: options?.strictSchemas ?? true,
3659
3713
  instrumentation: options?.instrumentation ?? [],
3660
3714
  logger,
3715
+ serde,
3661
3716
  nextLamportClock: () => 0
3662
3717
  // patched below
3663
3718
  },
@@ -3676,6 +3731,7 @@ var KafkaClient = class {
3676
3731
  strictSchemasEnabled: options?.strictSchemas ?? true,
3677
3732
  instrumentation: options?.instrumentation ?? [],
3678
3733
  logger,
3734
+ serde,
3679
3735
  nextLamportClock: () => ++ctx._lamportClock
3680
3736
  };
3681
3737
  ctx.retryTopicDeps = buildRetryTopicDeps(ctx);
@@ -3944,10 +4000,8 @@ function topic(name) {
3944
4000
  function keyable(desc) {
3945
4001
  return {
3946
4002
  ...desc,
3947
- key: (extractor) => ({
3948
- ...desc,
3949
- __key: extractor
3950
- })
4003
+ key: (extractor) => keyable({ ...desc, __key: extractor }),
4004
+ serde: (serde) => keyable({ ...desc, __serde: serde })
3951
4005
  };
3952
4006
  }
3953
4007
 
@@ -3978,112 +4032,6 @@ function versionedSchema(versions, options) {
3978
4032
  };
3979
4033
  }
3980
4034
 
3981
- // src/client/message/schema-registry.ts
3982
- var SchemaRegistryClient = class {
3983
- constructor(options) {
3984
- this.options = options;
3985
- if (!options.baseUrl) {
3986
- throw new Error("SchemaRegistryClient: baseUrl is required");
3987
- }
3988
- this.fetchFn = options.fetchFn ?? fetch;
3989
- this.cacheTtlMs = options.cacheTtlMs ?? 3e5;
3990
- }
3991
- options;
3992
- fetchFn;
3993
- cacheTtlMs;
3994
- latestCache = /* @__PURE__ */ new Map();
3995
- headers() {
3996
- const h = {
3997
- "Content-Type": "application/vnd.schemaregistry.v1+json"
3998
- };
3999
- if (this.options.auth) {
4000
- const { username, password } = this.options.auth;
4001
- h["Authorization"] = "Basic " + Buffer.from(`${username}:${password}`).toString("base64");
4002
- }
4003
- return h;
4004
- }
4005
- async request(method, path, body) {
4006
- const url = `${this.options.baseUrl.replace(/\/$/, "")}${path}`;
4007
- const res = await this.fetchFn(url, {
4008
- method,
4009
- headers: this.headers(),
4010
- ...body !== void 0 && { body: JSON.stringify(body) }
4011
- });
4012
- if (!res.ok) {
4013
- const text = await res.text().catch(() => "");
4014
- throw new Error(
4015
- `SchemaRegistry ${method} ${path} failed: ${res.status} ${res.statusText}${text ? ` \u2014 ${text}` : ""}`
4016
- );
4017
- }
4018
- return await res.json();
4019
- }
4020
- /** Fetch the latest schema registered under `subject`. Cached for `cacheTtlMs`. */
4021
- async getLatestSchema(subject) {
4022
- const cached = this.latestCache.get(subject);
4023
- if (cached && cached.expiresAt > Date.now()) return cached.value;
4024
- const raw = await this.request("GET", `/subjects/${encodeURIComponent(subject)}/versions/latest`);
4025
- const value = {
4026
- id: raw.id,
4027
- version: raw.version,
4028
- schema: raw.schema
4029
- };
4030
- this.latestCache.set(subject, {
4031
- value,
4032
- expiresAt: Date.now() + this.cacheTtlMs
4033
- });
4034
- return value;
4035
- }
4036
- /** Fetch a specific schema version of a subject. */
4037
- async getSchemaVersion(subject, version) {
4038
- const raw = await this.request(
4039
- "GET",
4040
- `/subjects/${encodeURIComponent(subject)}/versions/${version}`
4041
- );
4042
- return { id: raw.id, version: raw.version, schema: raw.schema };
4043
- }
4044
- /**
4045
- * Register a schema under `subject` (idempotent — re-registering the same
4046
- * schema returns the existing id). Returns the registry-assigned schema id.
4047
- */
4048
- async registerSchema(subject, schema, schemaType = "JSON") {
4049
- this.latestCache.delete(subject);
4050
- return this.request(
4051
- "POST",
4052
- `/subjects/${encodeURIComponent(subject)}/versions`,
4053
- { schema, schemaType }
4054
- );
4055
- }
4056
- /**
4057
- * Test `schema` against the subject's compatibility policy without registering.
4058
- * Returns `true` when the registry reports the schema as compatible.
4059
- */
4060
- async checkCompatibility(subject, schema, schemaType = "JSON") {
4061
- const res = await this.request(
4062
- "POST",
4063
- `/compatibility/subjects/${encodeURIComponent(subject)}/versions/latest`,
4064
- { schema, schemaType }
4065
- );
4066
- return res.is_compatible;
4067
- }
4068
- };
4069
- function registrySchema(client, subject, options) {
4070
- const enforceVersion = options?.enforceVersion ?? true;
4071
- return {
4072
- async parse(data, ctx) {
4073
- const latest = await client.getLatestSchema(subject);
4074
- if (enforceVersion && ctx?.version !== void 0 && ctx.version > latest.version) {
4075
- throw new Error(
4076
- `registrySchema: message version ${ctx.version} for subject "${subject}" is newer than the latest registered version ${latest.version} \u2014 register the schema before producing with it`
4077
- );
4078
- }
4079
- if (options?.validator) {
4080
- return options.validator.parse(data, ctx);
4081
- }
4082
- return data;
4083
- }
4084
- };
4085
- }
4086
-
4087
4035
  // src/client/outbox/outbox.store.ts
4088
4036
  var InMemoryOutboxStore = class {
4089
4037
  /** Insertion-ordered rows. `published` flips to true after `markPublished`. */
@@ -4798,8 +4746,6 @@ export {
4798
4746
  KafkaClient,
4799
4747
  topic,
4800
4748
  versionedSchema,
4801
- SchemaRegistryClient,
4802
- registrySchema,
4803
4749
  InMemoryOutboxStore,
4804
4750
  startOutboxRelay,
4805
4751
  awsMskIamProvider,
@@ -4811,4 +4757,4 @@ export {
4811
4757
  consumerOptionsFromEnv,
4812
4758
  mergeConsumerOptions
4813
4759
  };
4814
- //# sourceMappingURL=chunk-CMO7SMVK.mjs.map
4760
+ //# sourceMappingURL=chunk-OR7TPAAE.mjs.map