@drarzter/kafka-client 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +70 -2
- package/dist/{chunk-CMO7SMVK.mjs → chunk-OR7TPAAE.mjs} +110 -164
- package/dist/chunk-OR7TPAAE.mjs.map +1 -0
- package/dist/chunk-PQVBRDNV.mjs +149 -0
- package/dist/chunk-PQVBRDNV.mjs.map +1 -0
- package/dist/cli/index.js +115 -51
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/index.mjs +2 -1
- package/dist/cli/index.mjs.map +1 -1
- package/dist/client/kafka.client/consumer/features/delayed.d.ts.map +1 -1
- package/dist/client/kafka.client/consumer/features/dlq-replay.d.ts +1 -1
- package/dist/client/kafka.client/consumer/features/dlq-replay.d.ts.map +1 -1
- package/dist/client/kafka.client/consumer/features/snapshot.d.ts.map +1 -1
- package/dist/client/kafka.client/consumer/handler.d.ts +16 -2
- package/dist/client/kafka.client/consumer/handler.d.ts.map +1 -1
- package/dist/client/kafka.client/consumer/ops.d.ts +13 -0
- package/dist/client/kafka.client/consumer/ops.d.ts.map +1 -1
- package/dist/client/kafka.client/consumer/pipeline.d.ts +14 -13
- package/dist/client/kafka.client/consumer/pipeline.d.ts.map +1 -1
- package/dist/client/kafka.client/consumer/retry-topic.d.ts +4 -1
- package/dist/client/kafka.client/consumer/retry-topic.d.ts.map +1 -1
- package/dist/client/kafka.client/consumer/setup.d.ts +3 -0
- package/dist/client/kafka.client/consumer/setup.d.ts.map +1 -1
- package/dist/client/kafka.client/consumer/start.d.ts.map +1 -1
- package/dist/client/kafka.client/context.d.ts +3 -0
- package/dist/client/kafka.client/context.d.ts.map +1 -1
- package/dist/client/kafka.client/index.d.ts.map +1 -1
- package/dist/client/kafka.client/producer/ops.d.ts +12 -3
- package/dist/client/kafka.client/producer/ops.d.ts.map +1 -1
- package/dist/client/kafka.client/producer/send.d.ts +1 -1
- package/dist/client/message/schema-registry.d.ts +23 -4
- package/dist/client/message/schema-registry.d.ts.map +1 -1
- package/dist/client/message/serde.d.ts +68 -0
- package/dist/client/message/serde.d.ts.map +1 -0
- package/dist/client/message/topic.d.ts +25 -4
- package/dist/client/message/topic.d.ts.map +1 -1
- package/dist/client/transport/transport.interface.d.ts +6 -1
- package/dist/client/transport/transport.interface.d.ts.map +1 -1
- package/dist/client/types/config.types.d.ts +17 -0
- package/dist/client/types/config.types.d.ts.map +1 -1
- package/dist/core.d.ts +3 -0
- package/dist/core.d.ts.map +1 -1
- package/dist/core.js +146 -55
- package/dist/core.js.map +1 -1
- package/dist/core.mjs +9 -3
- package/dist/index.js +146 -55
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +9 -3
- package/dist/index.mjs.map +1 -1
- package/dist/serde.d.ts +157 -0
- package/dist/serde.d.ts.map +1 -0
- package/dist/serde.js +308 -0
- package/dist/serde.js.map +1 -0
- package/dist/serde.mjs +158 -0
- package/dist/serde.mjs.map +1 -0
- package/package.json +20 -1
- package/dist/chunk-CMO7SMVK.mjs.map +0 -1
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
import {
|
|
2
|
+
JsonSerde
|
|
3
|
+
} from "./chunk-PQVBRDNV.mjs";
|
|
4
|
+
|
|
1
5
|
// src/client/transport/confluent.transport.ts
|
|
2
6
|
import { KafkaJS } from "@confluentinc/kafka-javascript";
|
|
3
7
|
var { Kafka: KafkaClass, logLevel: KafkaLogLevel, PartitionAssigners } = KafkaJS;
|
|
@@ -325,6 +329,9 @@ var KafkaRetryExhaustedError = class extends KafkaProcessingError {
|
|
|
325
329
|
};
|
|
326
330
|
|
|
327
331
|
// src/client/kafka.client/producer/ops.ts
|
|
332
|
+
function resolveSerde(topicOrDesc, clientSerde) {
|
|
333
|
+
return topicOrDesc?.__serde ?? clientSerde;
|
|
334
|
+
}
|
|
328
335
|
function resolveTopicName(topicOrDescriptor) {
|
|
329
336
|
if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
|
|
330
337
|
if (topicOrDescriptor && typeof topicOrDescriptor === "object" && "__topic" in topicOrDescriptor) {
|
|
@@ -371,6 +378,7 @@ async function validateMessage(topicOrDesc, message, deps, ctx) {
|
|
|
371
378
|
}
|
|
372
379
|
async function buildSendPayload(topicOrDesc, messages, deps, compression) {
|
|
373
380
|
const topic2 = resolveTopicName(topicOrDesc);
|
|
381
|
+
const serde = resolveSerde(topicOrDesc, deps.serde);
|
|
374
382
|
const builtMessages = await Promise.all(
|
|
375
383
|
messages.map(async (m) => {
|
|
376
384
|
const envelopeHeaders = buildEnvelopeHeaders({
|
|
@@ -390,10 +398,13 @@ async function buildSendPayload(topicOrDesc, messages, deps, compression) {
|
|
|
390
398
|
headers: envelopeHeaders,
|
|
391
399
|
version: m.schemaVersion ?? 1
|
|
392
400
|
};
|
|
401
|
+
const validated = await validateMessage(topicOrDesc, m.value, deps, sendCtx);
|
|
393
402
|
return {
|
|
394
|
-
value:
|
|
395
|
-
|
|
396
|
-
|
|
403
|
+
value: await serde.serialize(validated, {
|
|
404
|
+
topic: topic2,
|
|
405
|
+
headers: envelopeHeaders,
|
|
406
|
+
isKey: false
|
|
407
|
+
}),
|
|
397
408
|
// Explicit key wins; otherwise fall back to the descriptor's .key()
|
|
398
409
|
// extractor (runs on the original, pre-validation payload).
|
|
399
410
|
key: m.key ?? topicOrDesc?.__key?.(m.value) ?? null,
|
|
@@ -478,6 +489,15 @@ function buildSchemaMap(topics, schemaRegistry, optionSchemas, logger) {
|
|
|
478
489
|
}
|
|
479
490
|
return schemaMap;
|
|
480
491
|
}
|
|
492
|
+
function buildSerdeMap(topics) {
|
|
493
|
+
let serdeMap;
|
|
494
|
+
for (const t of topics) {
|
|
495
|
+
if (t?.__serde) {
|
|
496
|
+
(serdeMap ??= /* @__PURE__ */ new Map()).set(resolveTopicName(t), t.__serde);
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
return serdeMap;
|
|
500
|
+
}
|
|
481
501
|
|
|
482
502
|
// src/client/kafka.client/admin/ops.ts
|
|
483
503
|
var AdminOps = class {
|
|
@@ -763,17 +783,6 @@ var AdminOps = class {
|
|
|
763
783
|
function sleep(ms) {
|
|
764
784
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
765
785
|
}
|
|
766
|
-
function parseJsonMessage(raw, topic2, logger) {
|
|
767
|
-
try {
|
|
768
|
-
return JSON.parse(raw);
|
|
769
|
-
} catch (error) {
|
|
770
|
-
logger.error(
|
|
771
|
-
`Failed to parse message from topic ${topic2}:`,
|
|
772
|
-
toError(error).stack
|
|
773
|
-
);
|
|
774
|
-
return null;
|
|
775
|
-
}
|
|
776
|
-
}
|
|
777
786
|
async function validateWithSchema(message, raw, topic2, schemaMap, interceptors, dlq, deps) {
|
|
778
787
|
const schema = schemaMap.get(topic2);
|
|
779
788
|
if (!schema) return message;
|
|
@@ -1166,15 +1175,15 @@ async function replayDlqTopic(topic2, deps, options = {}) {
|
|
|
1166
1175
|
const originalHeaders = Object.fromEntries(
|
|
1167
1176
|
Object.entries(headers).filter(([k]) => !deps.dlqHeaderKeys.has(k))
|
|
1168
1177
|
);
|
|
1169
|
-
const
|
|
1170
|
-
const shouldProcess = !options.filter || options.filter(headers,
|
|
1178
|
+
const bytes = message.value;
|
|
1179
|
+
const shouldProcess = !options.filter || options.filter(headers, bytes.toString("utf8"));
|
|
1171
1180
|
if (!targetTopic || !shouldProcess) {
|
|
1172
1181
|
skipped++;
|
|
1173
1182
|
} else if (options.dryRun) {
|
|
1174
1183
|
deps.logger.log(`[DLQ replay dry-run] Would replay to "${targetTopic}"`);
|
|
1175
1184
|
replayed++;
|
|
1176
1185
|
} else {
|
|
1177
|
-
await deps.send(targetTopic, [{ value, headers: originalHeaders }]);
|
|
1186
|
+
await deps.send(targetTopic, [{ value: bytes, headers: originalHeaders }]);
|
|
1178
1187
|
replayed++;
|
|
1179
1188
|
}
|
|
1180
1189
|
const allDone = Array.from(highWatermarks.entries()).every(
|
|
@@ -2048,7 +2057,7 @@ async function waitForPartitionAssignment(consumer, topics, logger, timeoutMs =
|
|
|
2048
2057
|
`Retry consumer did not receive partition assignments for [${topics.join(", ")}] within ${timeoutMs}ms`
|
|
2049
2058
|
);
|
|
2050
2059
|
}
|
|
2051
|
-
async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopics, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs) {
|
|
2060
|
+
async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopics, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs, serdeMap) {
|
|
2052
2061
|
const {
|
|
2053
2062
|
logger,
|
|
2054
2063
|
producer,
|
|
@@ -2094,20 +2103,35 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
|
|
|
2094
2103
|
await sleep(remaining);
|
|
2095
2104
|
consumer.resume([{ topic: levelTopic, partitions: [partition] }]);
|
|
2096
2105
|
}
|
|
2097
|
-
const
|
|
2098
|
-
const parsed = parseJsonMessage(raw, levelTopic, logger);
|
|
2099
|
-
if (parsed === null) {
|
|
2100
|
-
await consumer.commitOffsets([nextOffset]);
|
|
2101
|
-
return;
|
|
2102
|
-
}
|
|
2106
|
+
const rawBytes = message.value;
|
|
2103
2107
|
const currentMaxRetries = parseInt(
|
|
2104
2108
|
headers[RETRY_HEADER_MAX_RETRIES] ?? String(retry.maxRetries),
|
|
2105
2109
|
10
|
|
2106
2110
|
);
|
|
2107
2111
|
const originalTopic = headers[RETRY_HEADER_ORIGINAL_TOPIC] ?? levelTopic.replace(/\.retry\.\d+$/, "");
|
|
2112
|
+
const serde = serdeMap?.get(originalTopic) ?? deps.serde;
|
|
2113
|
+
let parsed;
|
|
2114
|
+
try {
|
|
2115
|
+
parsed = await serde.deserialize(rawBytes, {
|
|
2116
|
+
topic: originalTopic,
|
|
2117
|
+
headers,
|
|
2118
|
+
isKey: false
|
|
2119
|
+
});
|
|
2120
|
+
} catch (err) {
|
|
2121
|
+
logger.error(
|
|
2122
|
+
`Failed to deserialize retry message from topic ${levelTopic}:`,
|
|
2123
|
+
toError(err).stack
|
|
2124
|
+
);
|
|
2125
|
+
await consumer.commitOffsets([nextOffset]);
|
|
2126
|
+
return;
|
|
2127
|
+
}
|
|
2128
|
+
if (parsed === null) {
|
|
2129
|
+
await consumer.commitOffsets([nextOffset]);
|
|
2130
|
+
return;
|
|
2131
|
+
}
|
|
2108
2132
|
const validated = await validateWithSchema(
|
|
2109
2133
|
parsed,
|
|
2110
|
-
|
|
2134
|
+
rawBytes,
|
|
2111
2135
|
originalTopic,
|
|
2112
2136
|
schemaMap,
|
|
2113
2137
|
interceptors,
|
|
@@ -2161,7 +2185,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
|
|
|
2161
2185
|
const delay = Math.floor(Math.random() * cap);
|
|
2162
2186
|
const { topic: rtTopic, messages: rtMsgs } = buildRetryTopicPayload(
|
|
2163
2187
|
originalTopic,
|
|
2164
|
-
[
|
|
2188
|
+
[rawBytes],
|
|
2165
2189
|
nextLevel,
|
|
2166
2190
|
currentMaxRetries,
|
|
2167
2191
|
delay,
|
|
@@ -2203,7 +2227,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
|
|
|
2203
2227
|
} else if (dlq) {
|
|
2204
2228
|
const { topic: dTopic, messages: dMsgs } = buildDlqPayload(
|
|
2205
2229
|
originalTopic,
|
|
2206
|
-
|
|
2230
|
+
rawBytes,
|
|
2207
2231
|
{
|
|
2208
2232
|
error,
|
|
2209
2233
|
// +1 to account for the main consumer's initial attempt before routing.
|
|
@@ -2265,7 +2289,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
|
|
|
2265
2289
|
`Retry level ${level}/${retry.maxRetries} consumer started for: ${originalTopics.join(", ")} (group: ${levelGroupId})`
|
|
2266
2290
|
);
|
|
2267
2291
|
}
|
|
2268
|
-
async function startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs) {
|
|
2292
|
+
async function startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs, serdeMap) {
|
|
2269
2293
|
const levelGroupIds = new Array(retry.maxRetries);
|
|
2270
2294
|
await Promise.all(
|
|
2271
2295
|
Array.from({ length: retry.maxRetries }, async (_, i) => {
|
|
@@ -2283,7 +2307,8 @@ async function startRetryTopicConsumers(originalTopics, originalGroupId, handleM
|
|
|
2283
2307
|
interceptors,
|
|
2284
2308
|
schemaMap,
|
|
2285
2309
|
deps,
|
|
2286
|
-
assignmentTimeoutMs
|
|
2310
|
+
assignmentTimeoutMs,
|
|
2311
|
+
serdeMap
|
|
2287
2312
|
);
|
|
2288
2313
|
levelGroupIds[i] = levelGroupId;
|
|
2289
2314
|
})
|
|
@@ -2367,6 +2392,7 @@ async function setupConsumer(ctx, topics, mode, options) {
|
|
|
2367
2392
|
optionSchemas,
|
|
2368
2393
|
ctx.logger
|
|
2369
2394
|
);
|
|
2395
|
+
const serdeMap = buildSerdeMap(stringTopics);
|
|
2370
2396
|
const topicNames = stringTopics.map((t) => resolveTopicName(t));
|
|
2371
2397
|
const subscribeTopics = [...topicNames, ...regexTopics];
|
|
2372
2398
|
await ensureConsumerTopics(ctx, topicNames, dlq, options.deduplication);
|
|
@@ -2384,7 +2410,7 @@ async function setupConsumer(ctx, topics, mode, options) {
|
|
|
2384
2410
|
ctx.logger.log(
|
|
2385
2411
|
`${mode === "eachBatch" ? "Batch consumer" : "Consumer"} subscribed to topics: ${displayTopics}`
|
|
2386
2412
|
);
|
|
2387
|
-
return { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, hasRegex, readyPromise };
|
|
2413
|
+
return { consumer, schemaMap, serdeMap, topicNames, gid, dlq, interceptors, retry, hasRegex, readyPromise };
|
|
2388
2414
|
}
|
|
2389
2415
|
function resolveDeduplicationContext(ctx, groupId, options) {
|
|
2390
2416
|
if (!options) return void 0;
|
|
@@ -2396,6 +2422,7 @@ function messageDepsFor(ctx, gid, options) {
|
|
|
2396
2422
|
return {
|
|
2397
2423
|
logger: ctx.logger,
|
|
2398
2424
|
producer: ctx.producer,
|
|
2425
|
+
serde: ctx.serde,
|
|
2399
2426
|
instrumentation: ctx.instrumentation,
|
|
2400
2427
|
onMessageLost: options?.onMessageLost ?? ctx.onMessageLost,
|
|
2401
2428
|
onTtlExpired: ctx.onTtlExpired,
|
|
@@ -2413,6 +2440,7 @@ function buildRetryTopicDeps(ctx) {
|
|
|
2413
2440
|
return {
|
|
2414
2441
|
logger: ctx.logger,
|
|
2415
2442
|
producer: ctx.producer,
|
|
2443
|
+
serde: ctx.serde,
|
|
2416
2444
|
instrumentation: ctx.instrumentation,
|
|
2417
2445
|
onMessageLost: ctx.onMessageLost,
|
|
2418
2446
|
onRetry: ctx.metrics.notifyRetry.bind(ctx.metrics),
|
|
@@ -2430,7 +2458,7 @@ async function makeEosMainContext(ctx, gid, consumer, options) {
|
|
|
2430
2458
|
return { txProducer, consumer };
|
|
2431
2459
|
}
|
|
2432
2460
|
async function launchRetryChain(ctx, gid, topicNames, handleMessage, opts) {
|
|
2433
|
-
const { retry, dlq, interceptors, schemaMap, assignmentTimeoutMs } = opts;
|
|
2461
|
+
const { retry, dlq, interceptors, schemaMap, serdeMap, assignmentTimeoutMs } = opts;
|
|
2434
2462
|
if (!ctx.autoCreateTopicsEnabled) {
|
|
2435
2463
|
await ctx.adminOps.validateRetryTopicsExist(topicNames, retry.maxRetries);
|
|
2436
2464
|
}
|
|
@@ -2454,7 +2482,8 @@ async function launchRetryChain(ctx, gid, topicNames, handleMessage, opts) {
|
|
|
2454
2482
|
ctx.companionGroupIds.get(gid).push(levelGroupId);
|
|
2455
2483
|
}
|
|
2456
2484
|
},
|
|
2457
|
-
assignmentTimeoutMs
|
|
2485
|
+
assignmentTimeoutMs,
|
|
2486
|
+
serdeMap
|
|
2458
2487
|
);
|
|
2459
2488
|
}
|
|
2460
2489
|
|
|
@@ -2512,18 +2541,29 @@ async function applyDeduplication(envelope, raw, dedup, dlq, deps) {
|
|
|
2512
2541
|
}
|
|
2513
2542
|
return false;
|
|
2514
2543
|
}
|
|
2515
|
-
async function parseSingleMessage(message, topic2, partition, schemaMap, interceptors, dlq, deps) {
|
|
2544
|
+
async function parseSingleMessage(message, topic2, partition, schemaMap, interceptors, dlq, deps, serdeMap) {
|
|
2516
2545
|
if (!message.value) {
|
|
2517
2546
|
deps.logger.warn(`Received empty message from topic ${topic2}`);
|
|
2518
2547
|
return null;
|
|
2519
2548
|
}
|
|
2520
|
-
const
|
|
2521
|
-
const parsed = parseJsonMessage(raw, topic2, deps.logger);
|
|
2522
|
-
if (parsed === null) return null;
|
|
2549
|
+
const bytes = message.value;
|
|
2523
2550
|
const headers = decodeHeaders(message.headers);
|
|
2551
|
+
const serde = serdeMap?.get(topic2) ?? deps.serde;
|
|
2552
|
+
let parsed;
|
|
2553
|
+
try {
|
|
2554
|
+
parsed = await serde.deserialize(bytes, { topic: topic2, headers, isKey: false });
|
|
2555
|
+
} catch (error) {
|
|
2556
|
+
deps.logger.error(
|
|
2557
|
+
`Failed to deserialize message from topic ${topic2}:`,
|
|
2558
|
+
toError(error).stack
|
|
2559
|
+
);
|
|
2560
|
+
return null;
|
|
2561
|
+
}
|
|
2562
|
+
if (parsed === null) return null;
|
|
2524
2563
|
const validated = await validateWithSchema(
|
|
2525
2564
|
parsed,
|
|
2526
|
-
|
|
2565
|
+
// Forward the ORIGINAL bytes to DLQ on validation failure (binary-safe).
|
|
2566
|
+
bytes,
|
|
2527
2567
|
topic2,
|
|
2528
2568
|
schemaMap,
|
|
2529
2569
|
interceptors,
|
|
@@ -2537,6 +2577,7 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2537
2577
|
const { topic: topic2, partition, message } = payload;
|
|
2538
2578
|
const {
|
|
2539
2579
|
schemaMap,
|
|
2580
|
+
serdeMap,
|
|
2540
2581
|
handleMessage,
|
|
2541
2582
|
interceptors,
|
|
2542
2583
|
dlq,
|
|
@@ -2545,6 +2586,7 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2545
2586
|
timeoutMs,
|
|
2546
2587
|
wrapWithTimeout
|
|
2547
2588
|
} = opts;
|
|
2589
|
+
const rawBytes = message.value;
|
|
2548
2590
|
const eos = opts.eosMainContext;
|
|
2549
2591
|
const nextOffsetStr = (parseInt(message.offset, 10) + 1).toString();
|
|
2550
2592
|
const commitOffset = eos ? async () => {
|
|
@@ -2589,7 +2631,8 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2589
2631
|
schemaMap,
|
|
2590
2632
|
interceptors,
|
|
2591
2633
|
dlq,
|
|
2592
|
-
deps
|
|
2634
|
+
deps,
|
|
2635
|
+
serdeMap
|
|
2593
2636
|
);
|
|
2594
2637
|
if (envelope === null) {
|
|
2595
2638
|
await commitOffset?.();
|
|
@@ -2598,7 +2641,7 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2598
2641
|
if (opts.deduplication) {
|
|
2599
2642
|
const isDuplicate = await applyDeduplication(
|
|
2600
2643
|
envelope,
|
|
2601
|
-
|
|
2644
|
+
rawBytes,
|
|
2602
2645
|
opts.deduplication,
|
|
2603
2646
|
dlq,
|
|
2604
2647
|
deps
|
|
@@ -2615,7 +2658,7 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2615
2658
|
`[KafkaClient] TTL expired on ${topic2}: age ${ageMs}ms > ${opts.messageTtlMs}ms`
|
|
2616
2659
|
);
|
|
2617
2660
|
if (dlq) {
|
|
2618
|
-
await sendToDlq(topic2,
|
|
2661
|
+
await sendToDlq(topic2, rawBytes, deps, {
|
|
2619
2662
|
error: new Error(`Message TTL expired: age ${ageMs}ms`),
|
|
2620
2663
|
attempt: 0,
|
|
2621
2664
|
originalHeaders: envelope.headers
|
|
@@ -2647,7 +2690,7 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2647
2690
|
},
|
|
2648
2691
|
{
|
|
2649
2692
|
envelope,
|
|
2650
|
-
rawMessages: [
|
|
2693
|
+
rawMessages: [rawBytes],
|
|
2651
2694
|
interceptors,
|
|
2652
2695
|
dlq,
|
|
2653
2696
|
retry,
|
|
@@ -2660,6 +2703,7 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
2660
2703
|
const { batch, heartbeat, resolveOffset, commitOffsetsIfNecessary } = payload;
|
|
2661
2704
|
const {
|
|
2662
2705
|
schemaMap,
|
|
2706
|
+
serdeMap,
|
|
2663
2707
|
handleBatch,
|
|
2664
2708
|
interceptors,
|
|
2665
2709
|
dlq,
|
|
@@ -2715,6 +2759,7 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
2715
2759
|
const envelopes = [];
|
|
2716
2760
|
const rawMessages = [];
|
|
2717
2761
|
for (const message of batch.messages) {
|
|
2762
|
+
const rawBytes = message.value;
|
|
2718
2763
|
const envelope = await parseSingleMessage(
|
|
2719
2764
|
message,
|
|
2720
2765
|
batch.topic,
|
|
@@ -2722,14 +2767,14 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
2722
2767
|
schemaMap,
|
|
2723
2768
|
interceptors,
|
|
2724
2769
|
dlq,
|
|
2725
|
-
deps
|
|
2770
|
+
deps,
|
|
2771
|
+
serdeMap
|
|
2726
2772
|
);
|
|
2727
2773
|
if (envelope === null) continue;
|
|
2728
2774
|
if (opts.deduplication) {
|
|
2729
|
-
const raw = message.value.toString();
|
|
2730
2775
|
const isDuplicate = await applyDeduplication(
|
|
2731
2776
|
envelope,
|
|
2732
|
-
|
|
2777
|
+
rawBytes,
|
|
2733
2778
|
opts.deduplication,
|
|
2734
2779
|
dlq,
|
|
2735
2780
|
deps
|
|
@@ -2743,7 +2788,7 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
2743
2788
|
`[KafkaClient] TTL expired on ${batch.topic}: age ${ageMs}ms > ${opts.messageTtlMs}ms`
|
|
2744
2789
|
);
|
|
2745
2790
|
if (dlq) {
|
|
2746
|
-
await sendToDlq(batch.topic,
|
|
2791
|
+
await sendToDlq(batch.topic, rawBytes, deps, {
|
|
2747
2792
|
error: new Error(`Message TTL expired: age ${ageMs}ms`),
|
|
2748
2793
|
attempt: 0,
|
|
2749
2794
|
originalHeaders: envelope.headers
|
|
@@ -2762,7 +2807,7 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
2762
2807
|
}
|
|
2763
2808
|
}
|
|
2764
2809
|
envelopes.push(envelope);
|
|
2765
|
-
rawMessages.push(
|
|
2810
|
+
rawMessages.push(rawBytes);
|
|
2766
2811
|
}
|
|
2767
2812
|
if (envelopes.length === 0) {
|
|
2768
2813
|
await commitBatchOffset?.();
|
|
@@ -2925,7 +2970,7 @@ function resumeTopicAllPartitions(ctx, gid, topic2) {
|
|
|
2925
2970
|
async function startConsumerImpl(ctx, topics, handleMessage, options = {}) {
|
|
2926
2971
|
validateTopicConsumerOpts(topics, options);
|
|
2927
2972
|
const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
|
|
2928
|
-
const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachMessage", setupOptions);
|
|
2973
|
+
const { consumer, schemaMap, serdeMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachMessage", setupOptions);
|
|
2929
2974
|
if (options.circuitBreaker)
|
|
2930
2975
|
ctx.circuitBreaker.setConfig(gid, options.circuitBreaker);
|
|
2931
2976
|
const deps = messageDepsFor(ctx, gid, options);
|
|
@@ -2936,6 +2981,7 @@ async function startConsumerImpl(ctx, topics, handleMessage, options = {}) {
|
|
|
2936
2981
|
payload,
|
|
2937
2982
|
{
|
|
2938
2983
|
schemaMap,
|
|
2984
|
+
serdeMap,
|
|
2939
2985
|
handleMessage,
|
|
2940
2986
|
interceptors,
|
|
2941
2987
|
dlq,
|
|
@@ -2963,6 +3009,7 @@ async function startConsumerImpl(ctx, topics, handleMessage, options = {}) {
|
|
|
2963
3009
|
dlq,
|
|
2964
3010
|
interceptors,
|
|
2965
3011
|
schemaMap,
|
|
3012
|
+
serdeMap,
|
|
2966
3013
|
assignmentTimeoutMs: options.retryTopicAssignmentTimeoutMs
|
|
2967
3014
|
});
|
|
2968
3015
|
}
|
|
@@ -2976,7 +3023,7 @@ async function startBatchConsumerImpl(ctx, topics, handleBatch, options = {}) {
|
|
|
2976
3023
|
);
|
|
2977
3024
|
}
|
|
2978
3025
|
const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
|
|
2979
|
-
const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachBatch", setupOptions);
|
|
3026
|
+
const { consumer, schemaMap, serdeMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachBatch", setupOptions);
|
|
2980
3027
|
if (options.circuitBreaker)
|
|
2981
3028
|
ctx.circuitBreaker.setConfig(gid, options.circuitBreaker);
|
|
2982
3029
|
const deps = messageDepsFor(ctx, gid, options);
|
|
@@ -2987,6 +3034,7 @@ async function startBatchConsumerImpl(ctx, topics, handleBatch, options = {}) {
|
|
|
2987
3034
|
payload,
|
|
2988
3035
|
{
|
|
2989
3036
|
schemaMap,
|
|
3037
|
+
serdeMap,
|
|
2990
3038
|
handleBatch,
|
|
2991
3039
|
interceptors,
|
|
2992
3040
|
dlq,
|
|
@@ -3024,6 +3072,7 @@ async function startBatchConsumerImpl(ctx, topics, handleBatch, options = {}) {
|
|
|
3024
3072
|
dlq,
|
|
3025
3073
|
interceptors,
|
|
3026
3074
|
schemaMap,
|
|
3075
|
+
serdeMap,
|
|
3027
3076
|
assignmentTimeoutMs: options.retryTopicAssignmentTimeoutMs
|
|
3028
3077
|
});
|
|
3029
3078
|
}
|
|
@@ -3036,7 +3085,7 @@ async function startTransactionalConsumerImpl(ctx, topics, handler, options = {}
|
|
|
3036
3085
|
);
|
|
3037
3086
|
}
|
|
3038
3087
|
const setupOptions = { ...options, autoCommit: false };
|
|
3039
|
-
const { consumer, schemaMap, gid, readyPromise } = await setupConsumer(
|
|
3088
|
+
const { consumer, schemaMap, serdeMap, gid, readyPromise } = await setupConsumer(
|
|
3040
3089
|
ctx,
|
|
3041
3090
|
topics,
|
|
3042
3091
|
"eachMessage",
|
|
@@ -3053,7 +3102,8 @@ async function startTransactionalConsumerImpl(ctx, topics, handler, options = {}
|
|
|
3053
3102
|
schemaMap,
|
|
3054
3103
|
options.interceptors ?? [],
|
|
3055
3104
|
false,
|
|
3056
|
-
deps
|
|
3105
|
+
deps,
|
|
3106
|
+
serdeMap
|
|
3057
3107
|
);
|
|
3058
3108
|
const nextOffset = String(Number.parseInt(message.offset, 10) + 1);
|
|
3059
3109
|
if (envelope === null) {
|
|
@@ -3271,7 +3321,9 @@ async function startDelayedRelayImpl(ctx, topics, options) {
|
|
|
3271
3321
|
topic: target,
|
|
3272
3322
|
messages: [
|
|
3273
3323
|
{
|
|
3274
|
-
|
|
3324
|
+
// Forward the ORIGINAL wire bytes unchanged — no re-serialization,
|
|
3325
|
+
// so binary payloads (Avro/Protobuf) are relayed losslessly.
|
|
3326
|
+
value: message.value,
|
|
3275
3327
|
key: message.key ? message.key.toString() : null,
|
|
3276
3328
|
headers: forwardHeaders
|
|
3277
3329
|
}
|
|
@@ -3598,6 +3650,7 @@ var KafkaClient = class {
|
|
|
3598
3650
|
const consumers = /* @__PURE__ */ new Map();
|
|
3599
3651
|
const consumerCreationOptions = /* @__PURE__ */ new Map();
|
|
3600
3652
|
const schemaRegistry = /* @__PURE__ */ new Map();
|
|
3653
|
+
const serde = options?.serde ?? new JsonSerde();
|
|
3601
3654
|
const adminOps = new AdminOps({
|
|
3602
3655
|
admin: transport.admin(),
|
|
3603
3656
|
logger,
|
|
@@ -3624,6 +3677,7 @@ var KafkaClient = class {
|
|
|
3624
3677
|
autoCreateTopicsEnabled: options?.autoCreateTopics ?? false,
|
|
3625
3678
|
strictSchemasEnabled: options?.strictSchemas ?? true,
|
|
3626
3679
|
numPartitions: options?.numPartitions ?? 1,
|
|
3680
|
+
serde,
|
|
3627
3681
|
txId: options?.transactionalId ?? `${clientId}-tx`,
|
|
3628
3682
|
clockRecoveryTopics: options?.clockRecovery?.topics ?? [],
|
|
3629
3683
|
clockRecoveryTimeoutMs: options?.clockRecovery?.timeoutMs ?? 3e4,
|
|
@@ -3658,6 +3712,7 @@ var KafkaClient = class {
|
|
|
3658
3712
|
strictSchemasEnabled: options?.strictSchemas ?? true,
|
|
3659
3713
|
instrumentation: options?.instrumentation ?? [],
|
|
3660
3714
|
logger,
|
|
3715
|
+
serde,
|
|
3661
3716
|
nextLamportClock: () => 0
|
|
3662
3717
|
// patched below
|
|
3663
3718
|
},
|
|
@@ -3676,6 +3731,7 @@ var KafkaClient = class {
|
|
|
3676
3731
|
strictSchemasEnabled: options?.strictSchemas ?? true,
|
|
3677
3732
|
instrumentation: options?.instrumentation ?? [],
|
|
3678
3733
|
logger,
|
|
3734
|
+
serde,
|
|
3679
3735
|
nextLamportClock: () => ++ctx._lamportClock
|
|
3680
3736
|
};
|
|
3681
3737
|
ctx.retryTopicDeps = buildRetryTopicDeps(ctx);
|
|
@@ -3944,10 +4000,8 @@ function topic(name) {
|
|
|
3944
4000
|
function keyable(desc) {
|
|
3945
4001
|
return {
|
|
3946
4002
|
...desc,
|
|
3947
|
-
key: (extractor) => ({
|
|
3948
|
-
|
|
3949
|
-
__key: extractor
|
|
3950
|
-
})
|
|
4003
|
+
key: (extractor) => keyable({ ...desc, __key: extractor }),
|
|
4004
|
+
serde: (serde) => keyable({ ...desc, __serde: serde })
|
|
3951
4005
|
};
|
|
3952
4006
|
}
|
|
3953
4007
|
|
|
@@ -3978,112 +4032,6 @@ function versionedSchema(versions, options) {
|
|
|
3978
4032
|
};
|
|
3979
4033
|
}
|
|
3980
4034
|
|
|
3981
|
-
// src/client/message/schema-registry.ts
|
|
3982
|
-
var SchemaRegistryClient = class {
|
|
3983
|
-
constructor(options) {
|
|
3984
|
-
this.options = options;
|
|
3985
|
-
if (!options.baseUrl) {
|
|
3986
|
-
throw new Error("SchemaRegistryClient: baseUrl is required");
|
|
3987
|
-
}
|
|
3988
|
-
this.fetchFn = options.fetchFn ?? fetch;
|
|
3989
|
-
this.cacheTtlMs = options.cacheTtlMs ?? 3e5;
|
|
3990
|
-
}
|
|
3991
|
-
options;
|
|
3992
|
-
fetchFn;
|
|
3993
|
-
cacheTtlMs;
|
|
3994
|
-
latestCache = /* @__PURE__ */ new Map();
|
|
3995
|
-
headers() {
|
|
3996
|
-
const h = {
|
|
3997
|
-
"Content-Type": "application/vnd.schemaregistry.v1+json"
|
|
3998
|
-
};
|
|
3999
|
-
if (this.options.auth) {
|
|
4000
|
-
const { username, password } = this.options.auth;
|
|
4001
|
-
h["Authorization"] = "Basic " + Buffer.from(`${username}:${password}`).toString("base64");
|
|
4002
|
-
}
|
|
4003
|
-
return h;
|
|
4004
|
-
}
|
|
4005
|
-
async request(method, path, body) {
|
|
4006
|
-
const url = `${this.options.baseUrl.replace(/\/$/, "")}${path}`;
|
|
4007
|
-
const res = await this.fetchFn(url, {
|
|
4008
|
-
method,
|
|
4009
|
-
headers: this.headers(),
|
|
4010
|
-
...body !== void 0 && { body: JSON.stringify(body) }
|
|
4011
|
-
});
|
|
4012
|
-
if (!res.ok) {
|
|
4013
|
-
const text = await res.text().catch(() => "");
|
|
4014
|
-
throw new Error(
|
|
4015
|
-
`SchemaRegistry ${method} ${path} failed: ${res.status} ${res.statusText}${text ? ` \u2014 ${text}` : ""}`
|
|
4016
|
-
);
|
|
4017
|
-
}
|
|
4018
|
-
return await res.json();
|
|
4019
|
-
}
|
|
4020
|
-
/** Fetch the latest schema registered under `subject`. Cached for `cacheTtlMs`. */
|
|
4021
|
-
async getLatestSchema(subject) {
|
|
4022
|
-
const cached = this.latestCache.get(subject);
|
|
4023
|
-
if (cached && cached.expiresAt > Date.now()) return cached.value;
|
|
4024
|
-
const raw = await this.request("GET", `/subjects/${encodeURIComponent(subject)}/versions/latest`);
|
|
4025
|
-
const value = {
|
|
4026
|
-
id: raw.id,
|
|
4027
|
-
version: raw.version,
|
|
4028
|
-
schema: raw.schema
|
|
4029
|
-
};
|
|
4030
|
-
this.latestCache.set(subject, {
|
|
4031
|
-
value,
|
|
4032
|
-
expiresAt: Date.now() + this.cacheTtlMs
|
|
4033
|
-
});
|
|
4034
|
-
return value;
|
|
4035
|
-
}
|
|
4036
|
-
/** Fetch a specific schema version of a subject. */
|
|
4037
|
-
async getSchemaVersion(subject, version) {
|
|
4038
|
-
const raw = await this.request(
|
|
4039
|
-
"GET",
|
|
4040
|
-
`/subjects/${encodeURIComponent(subject)}/versions/${version}`
|
|
4041
|
-
);
|
|
4042
|
-
return { id: raw.id, version: raw.version, schema: raw.schema };
|
|
4043
|
-
}
|
|
4044
|
-
/**
|
|
4045
|
-
* Register a schema under `subject` (idempotent — re-registering the same
|
|
4046
|
-
* schema returns the existing id). Returns the registry-assigned schema id.
|
|
4047
|
-
*/
|
|
4048
|
-
async registerSchema(subject, schema, schemaType = "JSON") {
|
|
4049
|
-
this.latestCache.delete(subject);
|
|
4050
|
-
return this.request(
|
|
4051
|
-
"POST",
|
|
4052
|
-
`/subjects/${encodeURIComponent(subject)}/versions`,
|
|
4053
|
-
{ schema, schemaType }
|
|
4054
|
-
);
|
|
4055
|
-
}
|
|
4056
|
-
/**
|
|
4057
|
-
* Test `schema` against the subject's compatibility policy without registering.
|
|
4058
|
-
* Returns `true` when the registry reports the schema as compatible.
|
|
4059
|
-
*/
|
|
4060
|
-
async checkCompatibility(subject, schema, schemaType = "JSON") {
|
|
4061
|
-
const res = await this.request(
|
|
4062
|
-
"POST",
|
|
4063
|
-
`/compatibility/subjects/${encodeURIComponent(subject)}/versions/latest`,
|
|
4064
|
-
{ schema, schemaType }
|
|
4065
|
-
);
|
|
4066
|
-
return res.is_compatible;
|
|
4067
|
-
}
|
|
4068
|
-
};
|
|
4069
|
-
function registrySchema(client, subject, options) {
|
|
4070
|
-
const enforceVersion = options?.enforceVersion ?? true;
|
|
4071
|
-
return {
|
|
4072
|
-
async parse(data, ctx) {
|
|
4073
|
-
const latest = await client.getLatestSchema(subject);
|
|
4074
|
-
if (enforceVersion && ctx?.version !== void 0 && ctx.version > latest.version) {
|
|
4075
|
-
throw new Error(
|
|
4076
|
-
`registrySchema: message version ${ctx.version} for subject "${subject}" is newer than the latest registered version ${latest.version} \u2014 register the schema before producing with it`
|
|
4077
|
-
);
|
|
4078
|
-
}
|
|
4079
|
-
if (options?.validator) {
|
|
4080
|
-
return options.validator.parse(data, ctx);
|
|
4081
|
-
}
|
|
4082
|
-
return data;
|
|
4083
|
-
}
|
|
4084
|
-
};
|
|
4085
|
-
}
|
|
4086
|
-
|
|
4087
4035
|
// src/client/outbox/outbox.store.ts
|
|
4088
4036
|
var InMemoryOutboxStore = class {
|
|
4089
4037
|
/** Insertion-ordered rows. `published` flips to true after `markPublished`. */
|
|
@@ -4798,8 +4746,6 @@ export {
|
|
|
4798
4746
|
KafkaClient,
|
|
4799
4747
|
topic,
|
|
4800
4748
|
versionedSchema,
|
|
4801
|
-
SchemaRegistryClient,
|
|
4802
|
-
registrySchema,
|
|
4803
4749
|
InMemoryOutboxStore,
|
|
4804
4750
|
startOutboxRelay,
|
|
4805
4751
|
awsMskIamProvider,
|
|
@@ -4811,4 +4757,4 @@ export {
|
|
|
4811
4757
|
consumerOptionsFromEnv,
|
|
4812
4758
|
mergeConsumerOptions
|
|
4813
4759
|
};
|
|
4814
|
-
//# sourceMappingURL=chunk-
|
|
4760
|
+
//# sourceMappingURL=chunk-OR7TPAAE.mjs.map
|