@drarzter/kafka-client 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +70 -2
- package/dist/{chunk-CMO7SMVK.mjs → chunk-OR7TPAAE.mjs} +110 -164
- package/dist/chunk-OR7TPAAE.mjs.map +1 -0
- package/dist/chunk-PQVBRDNV.mjs +149 -0
- package/dist/chunk-PQVBRDNV.mjs.map +1 -0
- package/dist/cli/index.js +115 -51
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/index.mjs +2 -1
- package/dist/cli/index.mjs.map +1 -1
- package/dist/client/kafka.client/consumer/features/delayed.d.ts.map +1 -1
- package/dist/client/kafka.client/consumer/features/dlq-replay.d.ts +1 -1
- package/dist/client/kafka.client/consumer/features/dlq-replay.d.ts.map +1 -1
- package/dist/client/kafka.client/consumer/features/snapshot.d.ts.map +1 -1
- package/dist/client/kafka.client/consumer/handler.d.ts +16 -2
- package/dist/client/kafka.client/consumer/handler.d.ts.map +1 -1
- package/dist/client/kafka.client/consumer/ops.d.ts +13 -0
- package/dist/client/kafka.client/consumer/ops.d.ts.map +1 -1
- package/dist/client/kafka.client/consumer/pipeline.d.ts +14 -13
- package/dist/client/kafka.client/consumer/pipeline.d.ts.map +1 -1
- package/dist/client/kafka.client/consumer/retry-topic.d.ts +4 -1
- package/dist/client/kafka.client/consumer/retry-topic.d.ts.map +1 -1
- package/dist/client/kafka.client/consumer/setup.d.ts +3 -0
- package/dist/client/kafka.client/consumer/setup.d.ts.map +1 -1
- package/dist/client/kafka.client/consumer/start.d.ts.map +1 -1
- package/dist/client/kafka.client/context.d.ts +3 -0
- package/dist/client/kafka.client/context.d.ts.map +1 -1
- package/dist/client/kafka.client/index.d.ts.map +1 -1
- package/dist/client/kafka.client/producer/ops.d.ts +12 -3
- package/dist/client/kafka.client/producer/ops.d.ts.map +1 -1
- package/dist/client/kafka.client/producer/send.d.ts +1 -1
- package/dist/client/message/schema-registry.d.ts +23 -4
- package/dist/client/message/schema-registry.d.ts.map +1 -1
- package/dist/client/message/serde.d.ts +68 -0
- package/dist/client/message/serde.d.ts.map +1 -0
- package/dist/client/message/topic.d.ts +25 -4
- package/dist/client/message/topic.d.ts.map +1 -1
- package/dist/client/transport/transport.interface.d.ts +6 -1
- package/dist/client/transport/transport.interface.d.ts.map +1 -1
- package/dist/client/types/config.types.d.ts +17 -0
- package/dist/client/types/config.types.d.ts.map +1 -1
- package/dist/core.d.ts +3 -0
- package/dist/core.d.ts.map +1 -1
- package/dist/core.js +146 -55
- package/dist/core.js.map +1 -1
- package/dist/core.mjs +9 -3
- package/dist/index.js +146 -55
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +9 -3
- package/dist/index.mjs.map +1 -1
- package/dist/serde.d.ts +157 -0
- package/dist/serde.d.ts.map +1 -0
- package/dist/serde.js +308 -0
- package/dist/serde.js.map +1 -0
- package/dist/serde.mjs +158 -0
- package/dist/serde.mjs.map +1 -0
- package/package.json +20 -1
- package/dist/chunk-CMO7SMVK.mjs.map +0 -1
package/dist/core.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import {
|
|
2
|
+
ConfluentTransport,
|
|
2
3
|
HEADER_CORRELATION_ID,
|
|
3
4
|
HEADER_DELAYED_TARGET,
|
|
4
5
|
HEADER_DELAYED_UNTIL,
|
|
@@ -13,7 +14,6 @@ import {
|
|
|
13
14
|
KafkaProcessingError,
|
|
14
15
|
KafkaRetryExhaustedError,
|
|
15
16
|
KafkaValidationError,
|
|
16
|
-
SchemaRegistryClient,
|
|
17
17
|
awsMskIamProvider,
|
|
18
18
|
buildEnvelopeHeaders,
|
|
19
19
|
consumerOptionsFromEnv,
|
|
@@ -24,7 +24,6 @@ import {
|
|
|
24
24
|
getEnvelopeContext,
|
|
25
25
|
kafkaClientConfigFromEnv,
|
|
26
26
|
mergeConsumerOptions,
|
|
27
|
-
registrySchema,
|
|
28
27
|
resolveSecurityOptions,
|
|
29
28
|
runWithEnvelopeContext,
|
|
30
29
|
startOutboxRelay,
|
|
@@ -33,9 +32,15 @@ import {
|
|
|
33
32
|
toMskIamPolicy,
|
|
34
33
|
topic,
|
|
35
34
|
versionedSchema
|
|
36
|
-
} from "./chunk-
|
|
35
|
+
} from "./chunk-OR7TPAAE.mjs";
|
|
36
|
+
import {
|
|
37
|
+
JsonSerde,
|
|
38
|
+
SchemaRegistryClient,
|
|
39
|
+
registrySchema
|
|
40
|
+
} from "./chunk-PQVBRDNV.mjs";
|
|
37
41
|
import "./chunk-EQQGB2QZ.mjs";
|
|
38
42
|
export {
|
|
43
|
+
ConfluentTransport,
|
|
39
44
|
HEADER_CORRELATION_ID,
|
|
40
45
|
HEADER_DELAYED_TARGET,
|
|
41
46
|
HEADER_DELAYED_UNTIL,
|
|
@@ -46,6 +51,7 @@ export {
|
|
|
46
51
|
HEADER_TRACEPARENT,
|
|
47
52
|
InMemoryDedupStore,
|
|
48
53
|
InMemoryOutboxStore,
|
|
54
|
+
JsonSerde,
|
|
49
55
|
KafkaClient,
|
|
50
56
|
KafkaProcessingError,
|
|
51
57
|
KafkaRetryExhaustedError,
|
package/dist/index.js
CHANGED
|
@@ -29,6 +29,7 @@ var __decorateParam = (index, decorator) => (target, key) => decorator(target, k
|
|
|
29
29
|
// src/index.ts
|
|
30
30
|
var index_exports = {};
|
|
31
31
|
__export(index_exports, {
|
|
32
|
+
ConfluentTransport: () => ConfluentTransport,
|
|
32
33
|
HEADER_CORRELATION_ID: () => HEADER_CORRELATION_ID,
|
|
33
34
|
HEADER_DELAYED_TARGET: () => HEADER_DELAYED_TARGET,
|
|
34
35
|
HEADER_DELAYED_UNTIL: () => HEADER_DELAYED_UNTIL,
|
|
@@ -40,6 +41,7 @@ __export(index_exports, {
|
|
|
40
41
|
InMemoryDedupStore: () => InMemoryDedupStore,
|
|
41
42
|
InMemoryOutboxStore: () => InMemoryOutboxStore,
|
|
42
43
|
InjectKafkaClient: () => InjectKafkaClient,
|
|
44
|
+
JsonSerde: () => JsonSerde,
|
|
43
45
|
KAFKA_CLIENT: () => KAFKA_CLIENT,
|
|
44
46
|
KAFKA_SUBSCRIBER_METADATA: () => KAFKA_SUBSCRIBER_METADATA,
|
|
45
47
|
KafkaClient: () => KafkaClient,
|
|
@@ -277,6 +279,18 @@ var ConfluentTransport = class {
|
|
|
277
279
|
}
|
|
278
280
|
};
|
|
279
281
|
|
|
282
|
+
// src/client/message/serde.ts
|
|
283
|
+
var JsonSerde = class {
|
|
284
|
+
/** JSON-stringify the validated payload. Returns a UTF-8 string. */
|
|
285
|
+
serialize(value) {
|
|
286
|
+
return JSON.stringify(value);
|
|
287
|
+
}
|
|
288
|
+
/** JSON-parse UTF-8 wire bytes into an object. */
|
|
289
|
+
deserialize(data) {
|
|
290
|
+
return JSON.parse(data.toString("utf8"));
|
|
291
|
+
}
|
|
292
|
+
};
|
|
293
|
+
|
|
280
294
|
// src/client/kafka.client/infra/dedup.store.ts
|
|
281
295
|
var InMemoryDedupStore = class {
|
|
282
296
|
constructor(states) {
|
|
@@ -401,6 +415,9 @@ var KafkaRetryExhaustedError = class extends KafkaProcessingError {
|
|
|
401
415
|
};
|
|
402
416
|
|
|
403
417
|
// src/client/kafka.client/producer/ops.ts
|
|
418
|
+
function resolveSerde(topicOrDesc, clientSerde) {
|
|
419
|
+
return topicOrDesc?.__serde ?? clientSerde;
|
|
420
|
+
}
|
|
404
421
|
function resolveTopicName(topicOrDescriptor) {
|
|
405
422
|
if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
|
|
406
423
|
if (topicOrDescriptor && typeof topicOrDescriptor === "object" && "__topic" in topicOrDescriptor) {
|
|
@@ -447,6 +464,7 @@ async function validateMessage(topicOrDesc, message, deps, ctx) {
|
|
|
447
464
|
}
|
|
448
465
|
async function buildSendPayload(topicOrDesc, messages, deps, compression) {
|
|
449
466
|
const topic2 = resolveTopicName(topicOrDesc);
|
|
467
|
+
const serde = resolveSerde(topicOrDesc, deps.serde);
|
|
450
468
|
const builtMessages = await Promise.all(
|
|
451
469
|
messages.map(async (m) => {
|
|
452
470
|
const envelopeHeaders = buildEnvelopeHeaders({
|
|
@@ -466,10 +484,13 @@ async function buildSendPayload(topicOrDesc, messages, deps, compression) {
|
|
|
466
484
|
headers: envelopeHeaders,
|
|
467
485
|
version: m.schemaVersion ?? 1
|
|
468
486
|
};
|
|
487
|
+
const validated = await validateMessage(topicOrDesc, m.value, deps, sendCtx);
|
|
469
488
|
return {
|
|
470
|
-
value:
|
|
471
|
-
|
|
472
|
-
|
|
489
|
+
value: await serde.serialize(validated, {
|
|
490
|
+
topic: topic2,
|
|
491
|
+
headers: envelopeHeaders,
|
|
492
|
+
isKey: false
|
|
493
|
+
}),
|
|
473
494
|
// Explicit key wins; otherwise fall back to the descriptor's .key()
|
|
474
495
|
// extractor (runs on the original, pre-validation payload).
|
|
475
496
|
key: m.key ?? topicOrDesc?.__key?.(m.value) ?? null,
|
|
@@ -554,6 +575,15 @@ function buildSchemaMap(topics, schemaRegistry, optionSchemas, logger) {
|
|
|
554
575
|
}
|
|
555
576
|
return schemaMap;
|
|
556
577
|
}
|
|
578
|
+
function buildSerdeMap(topics) {
|
|
579
|
+
let serdeMap;
|
|
580
|
+
for (const t of topics) {
|
|
581
|
+
if (t?.__serde) {
|
|
582
|
+
(serdeMap ??= /* @__PURE__ */ new Map()).set(resolveTopicName(t), t.__serde);
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
return serdeMap;
|
|
586
|
+
}
|
|
557
587
|
|
|
558
588
|
// src/client/kafka.client/admin/ops.ts
|
|
559
589
|
var AdminOps = class {
|
|
@@ -839,17 +869,6 @@ var AdminOps = class {
|
|
|
839
869
|
function sleep(ms) {
|
|
840
870
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
841
871
|
}
|
|
842
|
-
function parseJsonMessage(raw, topic2, logger) {
|
|
843
|
-
try {
|
|
844
|
-
return JSON.parse(raw);
|
|
845
|
-
} catch (error) {
|
|
846
|
-
logger.error(
|
|
847
|
-
`Failed to parse message from topic ${topic2}:`,
|
|
848
|
-
toError(error).stack
|
|
849
|
-
);
|
|
850
|
-
return null;
|
|
851
|
-
}
|
|
852
|
-
}
|
|
853
872
|
async function validateWithSchema(message, raw, topic2, schemaMap, interceptors, dlq, deps) {
|
|
854
873
|
const schema = schemaMap.get(topic2);
|
|
855
874
|
if (!schema) return message;
|
|
@@ -1242,15 +1261,15 @@ async function replayDlqTopic(topic2, deps, options = {}) {
|
|
|
1242
1261
|
const originalHeaders = Object.fromEntries(
|
|
1243
1262
|
Object.entries(headers).filter(([k]) => !deps.dlqHeaderKeys.has(k))
|
|
1244
1263
|
);
|
|
1245
|
-
const
|
|
1246
|
-
const shouldProcess = !options.filter || options.filter(headers,
|
|
1264
|
+
const bytes = message.value;
|
|
1265
|
+
const shouldProcess = !options.filter || options.filter(headers, bytes.toString("utf8"));
|
|
1247
1266
|
if (!targetTopic || !shouldProcess) {
|
|
1248
1267
|
skipped++;
|
|
1249
1268
|
} else if (options.dryRun) {
|
|
1250
1269
|
deps.logger.log(`[DLQ replay dry-run] Would replay to "${targetTopic}"`);
|
|
1251
1270
|
replayed++;
|
|
1252
1271
|
} else {
|
|
1253
|
-
await deps.send(targetTopic, [{ value, headers: originalHeaders }]);
|
|
1272
|
+
await deps.send(targetTopic, [{ value: bytes, headers: originalHeaders }]);
|
|
1254
1273
|
replayed++;
|
|
1255
1274
|
}
|
|
1256
1275
|
const allDone = Array.from(highWatermarks.entries()).every(
|
|
@@ -2124,7 +2143,7 @@ async function waitForPartitionAssignment(consumer, topics, logger, timeoutMs =
|
|
|
2124
2143
|
`Retry consumer did not receive partition assignments for [${topics.join(", ")}] within ${timeoutMs}ms`
|
|
2125
2144
|
);
|
|
2126
2145
|
}
|
|
2127
|
-
async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopics, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs) {
|
|
2146
|
+
async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopics, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs, serdeMap) {
|
|
2128
2147
|
const {
|
|
2129
2148
|
logger,
|
|
2130
2149
|
producer,
|
|
@@ -2170,20 +2189,35 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
|
|
|
2170
2189
|
await sleep(remaining);
|
|
2171
2190
|
consumer.resume([{ topic: levelTopic, partitions: [partition] }]);
|
|
2172
2191
|
}
|
|
2173
|
-
const
|
|
2174
|
-
const parsed = parseJsonMessage(raw, levelTopic, logger);
|
|
2175
|
-
if (parsed === null) {
|
|
2176
|
-
await consumer.commitOffsets([nextOffset]);
|
|
2177
|
-
return;
|
|
2178
|
-
}
|
|
2192
|
+
const rawBytes = message.value;
|
|
2179
2193
|
const currentMaxRetries = parseInt(
|
|
2180
2194
|
headers[RETRY_HEADER_MAX_RETRIES] ?? String(retry.maxRetries),
|
|
2181
2195
|
10
|
|
2182
2196
|
);
|
|
2183
2197
|
const originalTopic = headers[RETRY_HEADER_ORIGINAL_TOPIC] ?? levelTopic.replace(/\.retry\.\d+$/, "");
|
|
2198
|
+
const serde = serdeMap?.get(originalTopic) ?? deps.serde;
|
|
2199
|
+
let parsed;
|
|
2200
|
+
try {
|
|
2201
|
+
parsed = await serde.deserialize(rawBytes, {
|
|
2202
|
+
topic: originalTopic,
|
|
2203
|
+
headers,
|
|
2204
|
+
isKey: false
|
|
2205
|
+
});
|
|
2206
|
+
} catch (err) {
|
|
2207
|
+
logger.error(
|
|
2208
|
+
`Failed to deserialize retry message from topic ${levelTopic}:`,
|
|
2209
|
+
toError(err).stack
|
|
2210
|
+
);
|
|
2211
|
+
await consumer.commitOffsets([nextOffset]);
|
|
2212
|
+
return;
|
|
2213
|
+
}
|
|
2214
|
+
if (parsed === null) {
|
|
2215
|
+
await consumer.commitOffsets([nextOffset]);
|
|
2216
|
+
return;
|
|
2217
|
+
}
|
|
2184
2218
|
const validated = await validateWithSchema(
|
|
2185
2219
|
parsed,
|
|
2186
|
-
|
|
2220
|
+
rawBytes,
|
|
2187
2221
|
originalTopic,
|
|
2188
2222
|
schemaMap,
|
|
2189
2223
|
interceptors,
|
|
@@ -2237,7 +2271,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
|
|
|
2237
2271
|
const delay = Math.floor(Math.random() * cap);
|
|
2238
2272
|
const { topic: rtTopic, messages: rtMsgs } = buildRetryTopicPayload(
|
|
2239
2273
|
originalTopic,
|
|
2240
|
-
[
|
|
2274
|
+
[rawBytes],
|
|
2241
2275
|
nextLevel,
|
|
2242
2276
|
currentMaxRetries,
|
|
2243
2277
|
delay,
|
|
@@ -2279,7 +2313,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
|
|
|
2279
2313
|
} else if (dlq) {
|
|
2280
2314
|
const { topic: dTopic, messages: dMsgs } = buildDlqPayload(
|
|
2281
2315
|
originalTopic,
|
|
2282
|
-
|
|
2316
|
+
rawBytes,
|
|
2283
2317
|
{
|
|
2284
2318
|
error,
|
|
2285
2319
|
// +1 to account for the main consumer's initial attempt before routing.
|
|
@@ -2341,7 +2375,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
|
|
|
2341
2375
|
`Retry level ${level}/${retry.maxRetries} consumer started for: ${originalTopics.join(", ")} (group: ${levelGroupId})`
|
|
2342
2376
|
);
|
|
2343
2377
|
}
|
|
2344
|
-
async function startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs) {
|
|
2378
|
+
async function startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs, serdeMap) {
|
|
2345
2379
|
const levelGroupIds = new Array(retry.maxRetries);
|
|
2346
2380
|
await Promise.all(
|
|
2347
2381
|
Array.from({ length: retry.maxRetries }, async (_, i) => {
|
|
@@ -2359,7 +2393,8 @@ async function startRetryTopicConsumers(originalTopics, originalGroupId, handleM
|
|
|
2359
2393
|
interceptors,
|
|
2360
2394
|
schemaMap,
|
|
2361
2395
|
deps,
|
|
2362
|
-
assignmentTimeoutMs
|
|
2396
|
+
assignmentTimeoutMs,
|
|
2397
|
+
serdeMap
|
|
2363
2398
|
);
|
|
2364
2399
|
levelGroupIds[i] = levelGroupId;
|
|
2365
2400
|
})
|
|
@@ -2443,6 +2478,7 @@ async function setupConsumer(ctx, topics, mode, options) {
|
|
|
2443
2478
|
optionSchemas,
|
|
2444
2479
|
ctx.logger
|
|
2445
2480
|
);
|
|
2481
|
+
const serdeMap = buildSerdeMap(stringTopics);
|
|
2446
2482
|
const topicNames = stringTopics.map((t) => resolveTopicName(t));
|
|
2447
2483
|
const subscribeTopics = [...topicNames, ...regexTopics];
|
|
2448
2484
|
await ensureConsumerTopics(ctx, topicNames, dlq, options.deduplication);
|
|
@@ -2460,7 +2496,7 @@ async function setupConsumer(ctx, topics, mode, options) {
|
|
|
2460
2496
|
ctx.logger.log(
|
|
2461
2497
|
`${mode === "eachBatch" ? "Batch consumer" : "Consumer"} subscribed to topics: ${displayTopics}`
|
|
2462
2498
|
);
|
|
2463
|
-
return { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, hasRegex, readyPromise };
|
|
2499
|
+
return { consumer, schemaMap, serdeMap, topicNames, gid, dlq, interceptors, retry, hasRegex, readyPromise };
|
|
2464
2500
|
}
|
|
2465
2501
|
function resolveDeduplicationContext(ctx, groupId, options) {
|
|
2466
2502
|
if (!options) return void 0;
|
|
@@ -2472,6 +2508,7 @@ function messageDepsFor(ctx, gid, options) {
|
|
|
2472
2508
|
return {
|
|
2473
2509
|
logger: ctx.logger,
|
|
2474
2510
|
producer: ctx.producer,
|
|
2511
|
+
serde: ctx.serde,
|
|
2475
2512
|
instrumentation: ctx.instrumentation,
|
|
2476
2513
|
onMessageLost: options?.onMessageLost ?? ctx.onMessageLost,
|
|
2477
2514
|
onTtlExpired: ctx.onTtlExpired,
|
|
@@ -2489,6 +2526,7 @@ function buildRetryTopicDeps(ctx) {
|
|
|
2489
2526
|
return {
|
|
2490
2527
|
logger: ctx.logger,
|
|
2491
2528
|
producer: ctx.producer,
|
|
2529
|
+
serde: ctx.serde,
|
|
2492
2530
|
instrumentation: ctx.instrumentation,
|
|
2493
2531
|
onMessageLost: ctx.onMessageLost,
|
|
2494
2532
|
onRetry: ctx.metrics.notifyRetry.bind(ctx.metrics),
|
|
@@ -2506,7 +2544,7 @@ async function makeEosMainContext(ctx, gid, consumer, options) {
|
|
|
2506
2544
|
return { txProducer, consumer };
|
|
2507
2545
|
}
|
|
2508
2546
|
async function launchRetryChain(ctx, gid, topicNames, handleMessage, opts) {
|
|
2509
|
-
const { retry, dlq, interceptors, schemaMap, assignmentTimeoutMs } = opts;
|
|
2547
|
+
const { retry, dlq, interceptors, schemaMap, serdeMap, assignmentTimeoutMs } = opts;
|
|
2510
2548
|
if (!ctx.autoCreateTopicsEnabled) {
|
|
2511
2549
|
await ctx.adminOps.validateRetryTopicsExist(topicNames, retry.maxRetries);
|
|
2512
2550
|
}
|
|
@@ -2530,7 +2568,8 @@ async function launchRetryChain(ctx, gid, topicNames, handleMessage, opts) {
|
|
|
2530
2568
|
ctx.companionGroupIds.get(gid).push(levelGroupId);
|
|
2531
2569
|
}
|
|
2532
2570
|
},
|
|
2533
|
-
assignmentTimeoutMs
|
|
2571
|
+
assignmentTimeoutMs,
|
|
2572
|
+
serdeMap
|
|
2534
2573
|
);
|
|
2535
2574
|
}
|
|
2536
2575
|
|
|
@@ -2588,18 +2627,29 @@ async function applyDeduplication(envelope, raw, dedup, dlq, deps) {
|
|
|
2588
2627
|
}
|
|
2589
2628
|
return false;
|
|
2590
2629
|
}
|
|
2591
|
-
async function parseSingleMessage(message, topic2, partition, schemaMap, interceptors, dlq, deps) {
|
|
2630
|
+
async function parseSingleMessage(message, topic2, partition, schemaMap, interceptors, dlq, deps, serdeMap) {
|
|
2592
2631
|
if (!message.value) {
|
|
2593
2632
|
deps.logger.warn(`Received empty message from topic ${topic2}`);
|
|
2594
2633
|
return null;
|
|
2595
2634
|
}
|
|
2596
|
-
const
|
|
2597
|
-
const parsed = parseJsonMessage(raw, topic2, deps.logger);
|
|
2598
|
-
if (parsed === null) return null;
|
|
2635
|
+
const bytes = message.value;
|
|
2599
2636
|
const headers = decodeHeaders(message.headers);
|
|
2637
|
+
const serde = serdeMap?.get(topic2) ?? deps.serde;
|
|
2638
|
+
let parsed;
|
|
2639
|
+
try {
|
|
2640
|
+
parsed = await serde.deserialize(bytes, { topic: topic2, headers, isKey: false });
|
|
2641
|
+
} catch (error) {
|
|
2642
|
+
deps.logger.error(
|
|
2643
|
+
`Failed to deserialize message from topic ${topic2}:`,
|
|
2644
|
+
toError(error).stack
|
|
2645
|
+
);
|
|
2646
|
+
return null;
|
|
2647
|
+
}
|
|
2648
|
+
if (parsed === null) return null;
|
|
2600
2649
|
const validated = await validateWithSchema(
|
|
2601
2650
|
parsed,
|
|
2602
|
-
|
|
2651
|
+
// Forward the ORIGINAL bytes to DLQ on validation failure (binary-safe).
|
|
2652
|
+
bytes,
|
|
2603
2653
|
topic2,
|
|
2604
2654
|
schemaMap,
|
|
2605
2655
|
interceptors,
|
|
@@ -2613,6 +2663,7 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2613
2663
|
const { topic: topic2, partition, message } = payload;
|
|
2614
2664
|
const {
|
|
2615
2665
|
schemaMap,
|
|
2666
|
+
serdeMap,
|
|
2616
2667
|
handleMessage,
|
|
2617
2668
|
interceptors,
|
|
2618
2669
|
dlq,
|
|
@@ -2621,6 +2672,7 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2621
2672
|
timeoutMs,
|
|
2622
2673
|
wrapWithTimeout
|
|
2623
2674
|
} = opts;
|
|
2675
|
+
const rawBytes = message.value;
|
|
2624
2676
|
const eos = opts.eosMainContext;
|
|
2625
2677
|
const nextOffsetStr = (parseInt(message.offset, 10) + 1).toString();
|
|
2626
2678
|
const commitOffset = eos ? async () => {
|
|
@@ -2665,7 +2717,8 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2665
2717
|
schemaMap,
|
|
2666
2718
|
interceptors,
|
|
2667
2719
|
dlq,
|
|
2668
|
-
deps
|
|
2720
|
+
deps,
|
|
2721
|
+
serdeMap
|
|
2669
2722
|
);
|
|
2670
2723
|
if (envelope === null) {
|
|
2671
2724
|
await commitOffset?.();
|
|
@@ -2674,7 +2727,7 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2674
2727
|
if (opts.deduplication) {
|
|
2675
2728
|
const isDuplicate = await applyDeduplication(
|
|
2676
2729
|
envelope,
|
|
2677
|
-
|
|
2730
|
+
rawBytes,
|
|
2678
2731
|
opts.deduplication,
|
|
2679
2732
|
dlq,
|
|
2680
2733
|
deps
|
|
@@ -2691,7 +2744,7 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2691
2744
|
`[KafkaClient] TTL expired on ${topic2}: age ${ageMs}ms > ${opts.messageTtlMs}ms`
|
|
2692
2745
|
);
|
|
2693
2746
|
if (dlq) {
|
|
2694
|
-
await sendToDlq(topic2,
|
|
2747
|
+
await sendToDlq(topic2, rawBytes, deps, {
|
|
2695
2748
|
error: new Error(`Message TTL expired: age ${ageMs}ms`),
|
|
2696
2749
|
attempt: 0,
|
|
2697
2750
|
originalHeaders: envelope.headers
|
|
@@ -2723,7 +2776,7 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2723
2776
|
},
|
|
2724
2777
|
{
|
|
2725
2778
|
envelope,
|
|
2726
|
-
rawMessages: [
|
|
2779
|
+
rawMessages: [rawBytes],
|
|
2727
2780
|
interceptors,
|
|
2728
2781
|
dlq,
|
|
2729
2782
|
retry,
|
|
@@ -2736,6 +2789,7 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
2736
2789
|
const { batch, heartbeat, resolveOffset, commitOffsetsIfNecessary } = payload;
|
|
2737
2790
|
const {
|
|
2738
2791
|
schemaMap,
|
|
2792
|
+
serdeMap,
|
|
2739
2793
|
handleBatch,
|
|
2740
2794
|
interceptors,
|
|
2741
2795
|
dlq,
|
|
@@ -2791,6 +2845,7 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
2791
2845
|
const envelopes = [];
|
|
2792
2846
|
const rawMessages = [];
|
|
2793
2847
|
for (const message of batch.messages) {
|
|
2848
|
+
const rawBytes = message.value;
|
|
2794
2849
|
const envelope = await parseSingleMessage(
|
|
2795
2850
|
message,
|
|
2796
2851
|
batch.topic,
|
|
@@ -2798,14 +2853,14 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
2798
2853
|
schemaMap,
|
|
2799
2854
|
interceptors,
|
|
2800
2855
|
dlq,
|
|
2801
|
-
deps
|
|
2856
|
+
deps,
|
|
2857
|
+
serdeMap
|
|
2802
2858
|
);
|
|
2803
2859
|
if (envelope === null) continue;
|
|
2804
2860
|
if (opts.deduplication) {
|
|
2805
|
-
const raw = message.value.toString();
|
|
2806
2861
|
const isDuplicate = await applyDeduplication(
|
|
2807
2862
|
envelope,
|
|
2808
|
-
|
|
2863
|
+
rawBytes,
|
|
2809
2864
|
opts.deduplication,
|
|
2810
2865
|
dlq,
|
|
2811
2866
|
deps
|
|
@@ -2819,7 +2874,7 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
2819
2874
|
`[KafkaClient] TTL expired on ${batch.topic}: age ${ageMs}ms > ${opts.messageTtlMs}ms`
|
|
2820
2875
|
);
|
|
2821
2876
|
if (dlq) {
|
|
2822
|
-
await sendToDlq(batch.topic,
|
|
2877
|
+
await sendToDlq(batch.topic, rawBytes, deps, {
|
|
2823
2878
|
error: new Error(`Message TTL expired: age ${ageMs}ms`),
|
|
2824
2879
|
attempt: 0,
|
|
2825
2880
|
originalHeaders: envelope.headers
|
|
@@ -2838,7 +2893,7 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
2838
2893
|
}
|
|
2839
2894
|
}
|
|
2840
2895
|
envelopes.push(envelope);
|
|
2841
|
-
rawMessages.push(
|
|
2896
|
+
rawMessages.push(rawBytes);
|
|
2842
2897
|
}
|
|
2843
2898
|
if (envelopes.length === 0) {
|
|
2844
2899
|
await commitBatchOffset?.();
|
|
@@ -3001,7 +3056,7 @@ function resumeTopicAllPartitions(ctx, gid, topic2) {
|
|
|
3001
3056
|
async function startConsumerImpl(ctx, topics, handleMessage, options = {}) {
|
|
3002
3057
|
validateTopicConsumerOpts(topics, options);
|
|
3003
3058
|
const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
|
|
3004
|
-
const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachMessage", setupOptions);
|
|
3059
|
+
const { consumer, schemaMap, serdeMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachMessage", setupOptions);
|
|
3005
3060
|
if (options.circuitBreaker)
|
|
3006
3061
|
ctx.circuitBreaker.setConfig(gid, options.circuitBreaker);
|
|
3007
3062
|
const deps = messageDepsFor(ctx, gid, options);
|
|
@@ -3012,6 +3067,7 @@ async function startConsumerImpl(ctx, topics, handleMessage, options = {}) {
|
|
|
3012
3067
|
payload,
|
|
3013
3068
|
{
|
|
3014
3069
|
schemaMap,
|
|
3070
|
+
serdeMap,
|
|
3015
3071
|
handleMessage,
|
|
3016
3072
|
interceptors,
|
|
3017
3073
|
dlq,
|
|
@@ -3039,6 +3095,7 @@ async function startConsumerImpl(ctx, topics, handleMessage, options = {}) {
|
|
|
3039
3095
|
dlq,
|
|
3040
3096
|
interceptors,
|
|
3041
3097
|
schemaMap,
|
|
3098
|
+
serdeMap,
|
|
3042
3099
|
assignmentTimeoutMs: options.retryTopicAssignmentTimeoutMs
|
|
3043
3100
|
});
|
|
3044
3101
|
}
|
|
@@ -3052,7 +3109,7 @@ async function startBatchConsumerImpl(ctx, topics, handleBatch, options = {}) {
|
|
|
3052
3109
|
);
|
|
3053
3110
|
}
|
|
3054
3111
|
const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
|
|
3055
|
-
const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachBatch", setupOptions);
|
|
3112
|
+
const { consumer, schemaMap, serdeMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachBatch", setupOptions);
|
|
3056
3113
|
if (options.circuitBreaker)
|
|
3057
3114
|
ctx.circuitBreaker.setConfig(gid, options.circuitBreaker);
|
|
3058
3115
|
const deps = messageDepsFor(ctx, gid, options);
|
|
@@ -3063,6 +3120,7 @@ async function startBatchConsumerImpl(ctx, topics, handleBatch, options = {}) {
|
|
|
3063
3120
|
payload,
|
|
3064
3121
|
{
|
|
3065
3122
|
schemaMap,
|
|
3123
|
+
serdeMap,
|
|
3066
3124
|
handleBatch,
|
|
3067
3125
|
interceptors,
|
|
3068
3126
|
dlq,
|
|
@@ -3100,6 +3158,7 @@ async function startBatchConsumerImpl(ctx, topics, handleBatch, options = {}) {
|
|
|
3100
3158
|
dlq,
|
|
3101
3159
|
interceptors,
|
|
3102
3160
|
schemaMap,
|
|
3161
|
+
serdeMap,
|
|
3103
3162
|
assignmentTimeoutMs: options.retryTopicAssignmentTimeoutMs
|
|
3104
3163
|
});
|
|
3105
3164
|
}
|
|
@@ -3112,7 +3171,7 @@ async function startTransactionalConsumerImpl(ctx, topics, handler, options = {}
|
|
|
3112
3171
|
);
|
|
3113
3172
|
}
|
|
3114
3173
|
const setupOptions = { ...options, autoCommit: false };
|
|
3115
|
-
const { consumer, schemaMap, gid, readyPromise } = await setupConsumer(
|
|
3174
|
+
const { consumer, schemaMap, serdeMap, gid, readyPromise } = await setupConsumer(
|
|
3116
3175
|
ctx,
|
|
3117
3176
|
topics,
|
|
3118
3177
|
"eachMessage",
|
|
@@ -3129,7 +3188,8 @@ async function startTransactionalConsumerImpl(ctx, topics, handler, options = {}
|
|
|
3129
3188
|
schemaMap,
|
|
3130
3189
|
options.interceptors ?? [],
|
|
3131
3190
|
false,
|
|
3132
|
-
deps
|
|
3191
|
+
deps,
|
|
3192
|
+
serdeMap
|
|
3133
3193
|
);
|
|
3134
3194
|
const nextOffset = String(Number.parseInt(message.offset, 10) + 1);
|
|
3135
3195
|
if (envelope === null) {
|
|
@@ -3347,7 +3407,9 @@ async function startDelayedRelayImpl(ctx, topics, options) {
|
|
|
3347
3407
|
topic: target,
|
|
3348
3408
|
messages: [
|
|
3349
3409
|
{
|
|
3350
|
-
|
|
3410
|
+
// Forward the ORIGINAL wire bytes unchanged — no re-serialization,
|
|
3411
|
+
// so binary payloads (Avro/Protobuf) are relayed losslessly.
|
|
3412
|
+
value: message.value,
|
|
3351
3413
|
key: message.key ? message.key.toString() : null,
|
|
3352
3414
|
headers: forwardHeaders
|
|
3353
3415
|
}
|
|
@@ -3674,6 +3736,7 @@ var KafkaClient = class {
|
|
|
3674
3736
|
const consumers = /* @__PURE__ */ new Map();
|
|
3675
3737
|
const consumerCreationOptions = /* @__PURE__ */ new Map();
|
|
3676
3738
|
const schemaRegistry = /* @__PURE__ */ new Map();
|
|
3739
|
+
const serde = options?.serde ?? new JsonSerde();
|
|
3677
3740
|
const adminOps = new AdminOps({
|
|
3678
3741
|
admin: transport.admin(),
|
|
3679
3742
|
logger,
|
|
@@ -3700,6 +3763,7 @@ var KafkaClient = class {
|
|
|
3700
3763
|
autoCreateTopicsEnabled: options?.autoCreateTopics ?? false,
|
|
3701
3764
|
strictSchemasEnabled: options?.strictSchemas ?? true,
|
|
3702
3765
|
numPartitions: options?.numPartitions ?? 1,
|
|
3766
|
+
serde,
|
|
3703
3767
|
txId: options?.transactionalId ?? `${clientId}-tx`,
|
|
3704
3768
|
clockRecoveryTopics: options?.clockRecovery?.topics ?? [],
|
|
3705
3769
|
clockRecoveryTimeoutMs: options?.clockRecovery?.timeoutMs ?? 3e4,
|
|
@@ -3734,6 +3798,7 @@ var KafkaClient = class {
|
|
|
3734
3798
|
strictSchemasEnabled: options?.strictSchemas ?? true,
|
|
3735
3799
|
instrumentation: options?.instrumentation ?? [],
|
|
3736
3800
|
logger,
|
|
3801
|
+
serde,
|
|
3737
3802
|
nextLamportClock: () => 0
|
|
3738
3803
|
// patched below
|
|
3739
3804
|
},
|
|
@@ -3752,6 +3817,7 @@ var KafkaClient = class {
|
|
|
3752
3817
|
strictSchemasEnabled: options?.strictSchemas ?? true,
|
|
3753
3818
|
instrumentation: options?.instrumentation ?? [],
|
|
3754
3819
|
logger,
|
|
3820
|
+
serde,
|
|
3755
3821
|
nextLamportClock: () => ++ctx._lamportClock
|
|
3756
3822
|
};
|
|
3757
3823
|
ctx.retryTopicDeps = buildRetryTopicDeps(ctx);
|
|
@@ -4020,10 +4086,8 @@ function topic(name) {
|
|
|
4020
4086
|
function keyable(desc) {
|
|
4021
4087
|
return {
|
|
4022
4088
|
...desc,
|
|
4023
|
-
key: (extractor) => ({
|
|
4024
|
-
|
|
4025
|
-
__key: extractor
|
|
4026
|
-
})
|
|
4089
|
+
key: (extractor) => keyable({ ...desc, __key: extractor }),
|
|
4090
|
+
serde: (serde) => keyable({ ...desc, __serde: serde })
|
|
4027
4091
|
};
|
|
4028
4092
|
}
|
|
4029
4093
|
|
|
@@ -4068,6 +4132,12 @@ var SchemaRegistryClient = class {
|
|
|
4068
4132
|
fetchFn;
|
|
4069
4133
|
cacheTtlMs;
|
|
4070
4134
|
latestCache = /* @__PURE__ */ new Map();
|
|
4135
|
+
/**
|
|
4136
|
+
* `id → schema` cache. Schema ids are immutable in a Confluent-compatible
|
|
4137
|
+
* registry (a given id always maps to the same schema string), so entries
|
|
4138
|
+
* are cached for the lifetime of the client with no TTL.
|
|
4139
|
+
*/
|
|
4140
|
+
byIdCache = /* @__PURE__ */ new Map();
|
|
4071
4141
|
headers() {
|
|
4072
4142
|
const h = {
|
|
4073
4143
|
"Content-Type": "application/vnd.schemaregistry.v1+json"
|
|
@@ -4109,6 +4179,25 @@ var SchemaRegistryClient = class {
|
|
|
4109
4179
|
});
|
|
4110
4180
|
return value;
|
|
4111
4181
|
}
|
|
4182
|
+
/**
|
|
4183
|
+
* Fetch a schema by its globally unique registry id (`GET /schemas/ids/{id}`).
|
|
4184
|
+
*
|
|
4185
|
+
* Used by the Avro/Protobuf serdes on the deserialize path: the writer schema
|
|
4186
|
+
* id is read from the Confluent wire-format prefix, then resolved here. Results
|
|
4187
|
+
* are cached forever (schema ids are immutable), so a given id triggers exactly
|
|
4188
|
+
* one registry round-trip regardless of how many messages reference it.
|
|
4189
|
+
*/
|
|
4190
|
+
async getSchemaById(id) {
|
|
4191
|
+
const cached = this.byIdCache.get(id);
|
|
4192
|
+
if (cached) return cached;
|
|
4193
|
+
const raw = await this.request(
|
|
4194
|
+
"GET",
|
|
4195
|
+
`/schemas/ids/${id}`
|
|
4196
|
+
);
|
|
4197
|
+
const value = { id, schema: raw.schema, schemaType: raw.schemaType };
|
|
4198
|
+
this.byIdCache.set(id, value);
|
|
4199
|
+
return value;
|
|
4200
|
+
}
|
|
4112
4201
|
/** Fetch a specific schema version of a subject. */
|
|
4113
4202
|
async getSchemaVersion(subject, version) {
|
|
4114
4203
|
const raw = await this.request(
|
|
@@ -5050,6 +5139,7 @@ KafkaHealthIndicator = __decorateClass([
|
|
|
5050
5139
|
], KafkaHealthIndicator);
|
|
5051
5140
|
// Annotate the CommonJS export names for ESM import in node:
|
|
5052
5141
|
0 && (module.exports = {
|
|
5142
|
+
ConfluentTransport,
|
|
5053
5143
|
HEADER_CORRELATION_ID,
|
|
5054
5144
|
HEADER_DELAYED_TARGET,
|
|
5055
5145
|
HEADER_DELAYED_UNTIL,
|
|
@@ -5061,6 +5151,7 @@ KafkaHealthIndicator = __decorateClass([
|
|
|
5061
5151
|
InMemoryDedupStore,
|
|
5062
5152
|
InMemoryOutboxStore,
|
|
5063
5153
|
InjectKafkaClient,
|
|
5154
|
+
JsonSerde,
|
|
5064
5155
|
KAFKA_CLIENT,
|
|
5065
5156
|
KAFKA_SUBSCRIBER_METADATA,
|
|
5066
5157
|
KafkaClient,
|