@drarzter/kafka-client 0.6.6 → 0.6.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +126 -4
- package/dist/{chunk-KCUKXR6B.mjs → chunk-4526Y4PV.mjs} +458 -40
- package/dist/chunk-4526Y4PV.mjs.map +1 -0
- package/dist/core.d.mts +29 -3
- package/dist/core.d.ts +29 -3
- package/dist/core.js +457 -39
- package/dist/core.js.map +1 -1
- package/dist/core.mjs +1 -1
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +457 -39
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1 -1
- package/dist/index.mjs.map +1 -1
- package/dist/otel.d.mts +1 -1
- package/dist/otel.d.ts +1 -1
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +11 -0
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +11 -0
- package/dist/testing.mjs.map +1 -1
- package/dist/{types-CTwLrJVU.d.mts → types-736Gj0J3.d.mts} +142 -3
- package/dist/{types-CTwLrJVU.d.ts → types-736Gj0J3.d.ts} +142 -3
- package/package.json +1 -1
- package/dist/chunk-KCUKXR6B.mjs.map +0 -1
|
@@ -318,7 +318,10 @@ async function sendToDlq(topic2, rawMessage, deps, meta) {
|
|
|
318
318
|
deps.logger.warn(`Message sent to DLQ: ${payload.topic}`);
|
|
319
319
|
} catch (error) {
|
|
320
320
|
const err = toError(error);
|
|
321
|
-
deps.logger.error(
|
|
321
|
+
deps.logger.error(
|
|
322
|
+
`Failed to send message to DLQ ${payload.topic}:`,
|
|
323
|
+
err.stack
|
|
324
|
+
);
|
|
322
325
|
await deps.onMessageLost?.({
|
|
323
326
|
topic: topic2,
|
|
324
327
|
error: err,
|
|
@@ -333,14 +336,9 @@ var RETRY_HEADER_MAX_RETRIES = "x-retry-max-retries";
|
|
|
333
336
|
var RETRY_HEADER_ORIGINAL_TOPIC = "x-retry-original-topic";
|
|
334
337
|
function buildRetryTopicPayload(originalTopic, rawMessages, attempt, maxRetries, delayMs, originalHeaders) {
|
|
335
338
|
const retryTopic = `${originalTopic}.retry.${attempt}`;
|
|
339
|
+
const STRIP = /* @__PURE__ */ new Set([RETRY_HEADER_ATTEMPT, RETRY_HEADER_AFTER, RETRY_HEADER_MAX_RETRIES, RETRY_HEADER_ORIGINAL_TOPIC]);
|
|
336
340
|
function buildHeaders(hdr) {
|
|
337
|
-
const
|
|
338
|
-
[RETRY_HEADER_ATTEMPT]: _a,
|
|
339
|
-
[RETRY_HEADER_AFTER]: _b,
|
|
340
|
-
[RETRY_HEADER_MAX_RETRIES]: _c,
|
|
341
|
-
[RETRY_HEADER_ORIGINAL_TOPIC]: _d,
|
|
342
|
-
...userHeaders
|
|
343
|
-
} = hdr;
|
|
341
|
+
const userHeaders = Object.fromEntries(Object.entries(hdr).filter(([k]) => !STRIP.has(k)));
|
|
344
342
|
return {
|
|
345
343
|
...userHeaders,
|
|
346
344
|
[RETRY_HEADER_ATTEMPT]: String(attempt),
|
|
@@ -396,10 +394,18 @@ function buildDuplicateTopicPayload(sourceTopic, rawMessage, destinationTopic, m
|
|
|
396
394
|
"x-duplicate-incoming-clock": String(meta?.incomingClock ?? 0),
|
|
397
395
|
"x-duplicate-last-processed-clock": String(meta?.lastProcessedClock ?? 0)
|
|
398
396
|
};
|
|
399
|
-
return {
|
|
397
|
+
return {
|
|
398
|
+
topic: destinationTopic,
|
|
399
|
+
messages: [{ value: rawMessage, headers }]
|
|
400
|
+
};
|
|
400
401
|
}
|
|
401
402
|
async function sendToDuplicatesTopic(sourceTopic, rawMessage, destinationTopic, deps, meta) {
|
|
402
|
-
const payload = buildDuplicateTopicPayload(
|
|
403
|
+
const payload = buildDuplicateTopicPayload(
|
|
404
|
+
sourceTopic,
|
|
405
|
+
rawMessage,
|
|
406
|
+
destinationTopic,
|
|
407
|
+
meta
|
|
408
|
+
);
|
|
403
409
|
try {
|
|
404
410
|
await deps.producer.send(payload);
|
|
405
411
|
deps.logger.warn(`Duplicate message forwarded to ${destinationTopic}`);
|
|
@@ -491,7 +497,21 @@ async function executeWithRetry(fn, ctx, deps) {
|
|
|
491
497
|
interceptors,
|
|
492
498
|
deps.instrumentation
|
|
493
499
|
);
|
|
494
|
-
if (!error)
|
|
500
|
+
if (!error) {
|
|
501
|
+
if (deps.eosCommitOnSuccess) {
|
|
502
|
+
try {
|
|
503
|
+
await deps.eosCommitOnSuccess();
|
|
504
|
+
} catch (commitErr) {
|
|
505
|
+
deps.logger.error(
|
|
506
|
+
`EOS offset commit failed after successful handler \u2014 message will be redelivered:`,
|
|
507
|
+
toError(commitErr).stack
|
|
508
|
+
);
|
|
509
|
+
return;
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
for (const env of envelopes) deps.onMessage?.(env);
|
|
513
|
+
return;
|
|
514
|
+
}
|
|
495
515
|
const isLastAttempt = attempt === maxAttempts;
|
|
496
516
|
const reportedError = isLastAttempt && maxAttempts > 1 ? new KafkaRetryExhaustedError(
|
|
497
517
|
topic2,
|
|
@@ -507,15 +527,28 @@ async function executeWithRetry(fn, ctx, deps) {
|
|
|
507
527
|
if (retryTopics && retry) {
|
|
508
528
|
const cap = Math.min(backoffMs, maxBackoffMs);
|
|
509
529
|
const delay = Math.floor(Math.random() * cap);
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
530
|
+
if (deps.eosRouteToRetry) {
|
|
531
|
+
try {
|
|
532
|
+
await deps.eosRouteToRetry(rawMessages, envelopes, delay);
|
|
533
|
+
deps.onRetry?.(envelopes[0], 1, retry.maxRetries);
|
|
534
|
+
} catch (txErr) {
|
|
535
|
+
deps.logger.error(
|
|
536
|
+
`EOS routing to retry topic failed \u2014 message will be redelivered:`,
|
|
537
|
+
toError(txErr).stack
|
|
538
|
+
);
|
|
539
|
+
}
|
|
540
|
+
} else {
|
|
541
|
+
await sendToRetryTopic(
|
|
542
|
+
topic2,
|
|
543
|
+
rawMessages,
|
|
544
|
+
1,
|
|
545
|
+
retry.maxRetries,
|
|
546
|
+
delay,
|
|
547
|
+
isBatch ? envelopes.map((e) => e.headers) : envelopes[0]?.headers ?? {},
|
|
548
|
+
deps
|
|
549
|
+
);
|
|
550
|
+
deps.onRetry?.(envelopes[0], 1, retry.maxRetries);
|
|
551
|
+
}
|
|
519
552
|
} else if (isLastAttempt) {
|
|
520
553
|
if (dlq) {
|
|
521
554
|
for (let i = 0; i < rawMessages.length; i++) {
|
|
@@ -524,6 +557,7 @@ async function executeWithRetry(fn, ctx, deps) {
|
|
|
524
557
|
attempt,
|
|
525
558
|
originalHeaders: envelopes[i]?.headers
|
|
526
559
|
});
|
|
560
|
+
deps.onDlq?.(envelopes[i] ?? envelopes[0], "handler-error");
|
|
527
561
|
}
|
|
528
562
|
} else {
|
|
529
563
|
await deps.onMessageLost?.({
|
|
@@ -535,6 +569,7 @@ async function executeWithRetry(fn, ctx, deps) {
|
|
|
535
569
|
}
|
|
536
570
|
} else {
|
|
537
571
|
const cap = Math.min(backoffMs * 2 ** (attempt - 1), maxBackoffMs);
|
|
572
|
+
deps.onRetry?.(envelopes[0], attempt, maxAttempts - 1);
|
|
538
573
|
await sleep(Math.floor(Math.random() * cap));
|
|
539
574
|
}
|
|
540
575
|
}
|
|
@@ -558,6 +593,7 @@ async function applyDeduplication(envelope, raw, dedup, dlq, deps) {
|
|
|
558
593
|
deps.logger.warn(
|
|
559
594
|
`Duplicate message on ${envelope.topic}[${envelope.partition}]: clock=${incomingClock} <= last=${lastProcessedClock} \u2014 strategy=${strategy}`
|
|
560
595
|
);
|
|
596
|
+
deps.onDuplicate?.(envelope, strategy);
|
|
561
597
|
if (strategy === "dlq" && dlq) {
|
|
562
598
|
const augmentedHeaders = {
|
|
563
599
|
...envelope.headers,
|
|
@@ -612,6 +648,43 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
612
648
|
timeoutMs,
|
|
613
649
|
wrapWithTimeout
|
|
614
650
|
} = opts;
|
|
651
|
+
const eos = opts.eosMainContext;
|
|
652
|
+
const nextOffsetStr = (parseInt(message.offset, 10) + 1).toString();
|
|
653
|
+
const commitOffset = eos ? async () => {
|
|
654
|
+
await eos.consumer.commitOffsets([
|
|
655
|
+
{ topic: topic2, partition, offset: nextOffsetStr }
|
|
656
|
+
]);
|
|
657
|
+
} : void 0;
|
|
658
|
+
const eosRouteToRetry = eos && retry ? async (rawMsgs, envelopes, delay) => {
|
|
659
|
+
const { topic: rtTopic, messages: rtMsgs } = buildRetryTopicPayload(
|
|
660
|
+
topic2,
|
|
661
|
+
rawMsgs,
|
|
662
|
+
1,
|
|
663
|
+
retry.maxRetries,
|
|
664
|
+
delay,
|
|
665
|
+
envelopes[0]?.headers ?? {}
|
|
666
|
+
);
|
|
667
|
+
const tx = await eos.txProducer.transaction();
|
|
668
|
+
try {
|
|
669
|
+
await tx.send({ topic: rtTopic, messages: rtMsgs });
|
|
670
|
+
await tx.sendOffsets({
|
|
671
|
+
consumer: eos.consumer,
|
|
672
|
+
topics: [
|
|
673
|
+
{
|
|
674
|
+
topic: topic2,
|
|
675
|
+
partitions: [{ partition, offset: nextOffsetStr }]
|
|
676
|
+
}
|
|
677
|
+
]
|
|
678
|
+
});
|
|
679
|
+
await tx.commit();
|
|
680
|
+
} catch (txErr) {
|
|
681
|
+
try {
|
|
682
|
+
await tx.abort();
|
|
683
|
+
} catch {
|
|
684
|
+
}
|
|
685
|
+
throw txErr;
|
|
686
|
+
}
|
|
687
|
+
} : void 0;
|
|
615
688
|
const envelope = await parseSingleMessage(
|
|
616
689
|
message,
|
|
617
690
|
topic2,
|
|
@@ -621,7 +694,10 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
621
694
|
dlq,
|
|
622
695
|
deps
|
|
623
696
|
);
|
|
624
|
-
if (envelope === null)
|
|
697
|
+
if (envelope === null) {
|
|
698
|
+
await commitOffset?.();
|
|
699
|
+
return;
|
|
700
|
+
}
|
|
625
701
|
if (opts.deduplication) {
|
|
626
702
|
const isDuplicate = await applyDeduplication(
|
|
627
703
|
envelope,
|
|
@@ -630,7 +706,10 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
630
706
|
dlq,
|
|
631
707
|
deps
|
|
632
708
|
);
|
|
633
|
-
if (isDuplicate)
|
|
709
|
+
if (isDuplicate) {
|
|
710
|
+
await commitOffset?.();
|
|
711
|
+
return;
|
|
712
|
+
}
|
|
634
713
|
}
|
|
635
714
|
await executeWithRetry(
|
|
636
715
|
() => {
|
|
@@ -651,7 +730,7 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
651
730
|
retry,
|
|
652
731
|
retryTopics
|
|
653
732
|
},
|
|
654
|
-
deps
|
|
733
|
+
{ ...deps, eosRouteToRetry, eosCommitOnSuccess: commitOffset }
|
|
655
734
|
);
|
|
656
735
|
}
|
|
657
736
|
async function handleEachBatch(payload, opts, deps) {
|
|
@@ -666,6 +745,50 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
666
745
|
timeoutMs,
|
|
667
746
|
wrapWithTimeout
|
|
668
747
|
} = opts;
|
|
748
|
+
const eos = opts.eosMainContext;
|
|
749
|
+
const lastRawOffset = batch.messages.length > 0 ? batch.messages[batch.messages.length - 1].offset : void 0;
|
|
750
|
+
const batchNextOffsetStr = lastRawOffset ? (parseInt(lastRawOffset, 10) + 1).toString() : void 0;
|
|
751
|
+
const commitBatchOffset = eos && batchNextOffsetStr ? async () => {
|
|
752
|
+
await eos.consumer.commitOffsets([
|
|
753
|
+
{
|
|
754
|
+
topic: batch.topic,
|
|
755
|
+
partition: batch.partition,
|
|
756
|
+
offset: batchNextOffsetStr
|
|
757
|
+
}
|
|
758
|
+
]);
|
|
759
|
+
} : void 0;
|
|
760
|
+
const eosRouteToRetry = eos && retry && batchNextOffsetStr ? async (rawMsgs, envelopes2, delay) => {
|
|
761
|
+
const { topic: rtTopic, messages: rtMsgs } = buildRetryTopicPayload(
|
|
762
|
+
batch.topic,
|
|
763
|
+
rawMsgs,
|
|
764
|
+
1,
|
|
765
|
+
retry.maxRetries,
|
|
766
|
+
delay,
|
|
767
|
+
envelopes2.map((e) => e.headers)
|
|
768
|
+
);
|
|
769
|
+
const tx = await eos.txProducer.transaction();
|
|
770
|
+
try {
|
|
771
|
+
await tx.send({ topic: rtTopic, messages: rtMsgs });
|
|
772
|
+
await tx.sendOffsets({
|
|
773
|
+
consumer: eos.consumer,
|
|
774
|
+
topics: [
|
|
775
|
+
{
|
|
776
|
+
topic: batch.topic,
|
|
777
|
+
partitions: [
|
|
778
|
+
{ partition: batch.partition, offset: batchNextOffsetStr }
|
|
779
|
+
]
|
|
780
|
+
}
|
|
781
|
+
]
|
|
782
|
+
});
|
|
783
|
+
await tx.commit();
|
|
784
|
+
} catch (txErr) {
|
|
785
|
+
try {
|
|
786
|
+
await tx.abort();
|
|
787
|
+
} catch {
|
|
788
|
+
}
|
|
789
|
+
throw txErr;
|
|
790
|
+
}
|
|
791
|
+
} : void 0;
|
|
669
792
|
const envelopes = [];
|
|
670
793
|
const rawMessages = [];
|
|
671
794
|
for (const message of batch.messages) {
|
|
@@ -693,7 +816,10 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
693
816
|
envelopes.push(envelope);
|
|
694
817
|
rawMessages.push(message.value.toString());
|
|
695
818
|
}
|
|
696
|
-
if (envelopes.length === 0)
|
|
819
|
+
if (envelopes.length === 0) {
|
|
820
|
+
await commitBatchOffset?.();
|
|
821
|
+
return;
|
|
822
|
+
}
|
|
697
823
|
const meta = {
|
|
698
824
|
partition: batch.partition,
|
|
699
825
|
highWatermark: batch.highWatermark,
|
|
@@ -715,7 +841,7 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
715
841
|
isBatch: true,
|
|
716
842
|
retryTopics
|
|
717
843
|
},
|
|
718
|
-
deps
|
|
844
|
+
{ ...deps, eosRouteToRetry, eosCommitOnSuccess: commitBatchOffset }
|
|
719
845
|
);
|
|
720
846
|
}
|
|
721
847
|
|
|
@@ -761,6 +887,9 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
|
|
|
761
887
|
producer,
|
|
762
888
|
instrumentation,
|
|
763
889
|
onMessageLost,
|
|
890
|
+
onRetry,
|
|
891
|
+
onDlq,
|
|
892
|
+
onMessage,
|
|
764
893
|
ensureTopic,
|
|
765
894
|
getOrCreateConsumer: getOrCreateConsumer2,
|
|
766
895
|
runningConsumers,
|
|
@@ -842,6 +971,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
|
|
|
842
971
|
instrumentation
|
|
843
972
|
);
|
|
844
973
|
if (!error) {
|
|
974
|
+
onMessage?.(envelope);
|
|
845
975
|
await consumer.commitOffsets([nextOffset]);
|
|
846
976
|
return;
|
|
847
977
|
}
|
|
@@ -874,12 +1004,23 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
|
|
|
874
1004
|
await tx.send({ topic: rtTopic, messages: rtMsgs });
|
|
875
1005
|
await tx.sendOffsets({
|
|
876
1006
|
consumer,
|
|
877
|
-
topics: [
|
|
1007
|
+
topics: [
|
|
1008
|
+
{
|
|
1009
|
+
topic: nextOffset.topic,
|
|
1010
|
+
partitions: [
|
|
1011
|
+
{
|
|
1012
|
+
partition: nextOffset.partition,
|
|
1013
|
+
offset: nextOffset.offset
|
|
1014
|
+
}
|
|
1015
|
+
]
|
|
1016
|
+
}
|
|
1017
|
+
]
|
|
878
1018
|
});
|
|
879
1019
|
await tx.commit();
|
|
880
1020
|
logger.warn(
|
|
881
1021
|
`Message routed to ${rtTopic} (EOS, level ${nextLevel}/${currentMaxRetries})`
|
|
882
1022
|
);
|
|
1023
|
+
onRetry?.(envelope, nextLevel, currentMaxRetries);
|
|
883
1024
|
} catch (txErr) {
|
|
884
1025
|
try {
|
|
885
1026
|
await tx.abort();
|
|
@@ -907,10 +1048,21 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
|
|
|
907
1048
|
await tx.send({ topic: dTopic, messages: dMsgs });
|
|
908
1049
|
await tx.sendOffsets({
|
|
909
1050
|
consumer,
|
|
910
|
-
topics: [
|
|
1051
|
+
topics: [
|
|
1052
|
+
{
|
|
1053
|
+
topic: nextOffset.topic,
|
|
1054
|
+
partitions: [
|
|
1055
|
+
{
|
|
1056
|
+
partition: nextOffset.partition,
|
|
1057
|
+
offset: nextOffset.offset
|
|
1058
|
+
}
|
|
1059
|
+
]
|
|
1060
|
+
}
|
|
1061
|
+
]
|
|
911
1062
|
});
|
|
912
1063
|
await tx.commit();
|
|
913
1064
|
logger.warn(`Message sent to DLQ: ${dTopic} (EOS)`);
|
|
1065
|
+
onDlq?.(envelope, "handler-error");
|
|
914
1066
|
} catch (txErr) {
|
|
915
1067
|
try {
|
|
916
1068
|
await tx.abort();
|
|
@@ -934,7 +1086,12 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
|
|
|
934
1086
|
}
|
|
935
1087
|
});
|
|
936
1088
|
runningConsumers.set(levelGroupId, "eachMessage");
|
|
937
|
-
await waitForPartitionAssignment(
|
|
1089
|
+
await waitForPartitionAssignment(
|
|
1090
|
+
consumer,
|
|
1091
|
+
levelTopics,
|
|
1092
|
+
logger,
|
|
1093
|
+
assignmentTimeoutMs
|
|
1094
|
+
);
|
|
938
1095
|
logger.log(
|
|
939
1096
|
`Retry level ${level}/${retry.maxRetries} consumer started for: ${originalTopics.join(", ")} (group: ${levelGroupId})`
|
|
940
1097
|
);
|
|
@@ -964,7 +1121,8 @@ async function startRetryTopicConsumers(originalTopics, originalGroupId, handleM
|
|
|
964
1121
|
|
|
965
1122
|
// src/client/kafka.client/index.ts
|
|
966
1123
|
var { Kafka: KafkaClass, logLevel: KafkaLogLevel } = KafkaJS;
|
|
967
|
-
var
|
|
1124
|
+
var _activeTransactionalIds = /* @__PURE__ */ new Set();
|
|
1125
|
+
var KafkaClient = class _KafkaClient {
|
|
968
1126
|
kafka;
|
|
969
1127
|
producer;
|
|
970
1128
|
txProducer;
|
|
@@ -989,6 +1147,10 @@ var KafkaClient = class {
|
|
|
989
1147
|
instrumentation;
|
|
990
1148
|
onMessageLost;
|
|
991
1149
|
onRebalance;
|
|
1150
|
+
/** Transactional producer ID — configurable via `KafkaClientOptions.transactionalId`. */
|
|
1151
|
+
txId;
|
|
1152
|
+
/** Per-topic event counters, lazily created on first event. Aggregated by `getMetrics()`. */
|
|
1153
|
+
_topicMetrics = /* @__PURE__ */ new Map();
|
|
992
1154
|
/** Monotonically increasing Lamport clock stamped on every outgoing message. */
|
|
993
1155
|
_lamportClock = 0;
|
|
994
1156
|
/** Per-groupId deduplication state: `"topic:partition"` → last processed clock. */
|
|
@@ -1012,6 +1174,7 @@ var KafkaClient = class {
|
|
|
1012
1174
|
this.instrumentation = options?.instrumentation ?? [];
|
|
1013
1175
|
this.onMessageLost = options?.onMessageLost;
|
|
1014
1176
|
this.onRebalance = options?.onRebalance;
|
|
1177
|
+
this.txId = options?.transactionalId ?? `${clientId}-tx`;
|
|
1015
1178
|
this.kafka = new KafkaClass({
|
|
1016
1179
|
kafkaJS: {
|
|
1017
1180
|
clientId: this.clientId,
|
|
@@ -1048,16 +1211,22 @@ var KafkaClient = class {
|
|
|
1048
1211
|
/** Execute multiple sends atomically. Commits on success, aborts on error. */
|
|
1049
1212
|
async transaction(fn) {
|
|
1050
1213
|
if (!this.txProducerInitPromise) {
|
|
1214
|
+
if (_activeTransactionalIds.has(this.txId)) {
|
|
1215
|
+
this.logger.warn(
|
|
1216
|
+
`transactionalId "${this.txId}" is already in use by another KafkaClient in this process. Kafka will fence one of the producers. Set a unique \`transactionalId\` (or distinct \`clientId\`) per instance.`
|
|
1217
|
+
);
|
|
1218
|
+
}
|
|
1051
1219
|
const initPromise = (async () => {
|
|
1052
1220
|
const p = this.kafka.producer({
|
|
1053
1221
|
kafkaJS: {
|
|
1054
1222
|
acks: -1,
|
|
1055
1223
|
idempotent: true,
|
|
1056
|
-
transactionalId:
|
|
1224
|
+
transactionalId: this.txId,
|
|
1057
1225
|
maxInFlightRequests: 1
|
|
1058
1226
|
}
|
|
1059
1227
|
});
|
|
1060
1228
|
await p.connect();
|
|
1229
|
+
_activeTransactionalIds.add(this.txId);
|
|
1061
1230
|
return p;
|
|
1062
1231
|
})();
|
|
1063
1232
|
this.txProducerInitPromise = initPromise.catch((err) => {
|
|
@@ -1125,10 +1294,20 @@ var KafkaClient = class {
|
|
|
1125
1294
|
"retryTopics requires retry to be configured \u2014 set retry.maxRetries to enable the retry topic chain"
|
|
1126
1295
|
);
|
|
1127
1296
|
}
|
|
1128
|
-
const
|
|
1297
|
+
const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
|
|
1298
|
+
const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", setupOptions);
|
|
1129
1299
|
const deps = this.messageDeps;
|
|
1130
1300
|
const timeoutMs = options.handlerTimeoutMs;
|
|
1131
|
-
const deduplication = this.resolveDeduplicationContext(
|
|
1301
|
+
const deduplication = this.resolveDeduplicationContext(
|
|
1302
|
+
gid,
|
|
1303
|
+
options.deduplication
|
|
1304
|
+
);
|
|
1305
|
+
let eosMainContext;
|
|
1306
|
+
if (options.retryTopics && retry) {
|
|
1307
|
+
const mainTxId = `${gid}-main-tx`;
|
|
1308
|
+
const txProducer = await this.createRetryTxProducer(mainTxId);
|
|
1309
|
+
eosMainContext = { txProducer, consumer };
|
|
1310
|
+
}
|
|
1132
1311
|
await consumer.run({
|
|
1133
1312
|
eachMessage: (payload) => this.trackInFlight(
|
|
1134
1313
|
() => handleEachMessage(
|
|
@@ -1142,7 +1321,8 @@ var KafkaClient = class {
|
|
|
1142
1321
|
retryTopics: options.retryTopics,
|
|
1143
1322
|
timeoutMs,
|
|
1144
1323
|
wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this),
|
|
1145
|
-
deduplication
|
|
1324
|
+
deduplication,
|
|
1325
|
+
eosMainContext
|
|
1146
1326
|
},
|
|
1147
1327
|
deps
|
|
1148
1328
|
)
|
|
@@ -1174,15 +1354,26 @@ var KafkaClient = class {
|
|
|
1174
1354
|
"retryTopics requires retry to be configured \u2014 set retry.maxRetries to enable the retry topic chain"
|
|
1175
1355
|
);
|
|
1176
1356
|
}
|
|
1177
|
-
if (options.
|
|
1357
|
+
if (options.retryTopics) {
|
|
1358
|
+
} else if (options.autoCommit !== false) {
|
|
1178
1359
|
this.logger.debug?.(
|
|
1179
1360
|
`startBatchConsumer: autoCommit is enabled (default true). If your handler calls resolveOffset() or commitOffsetsIfNecessary(), set autoCommit: false to avoid offset conflicts.`
|
|
1180
1361
|
);
|
|
1181
1362
|
}
|
|
1182
|
-
const
|
|
1363
|
+
const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
|
|
1364
|
+
const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", setupOptions);
|
|
1183
1365
|
const deps = this.messageDeps;
|
|
1184
1366
|
const timeoutMs = options.handlerTimeoutMs;
|
|
1185
|
-
const deduplication = this.resolveDeduplicationContext(
|
|
1367
|
+
const deduplication = this.resolveDeduplicationContext(
|
|
1368
|
+
gid,
|
|
1369
|
+
options.deduplication
|
|
1370
|
+
);
|
|
1371
|
+
let eosMainContext;
|
|
1372
|
+
if (options.retryTopics && retry) {
|
|
1373
|
+
const mainTxId = `${gid}-main-tx`;
|
|
1374
|
+
const txProducer = await this.createRetryTxProducer(mainTxId);
|
|
1375
|
+
eosMainContext = { txProducer, consumer };
|
|
1376
|
+
}
|
|
1186
1377
|
await consumer.run({
|
|
1187
1378
|
eachBatch: (payload) => this.trackInFlight(
|
|
1188
1379
|
() => handleEachBatch(
|
|
@@ -1196,7 +1387,8 @@ var KafkaClient = class {
|
|
|
1196
1387
|
retryTopics: options.retryTopics,
|
|
1197
1388
|
timeoutMs,
|
|
1198
1389
|
wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this),
|
|
1199
|
-
deduplication
|
|
1390
|
+
deduplication,
|
|
1391
|
+
eosMainContext
|
|
1200
1392
|
},
|
|
1201
1393
|
deps
|
|
1202
1394
|
)
|
|
@@ -1209,7 +1401,7 @@ var KafkaClient = class {
|
|
|
1209
1401
|
}
|
|
1210
1402
|
const handleMessageForRetry = (env) => handleBatch([env], {
|
|
1211
1403
|
partition: env.partition,
|
|
1212
|
-
highWatermark:
|
|
1404
|
+
highWatermark: null,
|
|
1213
1405
|
heartbeat: async () => {
|
|
1214
1406
|
},
|
|
1215
1407
|
resolveOffset: () => {
|
|
@@ -1253,6 +1445,18 @@ var KafkaClient = class {
|
|
|
1253
1445
|
this.consumerCreationOptions.delete(groupId);
|
|
1254
1446
|
this.dedupStates.delete(groupId);
|
|
1255
1447
|
this.logger.log(`Consumer disconnected: group "${groupId}"`);
|
|
1448
|
+
const mainTxId = `${groupId}-main-tx`;
|
|
1449
|
+
const mainTxProducer = this.retryTxProducers.get(mainTxId);
|
|
1450
|
+
if (mainTxProducer) {
|
|
1451
|
+
await mainTxProducer.disconnect().catch(
|
|
1452
|
+
(e) => this.logger.warn(
|
|
1453
|
+
`Error disconnecting main tx producer "${mainTxId}":`,
|
|
1454
|
+
toError(e).message
|
|
1455
|
+
)
|
|
1456
|
+
);
|
|
1457
|
+
_activeTransactionalIds.delete(mainTxId);
|
|
1458
|
+
this.retryTxProducers.delete(mainTxId);
|
|
1459
|
+
}
|
|
1256
1460
|
const companions = this.companionGroupIds.get(groupId) ?? [];
|
|
1257
1461
|
for (const cGroupId of companions) {
|
|
1258
1462
|
const cConsumer = this.consumers.get(cGroupId);
|
|
@@ -1277,6 +1481,7 @@ var KafkaClient = class {
|
|
|
1277
1481
|
toError(e).message
|
|
1278
1482
|
)
|
|
1279
1483
|
);
|
|
1484
|
+
_activeTransactionalIds.delete(txId);
|
|
1280
1485
|
this.retryTxProducers.delete(txId);
|
|
1281
1486
|
}
|
|
1282
1487
|
}
|
|
@@ -1302,6 +1507,144 @@ var KafkaClient = class {
|
|
|
1302
1507
|
this.logger.log("All consumers disconnected");
|
|
1303
1508
|
}
|
|
1304
1509
|
}
|
|
1510
|
+
pauseConsumer(groupId, assignments) {
|
|
1511
|
+
const gid = groupId ?? this.defaultGroupId;
|
|
1512
|
+
const consumer = this.consumers.get(gid);
|
|
1513
|
+
if (!consumer) {
|
|
1514
|
+
this.logger.warn(`pauseConsumer: no active consumer for group "${gid}"`);
|
|
1515
|
+
return;
|
|
1516
|
+
}
|
|
1517
|
+
consumer.pause(
|
|
1518
|
+
assignments.flatMap(
|
|
1519
|
+
({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] }))
|
|
1520
|
+
)
|
|
1521
|
+
);
|
|
1522
|
+
}
|
|
1523
|
+
resumeConsumer(groupId, assignments) {
|
|
1524
|
+
const gid = groupId ?? this.defaultGroupId;
|
|
1525
|
+
const consumer = this.consumers.get(gid);
|
|
1526
|
+
if (!consumer) {
|
|
1527
|
+
this.logger.warn(`resumeConsumer: no active consumer for group "${gid}"`);
|
|
1528
|
+
return;
|
|
1529
|
+
}
|
|
1530
|
+
consumer.resume(
|
|
1531
|
+
assignments.flatMap(
|
|
1532
|
+
({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] }))
|
|
1533
|
+
)
|
|
1534
|
+
);
|
|
1535
|
+
}
|
|
1536
|
+
/** DLQ header keys added by `sendToDlq` — stripped before re-publishing. */
|
|
1537
|
+
static DLQ_HEADER_KEYS = /* @__PURE__ */ new Set([
|
|
1538
|
+
"x-dlq-original-topic",
|
|
1539
|
+
"x-dlq-failed-at",
|
|
1540
|
+
"x-dlq-error-message",
|
|
1541
|
+
"x-dlq-error-stack",
|
|
1542
|
+
"x-dlq-attempt-count"
|
|
1543
|
+
]);
|
|
1544
|
+
async replayDlq(topic2, options = {}) {
|
|
1545
|
+
const dlqTopic = `${topic2}.dlq`;
|
|
1546
|
+
await this.ensureAdminConnected();
|
|
1547
|
+
const partitionOffsets = await this.admin.fetchTopicOffsets(dlqTopic);
|
|
1548
|
+
const activePartitions = partitionOffsets.filter(
|
|
1549
|
+
(p) => parseInt(p.high, 10) > 0
|
|
1550
|
+
);
|
|
1551
|
+
if (activePartitions.length === 0) {
|
|
1552
|
+
this.logger.log(`replayDlq: "${dlqTopic}" is empty \u2014 nothing to replay`);
|
|
1553
|
+
return { replayed: 0, skipped: 0 };
|
|
1554
|
+
}
|
|
1555
|
+
const highWatermarks = new Map(
|
|
1556
|
+
activePartitions.map(({ partition, high }) => [
|
|
1557
|
+
partition,
|
|
1558
|
+
parseInt(high, 10)
|
|
1559
|
+
])
|
|
1560
|
+
);
|
|
1561
|
+
const processedOffsets = /* @__PURE__ */ new Map();
|
|
1562
|
+
let replayed = 0;
|
|
1563
|
+
let skipped = 0;
|
|
1564
|
+
const tempGroupId = `${dlqTopic}-replay-${Date.now()}`;
|
|
1565
|
+
await new Promise((resolve, reject) => {
|
|
1566
|
+
const consumer = getOrCreateConsumer(
|
|
1567
|
+
tempGroupId,
|
|
1568
|
+
true,
|
|
1569
|
+
true,
|
|
1570
|
+
this.consumerOpsDeps
|
|
1571
|
+
);
|
|
1572
|
+
const cleanup = () => {
|
|
1573
|
+
consumer.disconnect().catch(() => {
|
|
1574
|
+
}).finally(() => {
|
|
1575
|
+
this.consumers.delete(tempGroupId);
|
|
1576
|
+
this.runningConsumers.delete(tempGroupId);
|
|
1577
|
+
this.consumerCreationOptions.delete(tempGroupId);
|
|
1578
|
+
});
|
|
1579
|
+
};
|
|
1580
|
+
consumer.connect().then(
|
|
1581
|
+
() => subscribeWithRetry(consumer, [dlqTopic], this.logger)
|
|
1582
|
+
).then(
|
|
1583
|
+
() => consumer.run({
|
|
1584
|
+
eachMessage: async ({ partition, message }) => {
|
|
1585
|
+
if (!message.value) return;
|
|
1586
|
+
const offset = parseInt(message.offset, 10);
|
|
1587
|
+
processedOffsets.set(partition, offset);
|
|
1588
|
+
const headers = decodeHeaders(message.headers);
|
|
1589
|
+
const targetTopic = options.targetTopic ?? headers["x-dlq-original-topic"];
|
|
1590
|
+
const originalHeaders = Object.fromEntries(
|
|
1591
|
+
Object.entries(headers).filter(
|
|
1592
|
+
([k]) => !_KafkaClient.DLQ_HEADER_KEYS.has(k)
|
|
1593
|
+
)
|
|
1594
|
+
);
|
|
1595
|
+
const value = message.value.toString();
|
|
1596
|
+
const shouldProcess = !options.filter || options.filter(headers, value);
|
|
1597
|
+
if (!targetTopic || !shouldProcess) {
|
|
1598
|
+
skipped++;
|
|
1599
|
+
} else if (options.dryRun) {
|
|
1600
|
+
this.logger.log(
|
|
1601
|
+
`[DLQ replay dry-run] Would replay to "${targetTopic}"`
|
|
1602
|
+
);
|
|
1603
|
+
replayed++;
|
|
1604
|
+
} else {
|
|
1605
|
+
await this.producer.send({
|
|
1606
|
+
topic: targetTopic,
|
|
1607
|
+
messages: [{ value, headers: originalHeaders }]
|
|
1608
|
+
});
|
|
1609
|
+
replayed++;
|
|
1610
|
+
}
|
|
1611
|
+
const allDone = Array.from(highWatermarks.entries()).every(
|
|
1612
|
+
([p, hwm]) => (processedOffsets.get(p) ?? -1) >= hwm - 1
|
|
1613
|
+
);
|
|
1614
|
+
if (allDone) {
|
|
1615
|
+
cleanup();
|
|
1616
|
+
resolve();
|
|
1617
|
+
}
|
|
1618
|
+
}
|
|
1619
|
+
})
|
|
1620
|
+
).catch((err) => {
|
|
1621
|
+
cleanup();
|
|
1622
|
+
reject(err);
|
|
1623
|
+
});
|
|
1624
|
+
});
|
|
1625
|
+
this.logger.log(
|
|
1626
|
+
`replayDlq: replayed ${replayed}, skipped ${skipped} from "${dlqTopic}"`
|
|
1627
|
+
);
|
|
1628
|
+
return { replayed, skipped };
|
|
1629
|
+
}
|
|
1630
|
+
async resetOffsets(groupId, topic2, position) {
|
|
1631
|
+
const gid = groupId ?? this.defaultGroupId;
|
|
1632
|
+
if (this.runningConsumers.has(gid)) {
|
|
1633
|
+
throw new Error(
|
|
1634
|
+
`resetOffsets: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before resetting offsets.`
|
|
1635
|
+
);
|
|
1636
|
+
}
|
|
1637
|
+
await this.ensureAdminConnected();
|
|
1638
|
+
const partitionOffsets = await this.admin.fetchTopicOffsets(topic2);
|
|
1639
|
+
const partitions = partitionOffsets.map(({ partition, low, high }) => ({
|
|
1640
|
+
partition,
|
|
1641
|
+
offset: position === "earliest" ? low : high
|
|
1642
|
+
}));
|
|
1643
|
+
await this.admin.setOffsets({ groupId: gid, topic: topic2, partitions });
|
|
1644
|
+
this.logger.log(
|
|
1645
|
+
`Offsets reset to ${position} for group "${gid}" on topic "${topic2}"`
|
|
1646
|
+
);
|
|
1647
|
+
}
|
|
1305
1648
|
/**
|
|
1306
1649
|
* Query consumer group lag per partition.
|
|
1307
1650
|
* Lag = broker high-watermark − last committed offset.
|
|
@@ -1352,15 +1695,45 @@ var KafkaClient = class {
|
|
|
1352
1695
|
getClientId() {
|
|
1353
1696
|
return this.clientId;
|
|
1354
1697
|
}
|
|
1698
|
+
getMetrics(topic2) {
|
|
1699
|
+
if (topic2 !== void 0) {
|
|
1700
|
+
const m = this._topicMetrics.get(topic2);
|
|
1701
|
+
return m ? { ...m } : { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
1702
|
+
}
|
|
1703
|
+
const agg = {
|
|
1704
|
+
processedCount: 0,
|
|
1705
|
+
retryCount: 0,
|
|
1706
|
+
dlqCount: 0,
|
|
1707
|
+
dedupCount: 0
|
|
1708
|
+
};
|
|
1709
|
+
for (const m of this._topicMetrics.values()) {
|
|
1710
|
+
agg.processedCount += m.processedCount;
|
|
1711
|
+
agg.retryCount += m.retryCount;
|
|
1712
|
+
agg.dlqCount += m.dlqCount;
|
|
1713
|
+
agg.dedupCount += m.dedupCount;
|
|
1714
|
+
}
|
|
1715
|
+
return agg;
|
|
1716
|
+
}
|
|
1717
|
+
resetMetrics(topic2) {
|
|
1718
|
+
if (topic2 !== void 0) {
|
|
1719
|
+
this._topicMetrics.delete(topic2);
|
|
1720
|
+
return;
|
|
1721
|
+
}
|
|
1722
|
+
this._topicMetrics.clear();
|
|
1723
|
+
}
|
|
1355
1724
|
/** Gracefully disconnect producer, all consumers, and admin. */
|
|
1356
1725
|
async disconnect(drainTimeoutMs = 3e4) {
|
|
1357
1726
|
await this.waitForDrain(drainTimeoutMs);
|
|
1358
1727
|
const tasks = [this.producer.disconnect()];
|
|
1359
1728
|
if (this.txProducer) {
|
|
1360
1729
|
tasks.push(this.txProducer.disconnect());
|
|
1730
|
+
_activeTransactionalIds.delete(this.txId);
|
|
1361
1731
|
this.txProducer = void 0;
|
|
1362
1732
|
this.txProducerInitPromise = void 0;
|
|
1363
1733
|
}
|
|
1734
|
+
for (const txId of this.retryTxProducers.keys()) {
|
|
1735
|
+
_activeTransactionalIds.delete(txId);
|
|
1736
|
+
}
|
|
1364
1737
|
for (const p of this.retryTxProducers.values()) {
|
|
1365
1738
|
tasks.push(p.disconnect());
|
|
1366
1739
|
}
|
|
@@ -1456,6 +1829,38 @@ var KafkaClient = class {
|
|
|
1456
1829
|
}
|
|
1457
1830
|
}
|
|
1458
1831
|
}
|
|
1832
|
+
metricsFor(topic2) {
|
|
1833
|
+
let m = this._topicMetrics.get(topic2);
|
|
1834
|
+
if (!m) {
|
|
1835
|
+
m = { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
1836
|
+
this._topicMetrics.set(topic2, m);
|
|
1837
|
+
}
|
|
1838
|
+
return m;
|
|
1839
|
+
}
|
|
1840
|
+
notifyRetry(envelope, attempt, maxRetries) {
|
|
1841
|
+
this.metricsFor(envelope.topic).retryCount++;
|
|
1842
|
+
for (const inst of this.instrumentation) {
|
|
1843
|
+
inst.onRetry?.(envelope, attempt, maxRetries);
|
|
1844
|
+
}
|
|
1845
|
+
}
|
|
1846
|
+
notifyDlq(envelope, reason) {
|
|
1847
|
+
this.metricsFor(envelope.topic).dlqCount++;
|
|
1848
|
+
for (const inst of this.instrumentation) {
|
|
1849
|
+
inst.onDlq?.(envelope, reason);
|
|
1850
|
+
}
|
|
1851
|
+
}
|
|
1852
|
+
notifyDuplicate(envelope, strategy) {
|
|
1853
|
+
this.metricsFor(envelope.topic).dedupCount++;
|
|
1854
|
+
for (const inst of this.instrumentation) {
|
|
1855
|
+
inst.onDuplicate?.(envelope, strategy);
|
|
1856
|
+
}
|
|
1857
|
+
}
|
|
1858
|
+
notifyMessage(envelope) {
|
|
1859
|
+
this.metricsFor(envelope.topic).processedCount++;
|
|
1860
|
+
for (const inst of this.instrumentation) {
|
|
1861
|
+
inst.onMessage?.(envelope);
|
|
1862
|
+
}
|
|
1863
|
+
}
|
|
1459
1864
|
/**
|
|
1460
1865
|
* Start a timer that logs a warning if `fn` hasn't resolved within `timeoutMs`.
|
|
1461
1866
|
* The handler itself is not cancelled — the warning is diagnostic only.
|
|
@@ -1545,6 +1950,11 @@ var KafkaClient = class {
|
|
|
1545
1950
|
* so Kafka can fence stale producers on restart without affecting other levels.
|
|
1546
1951
|
*/
|
|
1547
1952
|
async createRetryTxProducer(transactionalId) {
|
|
1953
|
+
if (_activeTransactionalIds.has(transactionalId)) {
|
|
1954
|
+
this.logger.warn(
|
|
1955
|
+
`transactionalId "${transactionalId}" is already in use by another KafkaClient in this process. Kafka will fence one of the producers. Set a unique \`transactionalId\` (or distinct \`clientId\`) per instance.`
|
|
1956
|
+
);
|
|
1957
|
+
}
|
|
1548
1958
|
const p = this.kafka.producer({
|
|
1549
1959
|
kafkaJS: {
|
|
1550
1960
|
acks: -1,
|
|
@@ -1554,6 +1964,7 @@ var KafkaClient = class {
|
|
|
1554
1964
|
}
|
|
1555
1965
|
});
|
|
1556
1966
|
await p.connect();
|
|
1967
|
+
_activeTransactionalIds.add(transactionalId);
|
|
1557
1968
|
this.retryTxProducers.set(transactionalId, p);
|
|
1558
1969
|
return p;
|
|
1559
1970
|
}
|
|
@@ -1674,7 +2085,11 @@ var KafkaClient = class {
|
|
|
1674
2085
|
logger: this.logger,
|
|
1675
2086
|
producer: this.producer,
|
|
1676
2087
|
instrumentation: this.instrumentation,
|
|
1677
|
-
onMessageLost: this.onMessageLost
|
|
2088
|
+
onMessageLost: this.onMessageLost,
|
|
2089
|
+
onRetry: this.notifyRetry.bind(this),
|
|
2090
|
+
onDlq: this.notifyDlq.bind(this),
|
|
2091
|
+
onDuplicate: this.notifyDuplicate.bind(this),
|
|
2092
|
+
onMessage: this.notifyMessage.bind(this)
|
|
1678
2093
|
};
|
|
1679
2094
|
}
|
|
1680
2095
|
get retryTopicDeps() {
|
|
@@ -1683,6 +2098,9 @@ var KafkaClient = class {
|
|
|
1683
2098
|
producer: this.producer,
|
|
1684
2099
|
instrumentation: this.instrumentation,
|
|
1685
2100
|
onMessageLost: this.onMessageLost,
|
|
2101
|
+
onRetry: this.notifyRetry.bind(this),
|
|
2102
|
+
onDlq: this.notifyDlq.bind(this),
|
|
2103
|
+
onMessage: this.notifyMessage.bind(this),
|
|
1686
2104
|
ensureTopic: (t) => this.ensureTopic(t),
|
|
1687
2105
|
getOrCreateConsumer: (gid, fb, ac) => getOrCreateConsumer(gid, fb, ac, this.consumerOpsDeps),
|
|
1688
2106
|
runningConsumers: this.runningConsumers,
|
|
@@ -1725,4 +2143,4 @@ export {
|
|
|
1725
2143
|
KafkaClient,
|
|
1726
2144
|
topic
|
|
1727
2145
|
};
|
|
1728
|
-
//# sourceMappingURL=chunk-
|
|
2146
|
+
//# sourceMappingURL=chunk-4526Y4PV.mjs.map
|