@drarzter/kafka-client 0.6.7 → 0.6.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +84 -4
- package/dist/{chunk-ISYOEX4W.mjs → chunk-4526Y4PV.mjs} +340 -40
- package/dist/chunk-4526Y4PV.mjs.map +1 -0
- package/dist/core.d.mts +23 -7
- package/dist/core.d.ts +23 -7
- package/dist/core.js +339 -39
- package/dist/core.js.map +1 -1
- package/dist/core.mjs +1 -1
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +339 -39
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1 -1
- package/dist/otel.d.mts +1 -1
- package/dist/otel.d.ts +1 -1
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +11 -0
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +11 -0
- package/dist/testing.mjs.map +1 -1
- package/dist/{types-CqjRm-Cd.d.mts → types-736Gj0J3.d.mts} +79 -4
- package/dist/{types-CqjRm-Cd.d.ts → types-736Gj0J3.d.ts} +79 -4
- package/package.json +1 -1
- package/dist/chunk-ISYOEX4W.mjs.map +0 -1
package/README.md
CHANGED
|
@@ -33,6 +33,9 @@ Type-safe Kafka client for Node.js. Framework-agnostic core with a first-class N
|
|
|
33
33
|
- [Deduplication (Lamport Clock)](#deduplication-lamport-clock)
|
|
34
34
|
- [Retry topic chain](#retry-topic-chain)
|
|
35
35
|
- [stopConsumer](#stopconsumer)
|
|
36
|
+
- [Pause and resume](#pause-and-resume)
|
|
37
|
+
- [Reset consumer offsets](#reset-consumer-offsets)
|
|
38
|
+
- [DLQ replay](#dlq-replay)
|
|
36
39
|
- [Graceful shutdown](#graceful-shutdown)
|
|
37
40
|
- [Consumer handles](#consumer-handles)
|
|
38
41
|
- [onMessageLost](#onmessagelost)
|
|
@@ -776,12 +779,20 @@ const myInstrumentation: KafkaInstrumentation = {
|
|
|
776
779
|
`KafkaClient` maintains lightweight in-process event counters independently of any instrumentation:
|
|
777
780
|
|
|
778
781
|
```typescript
|
|
782
|
+
// Global snapshot — aggregate across all topics
|
|
779
783
|
const snapshot = kafka.getMetrics();
|
|
780
784
|
// { processedCount: number; retryCount: number; dlqCount: number; dedupCount: number }
|
|
781
785
|
|
|
782
|
-
|
|
786
|
+
// Per-topic snapshot
|
|
787
|
+
const orderMetrics = kafka.getMetrics('order.created');
|
|
788
|
+
// { processedCount: 5, retryCount: 1, dlqCount: 0, dedupCount: 0 }
|
|
789
|
+
|
|
790
|
+
kafka.resetMetrics(); // reset all counters
|
|
791
|
+
kafka.resetMetrics('order.created'); // reset only one topic's counters
|
|
783
792
|
```
|
|
784
793
|
|
|
794
|
+
Passing a topic name that has not seen any events returns a zero-valued snapshot — it never throws.
|
|
795
|
+
|
|
785
796
|
Counters are incremented in the same code paths that fire the corresponding hooks — they are always active regardless of whether any instrumentation is configured.
|
|
786
797
|
|
|
787
798
|
## Options reference
|
|
@@ -1025,7 +1036,7 @@ By default, retry is handled in-process: the consumer sleeps between attempts wh
|
|
|
1025
1036
|
|
|
1026
1037
|
Benefits over in-process retry:
|
|
1027
1038
|
|
|
1028
|
-
- **Durable** — retry messages survive a consumer restart; routing
|
|
1039
|
+
- **Durable** — retry messages survive a consumer restart; all routing (main → retry.1, level N → N+1, retry → DLQ) is exactly-once via Kafka transactions
|
|
1029
1040
|
- **Non-blocking** — the original consumer is free immediately; each level consumer only pauses its specific partition during the delay window, so other partitions continue processing
|
|
1030
1041
|
- **Isolated** — each retry level has its own consumer group, so a slow level 3 consumer never blocks a level 1 consumer
|
|
1031
1042
|
|
|
@@ -1057,9 +1068,9 @@ Each level consumer uses `consumer.pause → sleep(remaining) → consumer.resum
|
|
|
1057
1068
|
|
|
1058
1069
|
The retry topic messages carry scheduling headers (`x-retry-attempt`, `x-retry-after`, `x-retry-original-topic`, `x-retry-max-retries`) that each level consumer reads automatically — no manual configuration needed.
|
|
1059
1070
|
|
|
1060
|
-
> **Delivery guarantee:**
|
|
1071
|
+
> **Delivery guarantee:** the entire retry chain — including the **main consumer → retry.1** boundary — is **exactly-once**. Every routing step (main → retry.1, retry.N → retry.N+1, retry.N → DLQ) is wrapped in a Kafka transaction via `sendOffsetsToTransaction`: the produce and the consumer offset commit happen atomically. A crash at any point rolls back the transaction: the message is redelivered and the routing is retried, with no duplicate in the next level. If the EOS transaction fails (broker unavailable), the offset stays uncommitted and the message is safely redelivered — it is never lost.
|
|
1061
1072
|
>
|
|
1062
|
-
> The
|
|
1073
|
+
> The standard Kafka at-least-once guarantee still applies at the handler level: if your handler succeeds but the process crashes before the manual offset commit completes, the message is redelivered to the handler. Design handlers to be idempotent.
|
|
1063
1074
|
>
|
|
1064
1075
|
> **Startup validation:** `retryTopics` requires `retry` to be set — an error is thrown at startup if `retry` is missing. When `autoCreateTopics: false`, all `{topic}.retry.N` topics are validated to exist at startup and a clear error lists any missing ones. With `autoCreateTopics: true` the check is skipped — topics are created automatically by the `ensureTopic` path. Supported by both `startConsumer` and `startBatchConsumer`.
|
|
1065
1076
|
|
|
@@ -1079,6 +1090,75 @@ await kafka.stopConsumer();
|
|
|
1079
1090
|
|
|
1080
1091
|
`stopConsumer(groupId)` disconnects and removes only that group's consumer, leaving other groups running. Useful when you want to pause processing for a specific topic without restarting the whole client.
|
|
1081
1092
|
|
|
1093
|
+
## Pause and resume
|
|
1094
|
+
|
|
1095
|
+
Temporarily stop delivering messages from specific partitions without disconnecting the consumer:
|
|
1096
|
+
|
|
1097
|
+
```typescript
|
|
1098
|
+
// Pause partition 0 of 'orders' (default group)
|
|
1099
|
+
kafka.pauseConsumer(undefined, [{ topic: 'orders', partitions: [0] }]);
|
|
1100
|
+
|
|
1101
|
+
// Resume it later
|
|
1102
|
+
kafka.resumeConsumer(undefined, [{ topic: 'orders', partitions: [0] }]);
|
|
1103
|
+
|
|
1104
|
+
// Target a specific consumer group, multiple partitions
|
|
1105
|
+
kafka.pauseConsumer('payments-group', [{ topic: 'payments', partitions: [0, 1] }]);
|
|
1106
|
+
```
|
|
1107
|
+
|
|
1108
|
+
The first argument is the consumer group ID — pass `undefined` to target the default group. A warning is logged if the group is not found.
|
|
1109
|
+
|
|
1110
|
+
Pausing is non-destructive: the consumer stays connected and Kafka preserves the partition assignment for as long as the group session is alive. Messages accumulate in the topic and are delivered once the consumer resumes. Typical use: apply backpressure when a downstream dependency (e.g. a database) is temporarily overloaded.
|
|
1111
|
+
|
|
1112
|
+
## Reset consumer offsets
|
|
1113
|
+
|
|
1114
|
+
Seek a consumer group's committed offsets to the beginning or end of a topic:
|
|
1115
|
+
|
|
1116
|
+
```typescript
|
|
1117
|
+
// Seek to the beginning — re-process all existing messages
|
|
1118
|
+
await kafka.resetOffsets(undefined, 'orders', 'earliest');
|
|
1119
|
+
|
|
1120
|
+
// Seek to the end — skip existing messages, process only new ones
|
|
1121
|
+
await kafka.resetOffsets(undefined, 'orders', 'latest');
|
|
1122
|
+
|
|
1123
|
+
// Target a specific consumer group
|
|
1124
|
+
await kafka.resetOffsets('payments-group', 'orders', 'earliest');
|
|
1125
|
+
```
|
|
1126
|
+
|
|
1127
|
+
**Important:** the consumer for the specified group must be stopped before calling `resetOffsets`. An error is thrown if the group is currently running — this prevents the reset from racing with an active offset commit.
|
|
1128
|
+
|
|
1129
|
+
## DLQ replay
|
|
1130
|
+
|
|
1131
|
+
Re-publish messages from a dead letter queue back to the original topic:
|
|
1132
|
+
|
|
1133
|
+
```typescript
|
|
1134
|
+
// Re-publish all messages from 'orders.dlq' → 'orders'
|
|
1135
|
+
const result = await kafka.replayDlq('orders');
|
|
1136
|
+
// { replayed: 42, skipped: 0 }
|
|
1137
|
+
```
|
|
1138
|
+
|
|
1139
|
+
Options:
|
|
1140
|
+
|
|
1141
|
+
| Option | Default | Description |
|
|
1142
|
+
| ------ | ------- | ----------- |
|
|
1143
|
+
| `targetTopic` | `x-dlq-original-topic` header | Override the destination topic |
|
|
1144
|
+
| `dryRun` | `false` | Count messages without sending |
|
|
1145
|
+
| `filter` | — | `(headers) => boolean` — skip messages where the callback returns `false` |
|
|
1146
|
+
|
|
1147
|
+
```typescript
|
|
1148
|
+
// Dry run — see how many messages would be replayed
|
|
1149
|
+
const dry = await kafka.replayDlq('orders', { dryRun: true });
|
|
1150
|
+
|
|
1151
|
+
// Route to a different topic
|
|
1152
|
+
const result = await kafka.replayDlq('orders', { targetTopic: 'orders.v2' });
|
|
1153
|
+
|
|
1154
|
+
// Only replay messages with a specific correlation ID
|
|
1155
|
+
const filtered = await kafka.replayDlq('orders', {
|
|
1156
|
+
filter: (headers) => headers['x-correlation-id'] === 'corr-123',
|
|
1157
|
+
});
|
|
1158
|
+
```
|
|
1159
|
+
|
|
1160
|
+
`replayDlq` creates a temporary consumer group that reads the DLQ topic up to the high-watermark at the time of the call — messages published after replay starts are not included. DLQ metadata headers (`x-dlq-original-topic`, `x-dlq-error-message`, `x-dlq-error-stack`, `x-dlq-failed-at`, `x-dlq-attempt-count`) are stripped from the replayed messages; all other headers (e.g. `x-correlation-id`) are preserved.
|
|
1161
|
+
|
|
1082
1162
|
## Graceful shutdown
|
|
1083
1163
|
|
|
1084
1164
|
`disconnect()` now drains in-flight handlers before tearing down connections — no messages are silently cut off mid-processing.
|
|
@@ -498,6 +498,17 @@ async function executeWithRetry(fn, ctx, deps) {
|
|
|
498
498
|
deps.instrumentation
|
|
499
499
|
);
|
|
500
500
|
if (!error) {
|
|
501
|
+
if (deps.eosCommitOnSuccess) {
|
|
502
|
+
try {
|
|
503
|
+
await deps.eosCommitOnSuccess();
|
|
504
|
+
} catch (commitErr) {
|
|
505
|
+
deps.logger.error(
|
|
506
|
+
`EOS offset commit failed after successful handler \u2014 message will be redelivered:`,
|
|
507
|
+
toError(commitErr).stack
|
|
508
|
+
);
|
|
509
|
+
return;
|
|
510
|
+
}
|
|
511
|
+
}
|
|
501
512
|
for (const env of envelopes) deps.onMessage?.(env);
|
|
502
513
|
return;
|
|
503
514
|
}
|
|
@@ -516,16 +527,28 @@ async function executeWithRetry(fn, ctx, deps) {
|
|
|
516
527
|
if (retryTopics && retry) {
|
|
517
528
|
const cap = Math.min(backoffMs, maxBackoffMs);
|
|
518
529
|
const delay = Math.floor(Math.random() * cap);
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
530
|
+
if (deps.eosRouteToRetry) {
|
|
531
|
+
try {
|
|
532
|
+
await deps.eosRouteToRetry(rawMessages, envelopes, delay);
|
|
533
|
+
deps.onRetry?.(envelopes[0], 1, retry.maxRetries);
|
|
534
|
+
} catch (txErr) {
|
|
535
|
+
deps.logger.error(
|
|
536
|
+
`EOS routing to retry topic failed \u2014 message will be redelivered:`,
|
|
537
|
+
toError(txErr).stack
|
|
538
|
+
);
|
|
539
|
+
}
|
|
540
|
+
} else {
|
|
541
|
+
await sendToRetryTopic(
|
|
542
|
+
topic2,
|
|
543
|
+
rawMessages,
|
|
544
|
+
1,
|
|
545
|
+
retry.maxRetries,
|
|
546
|
+
delay,
|
|
547
|
+
isBatch ? envelopes.map((e) => e.headers) : envelopes[0]?.headers ?? {},
|
|
548
|
+
deps
|
|
549
|
+
);
|
|
550
|
+
deps.onRetry?.(envelopes[0], 1, retry.maxRetries);
|
|
551
|
+
}
|
|
529
552
|
} else if (isLastAttempt) {
|
|
530
553
|
if (dlq) {
|
|
531
554
|
for (let i = 0; i < rawMessages.length; i++) {
|
|
@@ -625,6 +648,43 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
625
648
|
timeoutMs,
|
|
626
649
|
wrapWithTimeout
|
|
627
650
|
} = opts;
|
|
651
|
+
const eos = opts.eosMainContext;
|
|
652
|
+
const nextOffsetStr = (parseInt(message.offset, 10) + 1).toString();
|
|
653
|
+
const commitOffset = eos ? async () => {
|
|
654
|
+
await eos.consumer.commitOffsets([
|
|
655
|
+
{ topic: topic2, partition, offset: nextOffsetStr }
|
|
656
|
+
]);
|
|
657
|
+
} : void 0;
|
|
658
|
+
const eosRouteToRetry = eos && retry ? async (rawMsgs, envelopes, delay) => {
|
|
659
|
+
const { topic: rtTopic, messages: rtMsgs } = buildRetryTopicPayload(
|
|
660
|
+
topic2,
|
|
661
|
+
rawMsgs,
|
|
662
|
+
1,
|
|
663
|
+
retry.maxRetries,
|
|
664
|
+
delay,
|
|
665
|
+
envelopes[0]?.headers ?? {}
|
|
666
|
+
);
|
|
667
|
+
const tx = await eos.txProducer.transaction();
|
|
668
|
+
try {
|
|
669
|
+
await tx.send({ topic: rtTopic, messages: rtMsgs });
|
|
670
|
+
await tx.sendOffsets({
|
|
671
|
+
consumer: eos.consumer,
|
|
672
|
+
topics: [
|
|
673
|
+
{
|
|
674
|
+
topic: topic2,
|
|
675
|
+
partitions: [{ partition, offset: nextOffsetStr }]
|
|
676
|
+
}
|
|
677
|
+
]
|
|
678
|
+
});
|
|
679
|
+
await tx.commit();
|
|
680
|
+
} catch (txErr) {
|
|
681
|
+
try {
|
|
682
|
+
await tx.abort();
|
|
683
|
+
} catch {
|
|
684
|
+
}
|
|
685
|
+
throw txErr;
|
|
686
|
+
}
|
|
687
|
+
} : void 0;
|
|
628
688
|
const envelope = await parseSingleMessage(
|
|
629
689
|
message,
|
|
630
690
|
topic2,
|
|
@@ -634,7 +694,10 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
634
694
|
dlq,
|
|
635
695
|
deps
|
|
636
696
|
);
|
|
637
|
-
if (envelope === null)
|
|
697
|
+
if (envelope === null) {
|
|
698
|
+
await commitOffset?.();
|
|
699
|
+
return;
|
|
700
|
+
}
|
|
638
701
|
if (opts.deduplication) {
|
|
639
702
|
const isDuplicate = await applyDeduplication(
|
|
640
703
|
envelope,
|
|
@@ -643,7 +706,10 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
643
706
|
dlq,
|
|
644
707
|
deps
|
|
645
708
|
);
|
|
646
|
-
if (isDuplicate)
|
|
709
|
+
if (isDuplicate) {
|
|
710
|
+
await commitOffset?.();
|
|
711
|
+
return;
|
|
712
|
+
}
|
|
647
713
|
}
|
|
648
714
|
await executeWithRetry(
|
|
649
715
|
() => {
|
|
@@ -664,7 +730,7 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
664
730
|
retry,
|
|
665
731
|
retryTopics
|
|
666
732
|
},
|
|
667
|
-
deps
|
|
733
|
+
{ ...deps, eosRouteToRetry, eosCommitOnSuccess: commitOffset }
|
|
668
734
|
);
|
|
669
735
|
}
|
|
670
736
|
async function handleEachBatch(payload, opts, deps) {
|
|
@@ -679,6 +745,50 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
679
745
|
timeoutMs,
|
|
680
746
|
wrapWithTimeout
|
|
681
747
|
} = opts;
|
|
748
|
+
const eos = opts.eosMainContext;
|
|
749
|
+
const lastRawOffset = batch.messages.length > 0 ? batch.messages[batch.messages.length - 1].offset : void 0;
|
|
750
|
+
const batchNextOffsetStr = lastRawOffset ? (parseInt(lastRawOffset, 10) + 1).toString() : void 0;
|
|
751
|
+
const commitBatchOffset = eos && batchNextOffsetStr ? async () => {
|
|
752
|
+
await eos.consumer.commitOffsets([
|
|
753
|
+
{
|
|
754
|
+
topic: batch.topic,
|
|
755
|
+
partition: batch.partition,
|
|
756
|
+
offset: batchNextOffsetStr
|
|
757
|
+
}
|
|
758
|
+
]);
|
|
759
|
+
} : void 0;
|
|
760
|
+
const eosRouteToRetry = eos && retry && batchNextOffsetStr ? async (rawMsgs, envelopes2, delay) => {
|
|
761
|
+
const { topic: rtTopic, messages: rtMsgs } = buildRetryTopicPayload(
|
|
762
|
+
batch.topic,
|
|
763
|
+
rawMsgs,
|
|
764
|
+
1,
|
|
765
|
+
retry.maxRetries,
|
|
766
|
+
delay,
|
|
767
|
+
envelopes2.map((e) => e.headers)
|
|
768
|
+
);
|
|
769
|
+
const tx = await eos.txProducer.transaction();
|
|
770
|
+
try {
|
|
771
|
+
await tx.send({ topic: rtTopic, messages: rtMsgs });
|
|
772
|
+
await tx.sendOffsets({
|
|
773
|
+
consumer: eos.consumer,
|
|
774
|
+
topics: [
|
|
775
|
+
{
|
|
776
|
+
topic: batch.topic,
|
|
777
|
+
partitions: [
|
|
778
|
+
{ partition: batch.partition, offset: batchNextOffsetStr }
|
|
779
|
+
]
|
|
780
|
+
}
|
|
781
|
+
]
|
|
782
|
+
});
|
|
783
|
+
await tx.commit();
|
|
784
|
+
} catch (txErr) {
|
|
785
|
+
try {
|
|
786
|
+
await tx.abort();
|
|
787
|
+
} catch {
|
|
788
|
+
}
|
|
789
|
+
throw txErr;
|
|
790
|
+
}
|
|
791
|
+
} : void 0;
|
|
682
792
|
const envelopes = [];
|
|
683
793
|
const rawMessages = [];
|
|
684
794
|
for (const message of batch.messages) {
|
|
@@ -706,7 +816,10 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
706
816
|
envelopes.push(envelope);
|
|
707
817
|
rawMessages.push(message.value.toString());
|
|
708
818
|
}
|
|
709
|
-
if (envelopes.length === 0)
|
|
819
|
+
if (envelopes.length === 0) {
|
|
820
|
+
await commitBatchOffset?.();
|
|
821
|
+
return;
|
|
822
|
+
}
|
|
710
823
|
const meta = {
|
|
711
824
|
partition: batch.partition,
|
|
712
825
|
highWatermark: batch.highWatermark,
|
|
@@ -728,7 +841,7 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
728
841
|
isBatch: true,
|
|
729
842
|
retryTopics
|
|
730
843
|
},
|
|
731
|
-
deps
|
|
844
|
+
{ ...deps, eosRouteToRetry, eosCommitOnSuccess: commitBatchOffset }
|
|
732
845
|
);
|
|
733
846
|
}
|
|
734
847
|
|
|
@@ -1009,7 +1122,7 @@ async function startRetryTopicConsumers(originalTopics, originalGroupId, handleM
|
|
|
1009
1122
|
// src/client/kafka.client/index.ts
|
|
1010
1123
|
var { Kafka: KafkaClass, logLevel: KafkaLogLevel } = KafkaJS;
|
|
1011
1124
|
var _activeTransactionalIds = /* @__PURE__ */ new Set();
|
|
1012
|
-
var KafkaClient = class {
|
|
1125
|
+
var KafkaClient = class _KafkaClient {
|
|
1013
1126
|
kafka;
|
|
1014
1127
|
producer;
|
|
1015
1128
|
txProducer;
|
|
@@ -1036,13 +1149,8 @@ var KafkaClient = class {
|
|
|
1036
1149
|
onRebalance;
|
|
1037
1150
|
/** Transactional producer ID — configurable via `KafkaClientOptions.transactionalId`. */
|
|
1038
1151
|
txId;
|
|
1039
|
-
/**
|
|
1040
|
-
|
|
1041
|
-
processedCount: 0,
|
|
1042
|
-
retryCount: 0,
|
|
1043
|
-
dlqCount: 0,
|
|
1044
|
-
dedupCount: 0
|
|
1045
|
-
};
|
|
1152
|
+
/** Per-topic event counters, lazily created on first event. Aggregated by `getMetrics()`. */
|
|
1153
|
+
_topicMetrics = /* @__PURE__ */ new Map();
|
|
1046
1154
|
/** Monotonically increasing Lamport clock stamped on every outgoing message. */
|
|
1047
1155
|
_lamportClock = 0;
|
|
1048
1156
|
/** Per-groupId deduplication state: `"topic:partition"` → last processed clock. */
|
|
@@ -1186,13 +1294,20 @@ var KafkaClient = class {
|
|
|
1186
1294
|
"retryTopics requires retry to be configured \u2014 set retry.maxRetries to enable the retry topic chain"
|
|
1187
1295
|
);
|
|
1188
1296
|
}
|
|
1189
|
-
const
|
|
1297
|
+
const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
|
|
1298
|
+
const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", setupOptions);
|
|
1190
1299
|
const deps = this.messageDeps;
|
|
1191
1300
|
const timeoutMs = options.handlerTimeoutMs;
|
|
1192
1301
|
const deduplication = this.resolveDeduplicationContext(
|
|
1193
1302
|
gid,
|
|
1194
1303
|
options.deduplication
|
|
1195
1304
|
);
|
|
1305
|
+
let eosMainContext;
|
|
1306
|
+
if (options.retryTopics && retry) {
|
|
1307
|
+
const mainTxId = `${gid}-main-tx`;
|
|
1308
|
+
const txProducer = await this.createRetryTxProducer(mainTxId);
|
|
1309
|
+
eosMainContext = { txProducer, consumer };
|
|
1310
|
+
}
|
|
1196
1311
|
await consumer.run({
|
|
1197
1312
|
eachMessage: (payload) => this.trackInFlight(
|
|
1198
1313
|
() => handleEachMessage(
|
|
@@ -1206,7 +1321,8 @@ var KafkaClient = class {
|
|
|
1206
1321
|
retryTopics: options.retryTopics,
|
|
1207
1322
|
timeoutMs,
|
|
1208
1323
|
wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this),
|
|
1209
|
-
deduplication
|
|
1324
|
+
deduplication,
|
|
1325
|
+
eosMainContext
|
|
1210
1326
|
},
|
|
1211
1327
|
deps
|
|
1212
1328
|
)
|
|
@@ -1238,18 +1354,26 @@ var KafkaClient = class {
|
|
|
1238
1354
|
"retryTopics requires retry to be configured \u2014 set retry.maxRetries to enable the retry topic chain"
|
|
1239
1355
|
);
|
|
1240
1356
|
}
|
|
1241
|
-
if (options.
|
|
1357
|
+
if (options.retryTopics) {
|
|
1358
|
+
} else if (options.autoCommit !== false) {
|
|
1242
1359
|
this.logger.debug?.(
|
|
1243
1360
|
`startBatchConsumer: autoCommit is enabled (default true). If your handler calls resolveOffset() or commitOffsetsIfNecessary(), set autoCommit: false to avoid offset conflicts.`
|
|
1244
1361
|
);
|
|
1245
1362
|
}
|
|
1246
|
-
const
|
|
1363
|
+
const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
|
|
1364
|
+
const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", setupOptions);
|
|
1247
1365
|
const deps = this.messageDeps;
|
|
1248
1366
|
const timeoutMs = options.handlerTimeoutMs;
|
|
1249
1367
|
const deduplication = this.resolveDeduplicationContext(
|
|
1250
1368
|
gid,
|
|
1251
1369
|
options.deduplication
|
|
1252
1370
|
);
|
|
1371
|
+
let eosMainContext;
|
|
1372
|
+
if (options.retryTopics && retry) {
|
|
1373
|
+
const mainTxId = `${gid}-main-tx`;
|
|
1374
|
+
const txProducer = await this.createRetryTxProducer(mainTxId);
|
|
1375
|
+
eosMainContext = { txProducer, consumer };
|
|
1376
|
+
}
|
|
1253
1377
|
await consumer.run({
|
|
1254
1378
|
eachBatch: (payload) => this.trackInFlight(
|
|
1255
1379
|
() => handleEachBatch(
|
|
@@ -1263,7 +1387,8 @@ var KafkaClient = class {
|
|
|
1263
1387
|
retryTopics: options.retryTopics,
|
|
1264
1388
|
timeoutMs,
|
|
1265
1389
|
wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this),
|
|
1266
|
-
deduplication
|
|
1390
|
+
deduplication,
|
|
1391
|
+
eosMainContext
|
|
1267
1392
|
},
|
|
1268
1393
|
deps
|
|
1269
1394
|
)
|
|
@@ -1320,6 +1445,18 @@ var KafkaClient = class {
|
|
|
1320
1445
|
this.consumerCreationOptions.delete(groupId);
|
|
1321
1446
|
this.dedupStates.delete(groupId);
|
|
1322
1447
|
this.logger.log(`Consumer disconnected: group "${groupId}"`);
|
|
1448
|
+
const mainTxId = `${groupId}-main-tx`;
|
|
1449
|
+
const mainTxProducer = this.retryTxProducers.get(mainTxId);
|
|
1450
|
+
if (mainTxProducer) {
|
|
1451
|
+
await mainTxProducer.disconnect().catch(
|
|
1452
|
+
(e) => this.logger.warn(
|
|
1453
|
+
`Error disconnecting main tx producer "${mainTxId}":`,
|
|
1454
|
+
toError(e).message
|
|
1455
|
+
)
|
|
1456
|
+
);
|
|
1457
|
+
_activeTransactionalIds.delete(mainTxId);
|
|
1458
|
+
this.retryTxProducers.delete(mainTxId);
|
|
1459
|
+
}
|
|
1323
1460
|
const companions = this.companionGroupIds.get(groupId) ?? [];
|
|
1324
1461
|
for (const cGroupId of companions) {
|
|
1325
1462
|
const cConsumer = this.consumers.get(cGroupId);
|
|
@@ -1370,6 +1507,144 @@ var KafkaClient = class {
|
|
|
1370
1507
|
this.logger.log("All consumers disconnected");
|
|
1371
1508
|
}
|
|
1372
1509
|
}
|
|
1510
|
+
pauseConsumer(groupId, assignments) {
|
|
1511
|
+
const gid = groupId ?? this.defaultGroupId;
|
|
1512
|
+
const consumer = this.consumers.get(gid);
|
|
1513
|
+
if (!consumer) {
|
|
1514
|
+
this.logger.warn(`pauseConsumer: no active consumer for group "${gid}"`);
|
|
1515
|
+
return;
|
|
1516
|
+
}
|
|
1517
|
+
consumer.pause(
|
|
1518
|
+
assignments.flatMap(
|
|
1519
|
+
({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] }))
|
|
1520
|
+
)
|
|
1521
|
+
);
|
|
1522
|
+
}
|
|
1523
|
+
resumeConsumer(groupId, assignments) {
|
|
1524
|
+
const gid = groupId ?? this.defaultGroupId;
|
|
1525
|
+
const consumer = this.consumers.get(gid);
|
|
1526
|
+
if (!consumer) {
|
|
1527
|
+
this.logger.warn(`resumeConsumer: no active consumer for group "${gid}"`);
|
|
1528
|
+
return;
|
|
1529
|
+
}
|
|
1530
|
+
consumer.resume(
|
|
1531
|
+
assignments.flatMap(
|
|
1532
|
+
({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] }))
|
|
1533
|
+
)
|
|
1534
|
+
);
|
|
1535
|
+
}
|
|
1536
|
+
/** DLQ header keys added by `sendToDlq` — stripped before re-publishing. */
|
|
1537
|
+
static DLQ_HEADER_KEYS = /* @__PURE__ */ new Set([
|
|
1538
|
+
"x-dlq-original-topic",
|
|
1539
|
+
"x-dlq-failed-at",
|
|
1540
|
+
"x-dlq-error-message",
|
|
1541
|
+
"x-dlq-error-stack",
|
|
1542
|
+
"x-dlq-attempt-count"
|
|
1543
|
+
]);
|
|
1544
|
+
async replayDlq(topic2, options = {}) {
|
|
1545
|
+
const dlqTopic = `${topic2}.dlq`;
|
|
1546
|
+
await this.ensureAdminConnected();
|
|
1547
|
+
const partitionOffsets = await this.admin.fetchTopicOffsets(dlqTopic);
|
|
1548
|
+
const activePartitions = partitionOffsets.filter(
|
|
1549
|
+
(p) => parseInt(p.high, 10) > 0
|
|
1550
|
+
);
|
|
1551
|
+
if (activePartitions.length === 0) {
|
|
1552
|
+
this.logger.log(`replayDlq: "${dlqTopic}" is empty \u2014 nothing to replay`);
|
|
1553
|
+
return { replayed: 0, skipped: 0 };
|
|
1554
|
+
}
|
|
1555
|
+
const highWatermarks = new Map(
|
|
1556
|
+
activePartitions.map(({ partition, high }) => [
|
|
1557
|
+
partition,
|
|
1558
|
+
parseInt(high, 10)
|
|
1559
|
+
])
|
|
1560
|
+
);
|
|
1561
|
+
const processedOffsets = /* @__PURE__ */ new Map();
|
|
1562
|
+
let replayed = 0;
|
|
1563
|
+
let skipped = 0;
|
|
1564
|
+
const tempGroupId = `${dlqTopic}-replay-${Date.now()}`;
|
|
1565
|
+
await new Promise((resolve, reject) => {
|
|
1566
|
+
const consumer = getOrCreateConsumer(
|
|
1567
|
+
tempGroupId,
|
|
1568
|
+
true,
|
|
1569
|
+
true,
|
|
1570
|
+
this.consumerOpsDeps
|
|
1571
|
+
);
|
|
1572
|
+
const cleanup = () => {
|
|
1573
|
+
consumer.disconnect().catch(() => {
|
|
1574
|
+
}).finally(() => {
|
|
1575
|
+
this.consumers.delete(tempGroupId);
|
|
1576
|
+
this.runningConsumers.delete(tempGroupId);
|
|
1577
|
+
this.consumerCreationOptions.delete(tempGroupId);
|
|
1578
|
+
});
|
|
1579
|
+
};
|
|
1580
|
+
consumer.connect().then(
|
|
1581
|
+
() => subscribeWithRetry(consumer, [dlqTopic], this.logger)
|
|
1582
|
+
).then(
|
|
1583
|
+
() => consumer.run({
|
|
1584
|
+
eachMessage: async ({ partition, message }) => {
|
|
1585
|
+
if (!message.value) return;
|
|
1586
|
+
const offset = parseInt(message.offset, 10);
|
|
1587
|
+
processedOffsets.set(partition, offset);
|
|
1588
|
+
const headers = decodeHeaders(message.headers);
|
|
1589
|
+
const targetTopic = options.targetTopic ?? headers["x-dlq-original-topic"];
|
|
1590
|
+
const originalHeaders = Object.fromEntries(
|
|
1591
|
+
Object.entries(headers).filter(
|
|
1592
|
+
([k]) => !_KafkaClient.DLQ_HEADER_KEYS.has(k)
|
|
1593
|
+
)
|
|
1594
|
+
);
|
|
1595
|
+
const value = message.value.toString();
|
|
1596
|
+
const shouldProcess = !options.filter || options.filter(headers, value);
|
|
1597
|
+
if (!targetTopic || !shouldProcess) {
|
|
1598
|
+
skipped++;
|
|
1599
|
+
} else if (options.dryRun) {
|
|
1600
|
+
this.logger.log(
|
|
1601
|
+
`[DLQ replay dry-run] Would replay to "${targetTopic}"`
|
|
1602
|
+
);
|
|
1603
|
+
replayed++;
|
|
1604
|
+
} else {
|
|
1605
|
+
await this.producer.send({
|
|
1606
|
+
topic: targetTopic,
|
|
1607
|
+
messages: [{ value, headers: originalHeaders }]
|
|
1608
|
+
});
|
|
1609
|
+
replayed++;
|
|
1610
|
+
}
|
|
1611
|
+
const allDone = Array.from(highWatermarks.entries()).every(
|
|
1612
|
+
([p, hwm]) => (processedOffsets.get(p) ?? -1) >= hwm - 1
|
|
1613
|
+
);
|
|
1614
|
+
if (allDone) {
|
|
1615
|
+
cleanup();
|
|
1616
|
+
resolve();
|
|
1617
|
+
}
|
|
1618
|
+
}
|
|
1619
|
+
})
|
|
1620
|
+
).catch((err) => {
|
|
1621
|
+
cleanup();
|
|
1622
|
+
reject(err);
|
|
1623
|
+
});
|
|
1624
|
+
});
|
|
1625
|
+
this.logger.log(
|
|
1626
|
+
`replayDlq: replayed ${replayed}, skipped ${skipped} from "${dlqTopic}"`
|
|
1627
|
+
);
|
|
1628
|
+
return { replayed, skipped };
|
|
1629
|
+
}
|
|
1630
|
+
async resetOffsets(groupId, topic2, position) {
|
|
1631
|
+
const gid = groupId ?? this.defaultGroupId;
|
|
1632
|
+
if (this.runningConsumers.has(gid)) {
|
|
1633
|
+
throw new Error(
|
|
1634
|
+
`resetOffsets: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before resetting offsets.`
|
|
1635
|
+
);
|
|
1636
|
+
}
|
|
1637
|
+
await this.ensureAdminConnected();
|
|
1638
|
+
const partitionOffsets = await this.admin.fetchTopicOffsets(topic2);
|
|
1639
|
+
const partitions = partitionOffsets.map(({ partition, low, high }) => ({
|
|
1640
|
+
partition,
|
|
1641
|
+
offset: position === "earliest" ? low : high
|
|
1642
|
+
}));
|
|
1643
|
+
await this.admin.setOffsets({ groupId: gid, topic: topic2, partitions });
|
|
1644
|
+
this.logger.log(
|
|
1645
|
+
`Offsets reset to ${position} for group "${gid}" on topic "${topic2}"`
|
|
1646
|
+
);
|
|
1647
|
+
}
|
|
1373
1648
|
/**
|
|
1374
1649
|
* Query consumer group lag per partition.
|
|
1375
1650
|
* Lag = broker high-watermark − last committed offset.
|
|
@@ -1420,14 +1695,31 @@ var KafkaClient = class {
|
|
|
1420
1695
|
getClientId() {
|
|
1421
1696
|
return this.clientId;
|
|
1422
1697
|
}
|
|
1423
|
-
getMetrics() {
|
|
1424
|
-
|
|
1698
|
+
getMetrics(topic2) {
|
|
1699
|
+
if (topic2 !== void 0) {
|
|
1700
|
+
const m = this._topicMetrics.get(topic2);
|
|
1701
|
+
return m ? { ...m } : { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
1702
|
+
}
|
|
1703
|
+
const agg = {
|
|
1704
|
+
processedCount: 0,
|
|
1705
|
+
retryCount: 0,
|
|
1706
|
+
dlqCount: 0,
|
|
1707
|
+
dedupCount: 0
|
|
1708
|
+
};
|
|
1709
|
+
for (const m of this._topicMetrics.values()) {
|
|
1710
|
+
agg.processedCount += m.processedCount;
|
|
1711
|
+
agg.retryCount += m.retryCount;
|
|
1712
|
+
agg.dlqCount += m.dlqCount;
|
|
1713
|
+
agg.dedupCount += m.dedupCount;
|
|
1714
|
+
}
|
|
1715
|
+
return agg;
|
|
1425
1716
|
}
|
|
1426
|
-
resetMetrics() {
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1717
|
+
resetMetrics(topic2) {
|
|
1718
|
+
if (topic2 !== void 0) {
|
|
1719
|
+
this._topicMetrics.delete(topic2);
|
|
1720
|
+
return;
|
|
1721
|
+
}
|
|
1722
|
+
this._topicMetrics.clear();
|
|
1431
1723
|
}
|
|
1432
1724
|
/** Gracefully disconnect producer, all consumers, and admin. */
|
|
1433
1725
|
async disconnect(drainTimeoutMs = 3e4) {
|
|
@@ -1537,26 +1829,34 @@ var KafkaClient = class {
|
|
|
1537
1829
|
}
|
|
1538
1830
|
}
|
|
1539
1831
|
}
|
|
1832
|
+
metricsFor(topic2) {
|
|
1833
|
+
let m = this._topicMetrics.get(topic2);
|
|
1834
|
+
if (!m) {
|
|
1835
|
+
m = { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
1836
|
+
this._topicMetrics.set(topic2, m);
|
|
1837
|
+
}
|
|
1838
|
+
return m;
|
|
1839
|
+
}
|
|
1540
1840
|
notifyRetry(envelope, attempt, maxRetries) {
|
|
1541
|
-
this.
|
|
1841
|
+
this.metricsFor(envelope.topic).retryCount++;
|
|
1542
1842
|
for (const inst of this.instrumentation) {
|
|
1543
1843
|
inst.onRetry?.(envelope, attempt, maxRetries);
|
|
1544
1844
|
}
|
|
1545
1845
|
}
|
|
1546
1846
|
notifyDlq(envelope, reason) {
|
|
1547
|
-
this.
|
|
1847
|
+
this.metricsFor(envelope.topic).dlqCount++;
|
|
1548
1848
|
for (const inst of this.instrumentation) {
|
|
1549
1849
|
inst.onDlq?.(envelope, reason);
|
|
1550
1850
|
}
|
|
1551
1851
|
}
|
|
1552
1852
|
notifyDuplicate(envelope, strategy) {
|
|
1553
|
-
this.
|
|
1853
|
+
this.metricsFor(envelope.topic).dedupCount++;
|
|
1554
1854
|
for (const inst of this.instrumentation) {
|
|
1555
1855
|
inst.onDuplicate?.(envelope, strategy);
|
|
1556
1856
|
}
|
|
1557
1857
|
}
|
|
1558
1858
|
notifyMessage(envelope) {
|
|
1559
|
-
this.
|
|
1859
|
+
this.metricsFor(envelope.topic).processedCount++;
|
|
1560
1860
|
for (const inst of this.instrumentation) {
|
|
1561
1861
|
inst.onMessage?.(envelope);
|
|
1562
1862
|
}
|
|
@@ -1843,4 +2143,4 @@ export {
|
|
|
1843
2143
|
KafkaClient,
|
|
1844
2144
|
topic
|
|
1845
2145
|
};
|
|
1846
|
-
//# sourceMappingURL=chunk-
|
|
2146
|
+
//# sourceMappingURL=chunk-4526Y4PV.mjs.map
|