@drarzter/kafka-client 0.6.6 → 0.6.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -318,7 +318,10 @@ async function sendToDlq(topic2, rawMessage, deps, meta) {
318
318
  deps.logger.warn(`Message sent to DLQ: ${payload.topic}`);
319
319
  } catch (error) {
320
320
  const err = toError(error);
321
- deps.logger.error(`Failed to send message to DLQ ${payload.topic}:`, err.stack);
321
+ deps.logger.error(
322
+ `Failed to send message to DLQ ${payload.topic}:`,
323
+ err.stack
324
+ );
322
325
  await deps.onMessageLost?.({
323
326
  topic: topic2,
324
327
  error: err,
@@ -333,14 +336,9 @@ var RETRY_HEADER_MAX_RETRIES = "x-retry-max-retries";
333
336
  var RETRY_HEADER_ORIGINAL_TOPIC = "x-retry-original-topic";
334
337
  function buildRetryTopicPayload(originalTopic, rawMessages, attempt, maxRetries, delayMs, originalHeaders) {
335
338
  const retryTopic = `${originalTopic}.retry.${attempt}`;
339
+ const STRIP = /* @__PURE__ */ new Set([RETRY_HEADER_ATTEMPT, RETRY_HEADER_AFTER, RETRY_HEADER_MAX_RETRIES, RETRY_HEADER_ORIGINAL_TOPIC]);
336
340
  function buildHeaders(hdr) {
337
- const {
338
- [RETRY_HEADER_ATTEMPT]: _a,
339
- [RETRY_HEADER_AFTER]: _b,
340
- [RETRY_HEADER_MAX_RETRIES]: _c,
341
- [RETRY_HEADER_ORIGINAL_TOPIC]: _d,
342
- ...userHeaders
343
- } = hdr;
341
+ const userHeaders = Object.fromEntries(Object.entries(hdr).filter(([k]) => !STRIP.has(k)));
344
342
  return {
345
343
  ...userHeaders,
346
344
  [RETRY_HEADER_ATTEMPT]: String(attempt),
@@ -396,10 +394,18 @@ function buildDuplicateTopicPayload(sourceTopic, rawMessage, destinationTopic, m
396
394
  "x-duplicate-incoming-clock": String(meta?.incomingClock ?? 0),
397
395
  "x-duplicate-last-processed-clock": String(meta?.lastProcessedClock ?? 0)
398
396
  };
399
- return { topic: destinationTopic, messages: [{ value: rawMessage, headers }] };
397
+ return {
398
+ topic: destinationTopic,
399
+ messages: [{ value: rawMessage, headers }]
400
+ };
400
401
  }
401
402
  async function sendToDuplicatesTopic(sourceTopic, rawMessage, destinationTopic, deps, meta) {
402
- const payload = buildDuplicateTopicPayload(sourceTopic, rawMessage, destinationTopic, meta);
403
+ const payload = buildDuplicateTopicPayload(
404
+ sourceTopic,
405
+ rawMessage,
406
+ destinationTopic,
407
+ meta
408
+ );
403
409
  try {
404
410
  await deps.producer.send(payload);
405
411
  deps.logger.warn(`Duplicate message forwarded to ${destinationTopic}`);
@@ -491,7 +497,21 @@ async function executeWithRetry(fn, ctx, deps) {
491
497
  interceptors,
492
498
  deps.instrumentation
493
499
  );
494
- if (!error) return;
500
+ if (!error) {
501
+ if (deps.eosCommitOnSuccess) {
502
+ try {
503
+ await deps.eosCommitOnSuccess();
504
+ } catch (commitErr) {
505
+ deps.logger.error(
506
+ `EOS offset commit failed after successful handler \u2014 message will be redelivered:`,
507
+ toError(commitErr).stack
508
+ );
509
+ return;
510
+ }
511
+ }
512
+ for (const env of envelopes) deps.onMessage?.(env);
513
+ return;
514
+ }
495
515
  const isLastAttempt = attempt === maxAttempts;
496
516
  const reportedError = isLastAttempt && maxAttempts > 1 ? new KafkaRetryExhaustedError(
497
517
  topic2,
@@ -507,15 +527,28 @@ async function executeWithRetry(fn, ctx, deps) {
507
527
  if (retryTopics && retry) {
508
528
  const cap = Math.min(backoffMs, maxBackoffMs);
509
529
  const delay = Math.floor(Math.random() * cap);
510
- await sendToRetryTopic(
511
- topic2,
512
- rawMessages,
513
- 1,
514
- retry.maxRetries,
515
- delay,
516
- isBatch ? envelopes.map((e) => e.headers) : envelopes[0]?.headers ?? {},
517
- deps
518
- );
530
+ if (deps.eosRouteToRetry) {
531
+ try {
532
+ await deps.eosRouteToRetry(rawMessages, envelopes, delay);
533
+ deps.onRetry?.(envelopes[0], 1, retry.maxRetries);
534
+ } catch (txErr) {
535
+ deps.logger.error(
536
+ `EOS routing to retry topic failed \u2014 message will be redelivered:`,
537
+ toError(txErr).stack
538
+ );
539
+ }
540
+ } else {
541
+ await sendToRetryTopic(
542
+ topic2,
543
+ rawMessages,
544
+ 1,
545
+ retry.maxRetries,
546
+ delay,
547
+ isBatch ? envelopes.map((e) => e.headers) : envelopes[0]?.headers ?? {},
548
+ deps
549
+ );
550
+ deps.onRetry?.(envelopes[0], 1, retry.maxRetries);
551
+ }
519
552
  } else if (isLastAttempt) {
520
553
  if (dlq) {
521
554
  for (let i = 0; i < rawMessages.length; i++) {
@@ -524,6 +557,7 @@ async function executeWithRetry(fn, ctx, deps) {
524
557
  attempt,
525
558
  originalHeaders: envelopes[i]?.headers
526
559
  });
560
+ deps.onDlq?.(envelopes[i] ?? envelopes[0], "handler-error");
527
561
  }
528
562
  } else {
529
563
  await deps.onMessageLost?.({
@@ -535,6 +569,7 @@ async function executeWithRetry(fn, ctx, deps) {
535
569
  }
536
570
  } else {
537
571
  const cap = Math.min(backoffMs * 2 ** (attempt - 1), maxBackoffMs);
572
+ deps.onRetry?.(envelopes[0], attempt, maxAttempts - 1);
538
573
  await sleep(Math.floor(Math.random() * cap));
539
574
  }
540
575
  }
@@ -558,6 +593,7 @@ async function applyDeduplication(envelope, raw, dedup, dlq, deps) {
558
593
  deps.logger.warn(
559
594
  `Duplicate message on ${envelope.topic}[${envelope.partition}]: clock=${incomingClock} <= last=${lastProcessedClock} \u2014 strategy=${strategy}`
560
595
  );
596
+ deps.onDuplicate?.(envelope, strategy);
561
597
  if (strategy === "dlq" && dlq) {
562
598
  const augmentedHeaders = {
563
599
  ...envelope.headers,
@@ -612,6 +648,43 @@ async function handleEachMessage(payload, opts, deps) {
612
648
  timeoutMs,
613
649
  wrapWithTimeout
614
650
  } = opts;
651
+ const eos = opts.eosMainContext;
652
+ const nextOffsetStr = (parseInt(message.offset, 10) + 1).toString();
653
+ const commitOffset = eos ? async () => {
654
+ await eos.consumer.commitOffsets([
655
+ { topic: topic2, partition, offset: nextOffsetStr }
656
+ ]);
657
+ } : void 0;
658
+ const eosRouteToRetry = eos && retry ? async (rawMsgs, envelopes, delay) => {
659
+ const { topic: rtTopic, messages: rtMsgs } = buildRetryTopicPayload(
660
+ topic2,
661
+ rawMsgs,
662
+ 1,
663
+ retry.maxRetries,
664
+ delay,
665
+ envelopes[0]?.headers ?? {}
666
+ );
667
+ const tx = await eos.txProducer.transaction();
668
+ try {
669
+ await tx.send({ topic: rtTopic, messages: rtMsgs });
670
+ await tx.sendOffsets({
671
+ consumer: eos.consumer,
672
+ topics: [
673
+ {
674
+ topic: topic2,
675
+ partitions: [{ partition, offset: nextOffsetStr }]
676
+ }
677
+ ]
678
+ });
679
+ await tx.commit();
680
+ } catch (txErr) {
681
+ try {
682
+ await tx.abort();
683
+ } catch {
684
+ }
685
+ throw txErr;
686
+ }
687
+ } : void 0;
615
688
  const envelope = await parseSingleMessage(
616
689
  message,
617
690
  topic2,
@@ -621,7 +694,10 @@ async function handleEachMessage(payload, opts, deps) {
621
694
  dlq,
622
695
  deps
623
696
  );
624
- if (envelope === null) return;
697
+ if (envelope === null) {
698
+ await commitOffset?.();
699
+ return;
700
+ }
625
701
  if (opts.deduplication) {
626
702
  const isDuplicate = await applyDeduplication(
627
703
  envelope,
@@ -630,7 +706,10 @@ async function handleEachMessage(payload, opts, deps) {
630
706
  dlq,
631
707
  deps
632
708
  );
633
- if (isDuplicate) return;
709
+ if (isDuplicate) {
710
+ await commitOffset?.();
711
+ return;
712
+ }
634
713
  }
635
714
  await executeWithRetry(
636
715
  () => {
@@ -651,7 +730,7 @@ async function handleEachMessage(payload, opts, deps) {
651
730
  retry,
652
731
  retryTopics
653
732
  },
654
- deps
733
+ { ...deps, eosRouteToRetry, eosCommitOnSuccess: commitOffset }
655
734
  );
656
735
  }
657
736
  async function handleEachBatch(payload, opts, deps) {
@@ -666,6 +745,50 @@ async function handleEachBatch(payload, opts, deps) {
666
745
  timeoutMs,
667
746
  wrapWithTimeout
668
747
  } = opts;
748
+ const eos = opts.eosMainContext;
749
+ const lastRawOffset = batch.messages.length > 0 ? batch.messages[batch.messages.length - 1].offset : void 0;
750
+ const batchNextOffsetStr = lastRawOffset ? (parseInt(lastRawOffset, 10) + 1).toString() : void 0;
751
+ const commitBatchOffset = eos && batchNextOffsetStr ? async () => {
752
+ await eos.consumer.commitOffsets([
753
+ {
754
+ topic: batch.topic,
755
+ partition: batch.partition,
756
+ offset: batchNextOffsetStr
757
+ }
758
+ ]);
759
+ } : void 0;
760
+ const eosRouteToRetry = eos && retry && batchNextOffsetStr ? async (rawMsgs, envelopes2, delay) => {
761
+ const { topic: rtTopic, messages: rtMsgs } = buildRetryTopicPayload(
762
+ batch.topic,
763
+ rawMsgs,
764
+ 1,
765
+ retry.maxRetries,
766
+ delay,
767
+ envelopes2.map((e) => e.headers)
768
+ );
769
+ const tx = await eos.txProducer.transaction();
770
+ try {
771
+ await tx.send({ topic: rtTopic, messages: rtMsgs });
772
+ await tx.sendOffsets({
773
+ consumer: eos.consumer,
774
+ topics: [
775
+ {
776
+ topic: batch.topic,
777
+ partitions: [
778
+ { partition: batch.partition, offset: batchNextOffsetStr }
779
+ ]
780
+ }
781
+ ]
782
+ });
783
+ await tx.commit();
784
+ } catch (txErr) {
785
+ try {
786
+ await tx.abort();
787
+ } catch {
788
+ }
789
+ throw txErr;
790
+ }
791
+ } : void 0;
669
792
  const envelopes = [];
670
793
  const rawMessages = [];
671
794
  for (const message of batch.messages) {
@@ -693,7 +816,10 @@ async function handleEachBatch(payload, opts, deps) {
693
816
  envelopes.push(envelope);
694
817
  rawMessages.push(message.value.toString());
695
818
  }
696
- if (envelopes.length === 0) return;
819
+ if (envelopes.length === 0) {
820
+ await commitBatchOffset?.();
821
+ return;
822
+ }
697
823
  const meta = {
698
824
  partition: batch.partition,
699
825
  highWatermark: batch.highWatermark,
@@ -715,7 +841,7 @@ async function handleEachBatch(payload, opts, deps) {
715
841
  isBatch: true,
716
842
  retryTopics
717
843
  },
718
- deps
844
+ { ...deps, eosRouteToRetry, eosCommitOnSuccess: commitBatchOffset }
719
845
  );
720
846
  }
721
847
 
@@ -761,6 +887,9 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
761
887
  producer,
762
888
  instrumentation,
763
889
  onMessageLost,
890
+ onRetry,
891
+ onDlq,
892
+ onMessage,
764
893
  ensureTopic,
765
894
  getOrCreateConsumer: getOrCreateConsumer2,
766
895
  runningConsumers,
@@ -842,6 +971,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
842
971
  instrumentation
843
972
  );
844
973
  if (!error) {
974
+ onMessage?.(envelope);
845
975
  await consumer.commitOffsets([nextOffset]);
846
976
  return;
847
977
  }
@@ -874,12 +1004,23 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
874
1004
  await tx.send({ topic: rtTopic, messages: rtMsgs });
875
1005
  await tx.sendOffsets({
876
1006
  consumer,
877
- topics: [{ topic: nextOffset.topic, partitions: [{ partition: nextOffset.partition, offset: nextOffset.offset }] }]
1007
+ topics: [
1008
+ {
1009
+ topic: nextOffset.topic,
1010
+ partitions: [
1011
+ {
1012
+ partition: nextOffset.partition,
1013
+ offset: nextOffset.offset
1014
+ }
1015
+ ]
1016
+ }
1017
+ ]
878
1018
  });
879
1019
  await tx.commit();
880
1020
  logger.warn(
881
1021
  `Message routed to ${rtTopic} (EOS, level ${nextLevel}/${currentMaxRetries})`
882
1022
  );
1023
+ onRetry?.(envelope, nextLevel, currentMaxRetries);
883
1024
  } catch (txErr) {
884
1025
  try {
885
1026
  await tx.abort();
@@ -907,10 +1048,21 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
907
1048
  await tx.send({ topic: dTopic, messages: dMsgs });
908
1049
  await tx.sendOffsets({
909
1050
  consumer,
910
- topics: [{ topic: nextOffset.topic, partitions: [{ partition: nextOffset.partition, offset: nextOffset.offset }] }]
1051
+ topics: [
1052
+ {
1053
+ topic: nextOffset.topic,
1054
+ partitions: [
1055
+ {
1056
+ partition: nextOffset.partition,
1057
+ offset: nextOffset.offset
1058
+ }
1059
+ ]
1060
+ }
1061
+ ]
911
1062
  });
912
1063
  await tx.commit();
913
1064
  logger.warn(`Message sent to DLQ: ${dTopic} (EOS)`);
1065
+ onDlq?.(envelope, "handler-error");
914
1066
  } catch (txErr) {
915
1067
  try {
916
1068
  await tx.abort();
@@ -934,7 +1086,12 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
934
1086
  }
935
1087
  });
936
1088
  runningConsumers.set(levelGroupId, "eachMessage");
937
- await waitForPartitionAssignment(consumer, levelTopics, logger, assignmentTimeoutMs);
1089
+ await waitForPartitionAssignment(
1090
+ consumer,
1091
+ levelTopics,
1092
+ logger,
1093
+ assignmentTimeoutMs
1094
+ );
938
1095
  logger.log(
939
1096
  `Retry level ${level}/${retry.maxRetries} consumer started for: ${originalTopics.join(", ")} (group: ${levelGroupId})`
940
1097
  );
@@ -964,7 +1121,8 @@ async function startRetryTopicConsumers(originalTopics, originalGroupId, handleM
964
1121
 
965
1122
  // src/client/kafka.client/index.ts
966
1123
  var { Kafka: KafkaClass, logLevel: KafkaLogLevel } = KafkaJS;
967
- var KafkaClient = class {
1124
+ var _activeTransactionalIds = /* @__PURE__ */ new Set();
1125
+ var KafkaClient = class _KafkaClient {
968
1126
  kafka;
969
1127
  producer;
970
1128
  txProducer;
@@ -989,6 +1147,10 @@ var KafkaClient = class {
989
1147
  instrumentation;
990
1148
  onMessageLost;
991
1149
  onRebalance;
1150
+ /** Transactional producer ID — configurable via `KafkaClientOptions.transactionalId`. */
1151
+ txId;
1152
+ /** Per-topic event counters, lazily created on first event. Aggregated by `getMetrics()`. */
1153
+ _topicMetrics = /* @__PURE__ */ new Map();
992
1154
  /** Monotonically increasing Lamport clock stamped on every outgoing message. */
993
1155
  _lamportClock = 0;
994
1156
  /** Per-groupId deduplication state: `"topic:partition"` → last processed clock. */
@@ -1012,6 +1174,7 @@ var KafkaClient = class {
1012
1174
  this.instrumentation = options?.instrumentation ?? [];
1013
1175
  this.onMessageLost = options?.onMessageLost;
1014
1176
  this.onRebalance = options?.onRebalance;
1177
+ this.txId = options?.transactionalId ?? `${clientId}-tx`;
1015
1178
  this.kafka = new KafkaClass({
1016
1179
  kafkaJS: {
1017
1180
  clientId: this.clientId,
@@ -1048,16 +1211,22 @@ var KafkaClient = class {
1048
1211
  /** Execute multiple sends atomically. Commits on success, aborts on error. */
1049
1212
  async transaction(fn) {
1050
1213
  if (!this.txProducerInitPromise) {
1214
+ if (_activeTransactionalIds.has(this.txId)) {
1215
+ this.logger.warn(
1216
+ `transactionalId "${this.txId}" is already in use by another KafkaClient in this process. Kafka will fence one of the producers. Set a unique \`transactionalId\` (or distinct \`clientId\`) per instance.`
1217
+ );
1218
+ }
1051
1219
  const initPromise = (async () => {
1052
1220
  const p = this.kafka.producer({
1053
1221
  kafkaJS: {
1054
1222
  acks: -1,
1055
1223
  idempotent: true,
1056
- transactionalId: `${this.clientId}-tx`,
1224
+ transactionalId: this.txId,
1057
1225
  maxInFlightRequests: 1
1058
1226
  }
1059
1227
  });
1060
1228
  await p.connect();
1229
+ _activeTransactionalIds.add(this.txId);
1061
1230
  return p;
1062
1231
  })();
1063
1232
  this.txProducerInitPromise = initPromise.catch((err) => {
@@ -1125,10 +1294,20 @@ var KafkaClient = class {
1125
1294
  "retryTopics requires retry to be configured \u2014 set retry.maxRetries to enable the retry topic chain"
1126
1295
  );
1127
1296
  }
1128
- const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", options);
1297
+ const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
1298
+ const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", setupOptions);
1129
1299
  const deps = this.messageDeps;
1130
1300
  const timeoutMs = options.handlerTimeoutMs;
1131
- const deduplication = this.resolveDeduplicationContext(gid, options.deduplication);
1301
+ const deduplication = this.resolveDeduplicationContext(
1302
+ gid,
1303
+ options.deduplication
1304
+ );
1305
+ let eosMainContext;
1306
+ if (options.retryTopics && retry) {
1307
+ const mainTxId = `${gid}-main-tx`;
1308
+ const txProducer = await this.createRetryTxProducer(mainTxId);
1309
+ eosMainContext = { txProducer, consumer };
1310
+ }
1132
1311
  await consumer.run({
1133
1312
  eachMessage: (payload) => this.trackInFlight(
1134
1313
  () => handleEachMessage(
@@ -1142,7 +1321,8 @@ var KafkaClient = class {
1142
1321
  retryTopics: options.retryTopics,
1143
1322
  timeoutMs,
1144
1323
  wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this),
1145
- deduplication
1324
+ deduplication,
1325
+ eosMainContext
1146
1326
  },
1147
1327
  deps
1148
1328
  )
@@ -1174,15 +1354,26 @@ var KafkaClient = class {
1174
1354
  "retryTopics requires retry to be configured \u2014 set retry.maxRetries to enable the retry topic chain"
1175
1355
  );
1176
1356
  }
1177
- if (options.autoCommit !== false) {
1357
+ if (options.retryTopics) {
1358
+ } else if (options.autoCommit !== false) {
1178
1359
  this.logger.debug?.(
1179
1360
  `startBatchConsumer: autoCommit is enabled (default true). If your handler calls resolveOffset() or commitOffsetsIfNecessary(), set autoCommit: false to avoid offset conflicts.`
1180
1361
  );
1181
1362
  }
1182
- const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", options);
1363
+ const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
1364
+ const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", setupOptions);
1183
1365
  const deps = this.messageDeps;
1184
1366
  const timeoutMs = options.handlerTimeoutMs;
1185
- const deduplication = this.resolveDeduplicationContext(gid, options.deduplication);
1367
+ const deduplication = this.resolveDeduplicationContext(
1368
+ gid,
1369
+ options.deduplication
1370
+ );
1371
+ let eosMainContext;
1372
+ if (options.retryTopics && retry) {
1373
+ const mainTxId = `${gid}-main-tx`;
1374
+ const txProducer = await this.createRetryTxProducer(mainTxId);
1375
+ eosMainContext = { txProducer, consumer };
1376
+ }
1186
1377
  await consumer.run({
1187
1378
  eachBatch: (payload) => this.trackInFlight(
1188
1379
  () => handleEachBatch(
@@ -1196,7 +1387,8 @@ var KafkaClient = class {
1196
1387
  retryTopics: options.retryTopics,
1197
1388
  timeoutMs,
1198
1389
  wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this),
1199
- deduplication
1390
+ deduplication,
1391
+ eosMainContext
1200
1392
  },
1201
1393
  deps
1202
1394
  )
@@ -1209,7 +1401,7 @@ var KafkaClient = class {
1209
1401
  }
1210
1402
  const handleMessageForRetry = (env) => handleBatch([env], {
1211
1403
  partition: env.partition,
1212
- highWatermark: env.offset,
1404
+ highWatermark: null,
1213
1405
  heartbeat: async () => {
1214
1406
  },
1215
1407
  resolveOffset: () => {
@@ -1253,6 +1445,18 @@ var KafkaClient = class {
1253
1445
  this.consumerCreationOptions.delete(groupId);
1254
1446
  this.dedupStates.delete(groupId);
1255
1447
  this.logger.log(`Consumer disconnected: group "${groupId}"`);
1448
+ const mainTxId = `${groupId}-main-tx`;
1449
+ const mainTxProducer = this.retryTxProducers.get(mainTxId);
1450
+ if (mainTxProducer) {
1451
+ await mainTxProducer.disconnect().catch(
1452
+ (e) => this.logger.warn(
1453
+ `Error disconnecting main tx producer "${mainTxId}":`,
1454
+ toError(e).message
1455
+ )
1456
+ );
1457
+ _activeTransactionalIds.delete(mainTxId);
1458
+ this.retryTxProducers.delete(mainTxId);
1459
+ }
1256
1460
  const companions = this.companionGroupIds.get(groupId) ?? [];
1257
1461
  for (const cGroupId of companions) {
1258
1462
  const cConsumer = this.consumers.get(cGroupId);
@@ -1277,6 +1481,7 @@ var KafkaClient = class {
1277
1481
  toError(e).message
1278
1482
  )
1279
1483
  );
1484
+ _activeTransactionalIds.delete(txId);
1280
1485
  this.retryTxProducers.delete(txId);
1281
1486
  }
1282
1487
  }
@@ -1302,6 +1507,144 @@ var KafkaClient = class {
1302
1507
  this.logger.log("All consumers disconnected");
1303
1508
  }
1304
1509
  }
1510
+ pauseConsumer(groupId, assignments) {
1511
+ const gid = groupId ?? this.defaultGroupId;
1512
+ const consumer = this.consumers.get(gid);
1513
+ if (!consumer) {
1514
+ this.logger.warn(`pauseConsumer: no active consumer for group "${gid}"`);
1515
+ return;
1516
+ }
1517
+ consumer.pause(
1518
+ assignments.flatMap(
1519
+ ({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] }))
1520
+ )
1521
+ );
1522
+ }
1523
+ resumeConsumer(groupId, assignments) {
1524
+ const gid = groupId ?? this.defaultGroupId;
1525
+ const consumer = this.consumers.get(gid);
1526
+ if (!consumer) {
1527
+ this.logger.warn(`resumeConsumer: no active consumer for group "${gid}"`);
1528
+ return;
1529
+ }
1530
+ consumer.resume(
1531
+ assignments.flatMap(
1532
+ ({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] }))
1533
+ )
1534
+ );
1535
+ }
1536
+ /** DLQ header keys added by `sendToDlq` — stripped before re-publishing. */
1537
+ static DLQ_HEADER_KEYS = /* @__PURE__ */ new Set([
1538
+ "x-dlq-original-topic",
1539
+ "x-dlq-failed-at",
1540
+ "x-dlq-error-message",
1541
+ "x-dlq-error-stack",
1542
+ "x-dlq-attempt-count"
1543
+ ]);
1544
+ async replayDlq(topic2, options = {}) {
1545
+ const dlqTopic = `${topic2}.dlq`;
1546
+ await this.ensureAdminConnected();
1547
+ const partitionOffsets = await this.admin.fetchTopicOffsets(dlqTopic);
1548
+ const activePartitions = partitionOffsets.filter(
1549
+ (p) => parseInt(p.high, 10) > 0
1550
+ );
1551
+ if (activePartitions.length === 0) {
1552
+ this.logger.log(`replayDlq: "${dlqTopic}" is empty \u2014 nothing to replay`);
1553
+ return { replayed: 0, skipped: 0 };
1554
+ }
1555
+ const highWatermarks = new Map(
1556
+ activePartitions.map(({ partition, high }) => [
1557
+ partition,
1558
+ parseInt(high, 10)
1559
+ ])
1560
+ );
1561
+ const processedOffsets = /* @__PURE__ */ new Map();
1562
+ let replayed = 0;
1563
+ let skipped = 0;
1564
+ const tempGroupId = `${dlqTopic}-replay-${Date.now()}`;
1565
+ await new Promise((resolve, reject) => {
1566
+ const consumer = getOrCreateConsumer(
1567
+ tempGroupId,
1568
+ true,
1569
+ true,
1570
+ this.consumerOpsDeps
1571
+ );
1572
+ const cleanup = () => {
1573
+ consumer.disconnect().catch(() => {
1574
+ }).finally(() => {
1575
+ this.consumers.delete(tempGroupId);
1576
+ this.runningConsumers.delete(tempGroupId);
1577
+ this.consumerCreationOptions.delete(tempGroupId);
1578
+ });
1579
+ };
1580
+ consumer.connect().then(
1581
+ () => subscribeWithRetry(consumer, [dlqTopic], this.logger)
1582
+ ).then(
1583
+ () => consumer.run({
1584
+ eachMessage: async ({ partition, message }) => {
1585
+ if (!message.value) return;
1586
+ const offset = parseInt(message.offset, 10);
1587
+ processedOffsets.set(partition, offset);
1588
+ const headers = decodeHeaders(message.headers);
1589
+ const targetTopic = options.targetTopic ?? headers["x-dlq-original-topic"];
1590
+ const originalHeaders = Object.fromEntries(
1591
+ Object.entries(headers).filter(
1592
+ ([k]) => !_KafkaClient.DLQ_HEADER_KEYS.has(k)
1593
+ )
1594
+ );
1595
+ const value = message.value.toString();
1596
+ const shouldProcess = !options.filter || options.filter(headers, value);
1597
+ if (!targetTopic || !shouldProcess) {
1598
+ skipped++;
1599
+ } else if (options.dryRun) {
1600
+ this.logger.log(
1601
+ `[DLQ replay dry-run] Would replay to "${targetTopic}"`
1602
+ );
1603
+ replayed++;
1604
+ } else {
1605
+ await this.producer.send({
1606
+ topic: targetTopic,
1607
+ messages: [{ value, headers: originalHeaders }]
1608
+ });
1609
+ replayed++;
1610
+ }
1611
+ const allDone = Array.from(highWatermarks.entries()).every(
1612
+ ([p, hwm]) => (processedOffsets.get(p) ?? -1) >= hwm - 1
1613
+ );
1614
+ if (allDone) {
1615
+ cleanup();
1616
+ resolve();
1617
+ }
1618
+ }
1619
+ })
1620
+ ).catch((err) => {
1621
+ cleanup();
1622
+ reject(err);
1623
+ });
1624
+ });
1625
+ this.logger.log(
1626
+ `replayDlq: replayed ${replayed}, skipped ${skipped} from "${dlqTopic}"`
1627
+ );
1628
+ return { replayed, skipped };
1629
+ }
1630
+ async resetOffsets(groupId, topic2, position) {
1631
+ const gid = groupId ?? this.defaultGroupId;
1632
+ if (this.runningConsumers.has(gid)) {
1633
+ throw new Error(
1634
+ `resetOffsets: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before resetting offsets.`
1635
+ );
1636
+ }
1637
+ await this.ensureAdminConnected();
1638
+ const partitionOffsets = await this.admin.fetchTopicOffsets(topic2);
1639
+ const partitions = partitionOffsets.map(({ partition, low, high }) => ({
1640
+ partition,
1641
+ offset: position === "earliest" ? low : high
1642
+ }));
1643
+ await this.admin.setOffsets({ groupId: gid, topic: topic2, partitions });
1644
+ this.logger.log(
1645
+ `Offsets reset to ${position} for group "${gid}" on topic "${topic2}"`
1646
+ );
1647
+ }
1305
1648
  /**
1306
1649
  * Query consumer group lag per partition.
1307
1650
  * Lag = broker high-watermark − last committed offset.
@@ -1352,15 +1695,45 @@ var KafkaClient = class {
1352
1695
  getClientId() {
1353
1696
  return this.clientId;
1354
1697
  }
1698
+ getMetrics(topic2) {
1699
+ if (topic2 !== void 0) {
1700
+ const m = this._topicMetrics.get(topic2);
1701
+ return m ? { ...m } : { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
1702
+ }
1703
+ const agg = {
1704
+ processedCount: 0,
1705
+ retryCount: 0,
1706
+ dlqCount: 0,
1707
+ dedupCount: 0
1708
+ };
1709
+ for (const m of this._topicMetrics.values()) {
1710
+ agg.processedCount += m.processedCount;
1711
+ agg.retryCount += m.retryCount;
1712
+ agg.dlqCount += m.dlqCount;
1713
+ agg.dedupCount += m.dedupCount;
1714
+ }
1715
+ return agg;
1716
+ }
1717
+ resetMetrics(topic2) {
1718
+ if (topic2 !== void 0) {
1719
+ this._topicMetrics.delete(topic2);
1720
+ return;
1721
+ }
1722
+ this._topicMetrics.clear();
1723
+ }
1355
1724
  /** Gracefully disconnect producer, all consumers, and admin. */
1356
1725
  async disconnect(drainTimeoutMs = 3e4) {
1357
1726
  await this.waitForDrain(drainTimeoutMs);
1358
1727
  const tasks = [this.producer.disconnect()];
1359
1728
  if (this.txProducer) {
1360
1729
  tasks.push(this.txProducer.disconnect());
1730
+ _activeTransactionalIds.delete(this.txId);
1361
1731
  this.txProducer = void 0;
1362
1732
  this.txProducerInitPromise = void 0;
1363
1733
  }
1734
+ for (const txId of this.retryTxProducers.keys()) {
1735
+ _activeTransactionalIds.delete(txId);
1736
+ }
1364
1737
  for (const p of this.retryTxProducers.values()) {
1365
1738
  tasks.push(p.disconnect());
1366
1739
  }
@@ -1456,6 +1829,38 @@ var KafkaClient = class {
1456
1829
  }
1457
1830
  }
1458
1831
  }
1832
+ metricsFor(topic2) {
1833
+ let m = this._topicMetrics.get(topic2);
1834
+ if (!m) {
1835
+ m = { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
1836
+ this._topicMetrics.set(topic2, m);
1837
+ }
1838
+ return m;
1839
+ }
1840
+ notifyRetry(envelope, attempt, maxRetries) {
1841
+ this.metricsFor(envelope.topic).retryCount++;
1842
+ for (const inst of this.instrumentation) {
1843
+ inst.onRetry?.(envelope, attempt, maxRetries);
1844
+ }
1845
+ }
1846
+ notifyDlq(envelope, reason) {
1847
+ this.metricsFor(envelope.topic).dlqCount++;
1848
+ for (const inst of this.instrumentation) {
1849
+ inst.onDlq?.(envelope, reason);
1850
+ }
1851
+ }
1852
+ notifyDuplicate(envelope, strategy) {
1853
+ this.metricsFor(envelope.topic).dedupCount++;
1854
+ for (const inst of this.instrumentation) {
1855
+ inst.onDuplicate?.(envelope, strategy);
1856
+ }
1857
+ }
1858
+ notifyMessage(envelope) {
1859
+ this.metricsFor(envelope.topic).processedCount++;
1860
+ for (const inst of this.instrumentation) {
1861
+ inst.onMessage?.(envelope);
1862
+ }
1863
+ }
1459
1864
  /**
1460
1865
  * Start a timer that logs a warning if `fn` hasn't resolved within `timeoutMs`.
1461
1866
  * The handler itself is not cancelled — the warning is diagnostic only.
@@ -1545,6 +1950,11 @@ var KafkaClient = class {
1545
1950
  * so Kafka can fence stale producers on restart without affecting other levels.
1546
1951
  */
1547
1952
  async createRetryTxProducer(transactionalId) {
1953
+ if (_activeTransactionalIds.has(transactionalId)) {
1954
+ this.logger.warn(
1955
+ `transactionalId "${transactionalId}" is already in use by another KafkaClient in this process. Kafka will fence one of the producers. Set a unique \`transactionalId\` (or distinct \`clientId\`) per instance.`
1956
+ );
1957
+ }
1548
1958
  const p = this.kafka.producer({
1549
1959
  kafkaJS: {
1550
1960
  acks: -1,
@@ -1554,6 +1964,7 @@ var KafkaClient = class {
1554
1964
  }
1555
1965
  });
1556
1966
  await p.connect();
1967
+ _activeTransactionalIds.add(transactionalId);
1557
1968
  this.retryTxProducers.set(transactionalId, p);
1558
1969
  return p;
1559
1970
  }
@@ -1674,7 +2085,11 @@ var KafkaClient = class {
1674
2085
  logger: this.logger,
1675
2086
  producer: this.producer,
1676
2087
  instrumentation: this.instrumentation,
1677
- onMessageLost: this.onMessageLost
2088
+ onMessageLost: this.onMessageLost,
2089
+ onRetry: this.notifyRetry.bind(this),
2090
+ onDlq: this.notifyDlq.bind(this),
2091
+ onDuplicate: this.notifyDuplicate.bind(this),
2092
+ onMessage: this.notifyMessage.bind(this)
1678
2093
  };
1679
2094
  }
1680
2095
  get retryTopicDeps() {
@@ -1683,6 +2098,9 @@ var KafkaClient = class {
1683
2098
  producer: this.producer,
1684
2099
  instrumentation: this.instrumentation,
1685
2100
  onMessageLost: this.onMessageLost,
2101
+ onRetry: this.notifyRetry.bind(this),
2102
+ onDlq: this.notifyDlq.bind(this),
2103
+ onMessage: this.notifyMessage.bind(this),
1686
2104
  ensureTopic: (t) => this.ensureTopic(t),
1687
2105
  getOrCreateConsumer: (gid, fb, ac) => getOrCreateConsumer(gid, fb, ac, this.consumerOpsDeps),
1688
2106
  runningConsumers: this.runningConsumers,
@@ -1725,4 +2143,4 @@ export {
1725
2143
  KafkaClient,
1726
2144
  topic
1727
2145
  };
1728
- //# sourceMappingURL=chunk-KCUKXR6B.mjs.map
2146
+ //# sourceMappingURL=chunk-4526Y4PV.mjs.map