@drarzter/kafka-client 0.6.7 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -394,9 +394,16 @@ var RETRY_HEADER_MAX_RETRIES = "x-retry-max-retries";
394
394
  var RETRY_HEADER_ORIGINAL_TOPIC = "x-retry-original-topic";
395
395
  function buildRetryTopicPayload(originalTopic, rawMessages, attempt, maxRetries, delayMs, originalHeaders) {
396
396
  const retryTopic = `${originalTopic}.retry.${attempt}`;
397
- const STRIP = /* @__PURE__ */ new Set([RETRY_HEADER_ATTEMPT, RETRY_HEADER_AFTER, RETRY_HEADER_MAX_RETRIES, RETRY_HEADER_ORIGINAL_TOPIC]);
397
+ const STRIP = /* @__PURE__ */ new Set([
398
+ RETRY_HEADER_ATTEMPT,
399
+ RETRY_HEADER_AFTER,
400
+ RETRY_HEADER_MAX_RETRIES,
401
+ RETRY_HEADER_ORIGINAL_TOPIC
402
+ ]);
398
403
  function buildHeaders(hdr) {
399
- const userHeaders = Object.fromEntries(Object.entries(hdr).filter(([k]) => !STRIP.has(k)));
404
+ const userHeaders = Object.fromEntries(
405
+ Object.entries(hdr).filter(([k]) => !STRIP.has(k))
406
+ );
400
407
  return {
401
408
  ...userHeaders,
402
409
  [RETRY_HEADER_ATTEMPT]: String(attempt),
@@ -556,6 +563,17 @@ async function executeWithRetry(fn, ctx, deps) {
556
563
  deps.instrumentation
557
564
  );
558
565
  if (!error) {
566
+ if (deps.eosCommitOnSuccess) {
567
+ try {
568
+ await deps.eosCommitOnSuccess();
569
+ } catch (commitErr) {
570
+ deps.logger.error(
571
+ `EOS offset commit failed after successful handler \u2014 message will be redelivered:`,
572
+ toError(commitErr).stack
573
+ );
574
+ return;
575
+ }
576
+ }
559
577
  for (const env of envelopes) deps.onMessage?.(env);
560
578
  return;
561
579
  }
@@ -574,16 +592,28 @@ async function executeWithRetry(fn, ctx, deps) {
574
592
  if (retryTopics && retry) {
575
593
  const cap = Math.min(backoffMs, maxBackoffMs);
576
594
  const delay = Math.floor(Math.random() * cap);
577
- await sendToRetryTopic(
578
- topic2,
579
- rawMessages,
580
- 1,
581
- retry.maxRetries,
582
- delay,
583
- isBatch ? envelopes.map((e) => e.headers) : envelopes[0]?.headers ?? {},
584
- deps
585
- );
586
- deps.onRetry?.(envelopes[0], 1, retry.maxRetries);
595
+ if (deps.eosRouteToRetry) {
596
+ try {
597
+ await deps.eosRouteToRetry(rawMessages, envelopes, delay);
598
+ deps.onRetry?.(envelopes[0], 1, retry.maxRetries);
599
+ } catch (txErr) {
600
+ deps.logger.error(
601
+ `EOS routing to retry topic failed \u2014 message will be redelivered:`,
602
+ toError(txErr).stack
603
+ );
604
+ }
605
+ } else {
606
+ await sendToRetryTopic(
607
+ topic2,
608
+ rawMessages,
609
+ 1,
610
+ retry.maxRetries,
611
+ delay,
612
+ isBatch ? envelopes.map((e) => e.headers) : envelopes[0]?.headers ?? {},
613
+ deps
614
+ );
615
+ deps.onRetry?.(envelopes[0], 1, retry.maxRetries);
616
+ }
587
617
  } else if (isLastAttempt) {
588
618
  if (dlq) {
589
619
  for (let i = 0; i < rawMessages.length; i++) {
@@ -683,6 +713,43 @@ async function handleEachMessage(payload, opts, deps) {
683
713
  timeoutMs,
684
714
  wrapWithTimeout
685
715
  } = opts;
716
+ const eos = opts.eosMainContext;
717
+ const nextOffsetStr = (parseInt(message.offset, 10) + 1).toString();
718
+ const commitOffset = eos ? async () => {
719
+ await eos.consumer.commitOffsets([
720
+ { topic: topic2, partition, offset: nextOffsetStr }
721
+ ]);
722
+ } : void 0;
723
+ const eosRouteToRetry = eos && retry ? async (rawMsgs, envelopes, delay) => {
724
+ const { topic: rtTopic, messages: rtMsgs } = buildRetryTopicPayload(
725
+ topic2,
726
+ rawMsgs,
727
+ 1,
728
+ retry.maxRetries,
729
+ delay,
730
+ envelopes[0]?.headers ?? {}
731
+ );
732
+ const tx = await eos.txProducer.transaction();
733
+ try {
734
+ await tx.send({ topic: rtTopic, messages: rtMsgs });
735
+ await tx.sendOffsets({
736
+ consumer: eos.consumer,
737
+ topics: [
738
+ {
739
+ topic: topic2,
740
+ partitions: [{ partition, offset: nextOffsetStr }]
741
+ }
742
+ ]
743
+ });
744
+ await tx.commit();
745
+ } catch (txErr) {
746
+ try {
747
+ await tx.abort();
748
+ } catch {
749
+ }
750
+ throw txErr;
751
+ }
752
+ } : void 0;
686
753
  const envelope = await parseSingleMessage(
687
754
  message,
688
755
  topic2,
@@ -692,7 +759,10 @@ async function handleEachMessage(payload, opts, deps) {
692
759
  dlq,
693
760
  deps
694
761
  );
695
- if (envelope === null) return;
762
+ if (envelope === null) {
763
+ await commitOffset?.();
764
+ return;
765
+ }
696
766
  if (opts.deduplication) {
697
767
  const isDuplicate = await applyDeduplication(
698
768
  envelope,
@@ -701,7 +771,35 @@ async function handleEachMessage(payload, opts, deps) {
701
771
  dlq,
702
772
  deps
703
773
  );
704
- if (isDuplicate) return;
774
+ if (isDuplicate) {
775
+ await commitOffset?.();
776
+ return;
777
+ }
778
+ }
779
+ if (opts.messageTtlMs !== void 0) {
780
+ const ageMs = Date.now() - new Date(envelope.timestamp).getTime();
781
+ if (ageMs > opts.messageTtlMs) {
782
+ deps.logger.warn(
783
+ `[KafkaClient] TTL expired on ${topic2}: age ${ageMs}ms > ${opts.messageTtlMs}ms`
784
+ );
785
+ if (dlq) {
786
+ await sendToDlq(topic2, message.value.toString(), deps, {
787
+ error: new Error(`Message TTL expired: age ${ageMs}ms`),
788
+ attempt: 0,
789
+ originalHeaders: envelope.headers
790
+ });
791
+ deps.onDlq?.(envelope, "ttl-expired");
792
+ } else {
793
+ await deps.onMessageLost?.({
794
+ topic: topic2,
795
+ error: new Error(`TTL expired: ${ageMs}ms`),
796
+ attempt: 0,
797
+ headers: envelope.headers
798
+ });
799
+ }
800
+ await commitOffset?.();
801
+ return;
802
+ }
705
803
  }
706
804
  await executeWithRetry(
707
805
  () => {
@@ -722,7 +820,7 @@ async function handleEachMessage(payload, opts, deps) {
722
820
  retry,
723
821
  retryTopics
724
822
  },
725
- deps
823
+ { ...deps, eosRouteToRetry, eosCommitOnSuccess: commitOffset }
726
824
  );
727
825
  }
728
826
  async function handleEachBatch(payload, opts, deps) {
@@ -737,6 +835,50 @@ async function handleEachBatch(payload, opts, deps) {
737
835
  timeoutMs,
738
836
  wrapWithTimeout
739
837
  } = opts;
838
+ const eos = opts.eosMainContext;
839
+ const lastRawOffset = batch.messages.length > 0 ? batch.messages[batch.messages.length - 1].offset : void 0;
840
+ const batchNextOffsetStr = lastRawOffset ? (parseInt(lastRawOffset, 10) + 1).toString() : void 0;
841
+ const commitBatchOffset = eos && batchNextOffsetStr ? async () => {
842
+ await eos.consumer.commitOffsets([
843
+ {
844
+ topic: batch.topic,
845
+ partition: batch.partition,
846
+ offset: batchNextOffsetStr
847
+ }
848
+ ]);
849
+ } : void 0;
850
+ const eosRouteToRetry = eos && retry && batchNextOffsetStr ? async (rawMsgs, envelopes2, delay) => {
851
+ const { topic: rtTopic, messages: rtMsgs } = buildRetryTopicPayload(
852
+ batch.topic,
853
+ rawMsgs,
854
+ 1,
855
+ retry.maxRetries,
856
+ delay,
857
+ envelopes2.map((e) => e.headers)
858
+ );
859
+ const tx = await eos.txProducer.transaction();
860
+ try {
861
+ await tx.send({ topic: rtTopic, messages: rtMsgs });
862
+ await tx.sendOffsets({
863
+ consumer: eos.consumer,
864
+ topics: [
865
+ {
866
+ topic: batch.topic,
867
+ partitions: [
868
+ { partition: batch.partition, offset: batchNextOffsetStr }
869
+ ]
870
+ }
871
+ ]
872
+ });
873
+ await tx.commit();
874
+ } catch (txErr) {
875
+ try {
876
+ await tx.abort();
877
+ } catch {
878
+ }
879
+ throw txErr;
880
+ }
881
+ } : void 0;
740
882
  const envelopes = [];
741
883
  const rawMessages = [];
742
884
  for (const message of batch.messages) {
@@ -761,10 +903,37 @@ async function handleEachBatch(payload, opts, deps) {
761
903
  );
762
904
  if (isDuplicate) continue;
763
905
  }
906
+ if (opts.messageTtlMs !== void 0) {
907
+ const ageMs = Date.now() - new Date(envelope.timestamp).getTime();
908
+ if (ageMs > opts.messageTtlMs) {
909
+ deps.logger.warn(
910
+ `[KafkaClient] TTL expired on ${batch.topic}: age ${ageMs}ms > ${opts.messageTtlMs}ms`
911
+ );
912
+ if (dlq) {
913
+ await sendToDlq(batch.topic, message.value.toString(), deps, {
914
+ error: new Error(`Message TTL expired: age ${ageMs}ms`),
915
+ attempt: 0,
916
+ originalHeaders: envelope.headers
917
+ });
918
+ deps.onDlq?.(envelope, "ttl-expired");
919
+ } else {
920
+ await deps.onMessageLost?.({
921
+ topic: batch.topic,
922
+ error: new Error(`TTL expired: ${ageMs}ms`),
923
+ attempt: 0,
924
+ headers: envelope.headers
925
+ });
926
+ }
927
+ continue;
928
+ }
929
+ }
764
930
  envelopes.push(envelope);
765
931
  rawMessages.push(message.value.toString());
766
932
  }
767
- if (envelopes.length === 0) return;
933
+ if (envelopes.length === 0) {
934
+ await commitBatchOffset?.();
935
+ return;
936
+ }
768
937
  const meta = {
769
938
  partition: batch.partition,
770
939
  highWatermark: batch.highWatermark,
@@ -786,7 +955,7 @@ async function handleEachBatch(payload, opts, deps) {
786
955
  isBatch: true,
787
956
  retryTopics
788
957
  },
789
- deps
958
+ { ...deps, eosRouteToRetry, eosCommitOnSuccess: commitBatchOffset }
790
959
  );
791
960
  }
792
961
 
@@ -1067,7 +1236,32 @@ async function startRetryTopicConsumers(originalTopics, originalGroupId, handleM
1067
1236
  // src/client/kafka.client/index.ts
1068
1237
  var { Kafka: KafkaClass, logLevel: KafkaLogLevel } = import_kafka_javascript.KafkaJS;
1069
1238
  var _activeTransactionalIds = /* @__PURE__ */ new Set();
1070
- var KafkaClient = class {
1239
+ var AsyncQueue = class {
1240
+ items = [];
1241
+ waiting = [];
1242
+ closed = false;
1243
+ push(item) {
1244
+ if (this.waiting.length > 0) {
1245
+ this.waiting.shift()({ value: item, done: false });
1246
+ } else {
1247
+ this.items.push(item);
1248
+ }
1249
+ }
1250
+ close() {
1251
+ this.closed = true;
1252
+ for (const r of this.waiting.splice(0)) {
1253
+ r({ value: void 0, done: true });
1254
+ }
1255
+ }
1256
+ next() {
1257
+ if (this.items.length > 0)
1258
+ return Promise.resolve({ value: this.items.shift(), done: false });
1259
+ if (this.closed)
1260
+ return Promise.resolve({ value: void 0, done: true });
1261
+ return new Promise((r) => this.waiting.push(r));
1262
+ }
1263
+ };
1264
+ var KafkaClient = class _KafkaClient {
1071
1265
  kafka;
1072
1266
  producer;
1073
1267
  txProducer;
@@ -1094,17 +1288,16 @@ var KafkaClient = class {
1094
1288
  onRebalance;
1095
1289
  /** Transactional producer ID — configurable via `KafkaClientOptions.transactionalId`. */
1096
1290
  txId;
1097
- /** Internal event counters exposed via `getMetrics()`. */
1098
- _metrics = {
1099
- processedCount: 0,
1100
- retryCount: 0,
1101
- dlqCount: 0,
1102
- dedupCount: 0
1103
- };
1291
+ /** Per-topic event counters, lazily created on first event. Aggregated by `getMetrics()`. */
1292
+ _topicMetrics = /* @__PURE__ */ new Map();
1104
1293
  /** Monotonically increasing Lamport clock stamped on every outgoing message. */
1105
1294
  _lamportClock = 0;
1106
1295
  /** Per-groupId deduplication state: `"topic:partition"` → last processed clock. */
1107
1296
  dedupStates = /* @__PURE__ */ new Map();
1297
+ /** Circuit breaker state per `"${gid}:${topic}:${partition}"` key. */
1298
+ circuitStates = /* @__PURE__ */ new Map();
1299
+ /** Circuit breaker config per groupId, set at startConsumer/startBatchConsumer time. */
1300
+ circuitConfigs = /* @__PURE__ */ new Map();
1108
1301
  isAdminConnected = false;
1109
1302
  inFlightTotal = 0;
1110
1303
  drainResolvers = [];
@@ -1244,13 +1437,22 @@ var KafkaClient = class {
1244
1437
  "retryTopics requires retry to be configured \u2014 set retry.maxRetries to enable the retry topic chain"
1245
1438
  );
1246
1439
  }
1247
- const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", options);
1248
- const deps = this.messageDeps;
1440
+ const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
1441
+ const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", setupOptions);
1442
+ if (options.circuitBreaker)
1443
+ this.circuitConfigs.set(gid, options.circuitBreaker);
1444
+ const deps = this.messageDepsFor(gid);
1249
1445
  const timeoutMs = options.handlerTimeoutMs;
1250
1446
  const deduplication = this.resolveDeduplicationContext(
1251
1447
  gid,
1252
1448
  options.deduplication
1253
1449
  );
1450
+ let eosMainContext;
1451
+ if (options.retryTopics && retry) {
1452
+ const mainTxId = `${gid}-main-tx`;
1453
+ const txProducer = await this.createRetryTxProducer(mainTxId);
1454
+ eosMainContext = { txProducer, consumer };
1455
+ }
1254
1456
  await consumer.run({
1255
1457
  eachMessage: (payload) => this.trackInFlight(
1256
1458
  () => handleEachMessage(
@@ -1264,7 +1466,9 @@ var KafkaClient = class {
1264
1466
  retryTopics: options.retryTopics,
1265
1467
  timeoutMs,
1266
1468
  wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this),
1267
- deduplication
1469
+ deduplication,
1470
+ messageTtlMs: options.messageTtlMs,
1471
+ eosMainContext
1268
1472
  },
1269
1473
  deps
1270
1474
  )
@@ -1296,18 +1500,28 @@ var KafkaClient = class {
1296
1500
  "retryTopics requires retry to be configured \u2014 set retry.maxRetries to enable the retry topic chain"
1297
1501
  );
1298
1502
  }
1299
- if (options.autoCommit !== false) {
1503
+ if (options.retryTopics) {
1504
+ } else if (options.autoCommit !== false) {
1300
1505
  this.logger.debug?.(
1301
1506
  `startBatchConsumer: autoCommit is enabled (default true). If your handler calls resolveOffset() or commitOffsetsIfNecessary(), set autoCommit: false to avoid offset conflicts.`
1302
1507
  );
1303
1508
  }
1304
- const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", options);
1305
- const deps = this.messageDeps;
1509
+ const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
1510
+ const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", setupOptions);
1511
+ if (options.circuitBreaker)
1512
+ this.circuitConfigs.set(gid, options.circuitBreaker);
1513
+ const deps = this.messageDepsFor(gid);
1306
1514
  const timeoutMs = options.handlerTimeoutMs;
1307
1515
  const deduplication = this.resolveDeduplicationContext(
1308
1516
  gid,
1309
1517
  options.deduplication
1310
1518
  );
1519
+ let eosMainContext;
1520
+ if (options.retryTopics && retry) {
1521
+ const mainTxId = `${gid}-main-tx`;
1522
+ const txProducer = await this.createRetryTxProducer(mainTxId);
1523
+ eosMainContext = { txProducer, consumer };
1524
+ }
1311
1525
  await consumer.run({
1312
1526
  eachBatch: (payload) => this.trackInFlight(
1313
1527
  () => handleEachBatch(
@@ -1321,7 +1535,9 @@ var KafkaClient = class {
1321
1535
  retryTopics: options.retryTopics,
1322
1536
  timeoutMs,
1323
1537
  wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this),
1324
- deduplication
1538
+ deduplication,
1539
+ messageTtlMs: options.messageTtlMs,
1540
+ eosMainContext
1325
1541
  },
1326
1542
  deps
1327
1543
  )
@@ -1357,6 +1573,37 @@ var KafkaClient = class {
1357
1573
  }
1358
1574
  return { groupId: gid, stop: () => this.stopConsumer(gid) };
1359
1575
  }
1576
+ /**
1577
+ * Consume messages from a topic as an AsyncIterableIterator.
1578
+ * Use with `for await` — breaking out of the loop automatically stops the consumer.
1579
+ *
1580
+ * @example
1581
+ * for await (const envelope of kafka.consume('my.topic')) {
1582
+ * console.log(envelope.data);
1583
+ * }
1584
+ */
1585
+ consume(topic2, options) {
1586
+ const queue = new AsyncQueue();
1587
+ const handlePromise = this.startConsumer(
1588
+ [topic2],
1589
+ async (envelope) => {
1590
+ queue.push(envelope);
1591
+ },
1592
+ options
1593
+ );
1594
+ return {
1595
+ [Symbol.asyncIterator]() {
1596
+ return this;
1597
+ },
1598
+ next: () => queue.next(),
1599
+ return: async () => {
1600
+ queue.close();
1601
+ const handle = await handlePromise;
1602
+ await handle.stop();
1603
+ return { value: void 0, done: true };
1604
+ }
1605
+ };
1606
+ }
1360
1607
  // ── Consumer lifecycle ───────────────────────────────────────────
1361
1608
  async stopConsumer(groupId) {
1362
1609
  if (groupId !== void 0) {
@@ -1377,7 +1624,26 @@ var KafkaClient = class {
1377
1624
  this.runningConsumers.delete(groupId);
1378
1625
  this.consumerCreationOptions.delete(groupId);
1379
1626
  this.dedupStates.delete(groupId);
1627
+ for (const key of [...this.circuitStates.keys()]) {
1628
+ if (key.startsWith(`${groupId}:`)) {
1629
+ clearTimeout(this.circuitStates.get(key).timer);
1630
+ this.circuitStates.delete(key);
1631
+ }
1632
+ }
1633
+ this.circuitConfigs.delete(groupId);
1380
1634
  this.logger.log(`Consumer disconnected: group "${groupId}"`);
1635
+ const mainTxId = `${groupId}-main-tx`;
1636
+ const mainTxProducer = this.retryTxProducers.get(mainTxId);
1637
+ if (mainTxProducer) {
1638
+ await mainTxProducer.disconnect().catch(
1639
+ (e) => this.logger.warn(
1640
+ `Error disconnecting main tx producer "${mainTxId}":`,
1641
+ toError(e).message
1642
+ )
1643
+ );
1644
+ _activeTransactionalIds.delete(mainTxId);
1645
+ this.retryTxProducers.delete(mainTxId);
1646
+ }
1381
1647
  const companions = this.companionGroupIds.get(groupId) ?? [];
1382
1648
  for (const cGroupId of companions) {
1383
1649
  const cConsumer = this.consumers.get(cGroupId);
@@ -1425,9 +1691,175 @@ var KafkaClient = class {
1425
1691
  this.companionGroupIds.clear();
1426
1692
  this.retryTxProducers.clear();
1427
1693
  this.dedupStates.clear();
1694
+ for (const state of this.circuitStates.values())
1695
+ clearTimeout(state.timer);
1696
+ this.circuitStates.clear();
1697
+ this.circuitConfigs.clear();
1428
1698
  this.logger.log("All consumers disconnected");
1429
1699
  }
1430
1700
  }
1701
+ pauseConsumer(groupId, assignments) {
1702
+ const gid = groupId ?? this.defaultGroupId;
1703
+ const consumer = this.consumers.get(gid);
1704
+ if (!consumer) {
1705
+ this.logger.warn(`pauseConsumer: no active consumer for group "${gid}"`);
1706
+ return;
1707
+ }
1708
+ consumer.pause(
1709
+ assignments.flatMap(
1710
+ ({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] }))
1711
+ )
1712
+ );
1713
+ }
1714
+ resumeConsumer(groupId, assignments) {
1715
+ const gid = groupId ?? this.defaultGroupId;
1716
+ const consumer = this.consumers.get(gid);
1717
+ if (!consumer) {
1718
+ this.logger.warn(`resumeConsumer: no active consumer for group "${gid}"`);
1719
+ return;
1720
+ }
1721
+ consumer.resume(
1722
+ assignments.flatMap(
1723
+ ({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] }))
1724
+ )
1725
+ );
1726
+ }
1727
+ /** DLQ header keys added by `sendToDlq` — stripped before re-publishing. */
1728
+ static DLQ_HEADER_KEYS = /* @__PURE__ */ new Set([
1729
+ "x-dlq-original-topic",
1730
+ "x-dlq-failed-at",
1731
+ "x-dlq-error-message",
1732
+ "x-dlq-error-stack",
1733
+ "x-dlq-attempt-count"
1734
+ ]);
1735
+ async replayDlq(topic2, options = {}) {
1736
+ const dlqTopic = `${topic2}.dlq`;
1737
+ await this.ensureAdminConnected();
1738
+ const partitionOffsets = await this.admin.fetchTopicOffsets(dlqTopic);
1739
+ const activePartitions = partitionOffsets.filter(
1740
+ (p) => parseInt(p.high, 10) > 0
1741
+ );
1742
+ if (activePartitions.length === 0) {
1743
+ this.logger.log(`replayDlq: "${dlqTopic}" is empty \u2014 nothing to replay`);
1744
+ return { replayed: 0, skipped: 0 };
1745
+ }
1746
+ const highWatermarks = new Map(
1747
+ activePartitions.map(({ partition, high }) => [
1748
+ partition,
1749
+ parseInt(high, 10)
1750
+ ])
1751
+ );
1752
+ const processedOffsets = /* @__PURE__ */ new Map();
1753
+ let replayed = 0;
1754
+ let skipped = 0;
1755
+ const tempGroupId = `${dlqTopic}-replay-${Date.now()}`;
1756
+ await new Promise((resolve, reject) => {
1757
+ const consumer = getOrCreateConsumer(
1758
+ tempGroupId,
1759
+ true,
1760
+ true,
1761
+ this.consumerOpsDeps
1762
+ );
1763
+ const cleanup = () => {
1764
+ consumer.disconnect().catch(() => {
1765
+ }).finally(() => {
1766
+ this.consumers.delete(tempGroupId);
1767
+ this.runningConsumers.delete(tempGroupId);
1768
+ this.consumerCreationOptions.delete(tempGroupId);
1769
+ });
1770
+ };
1771
+ consumer.connect().then(() => subscribeWithRetry(consumer, [dlqTopic], this.logger)).then(
1772
+ () => consumer.run({
1773
+ eachMessage: async ({ partition, message }) => {
1774
+ if (!message.value) return;
1775
+ const offset = parseInt(message.offset, 10);
1776
+ processedOffsets.set(partition, offset);
1777
+ const headers = decodeHeaders(message.headers);
1778
+ const targetTopic = options.targetTopic ?? headers["x-dlq-original-topic"];
1779
+ const originalHeaders = Object.fromEntries(
1780
+ Object.entries(headers).filter(
1781
+ ([k]) => !_KafkaClient.DLQ_HEADER_KEYS.has(k)
1782
+ )
1783
+ );
1784
+ const value = message.value.toString();
1785
+ const shouldProcess = !options.filter || options.filter(headers, value);
1786
+ if (!targetTopic || !shouldProcess) {
1787
+ skipped++;
1788
+ } else if (options.dryRun) {
1789
+ this.logger.log(
1790
+ `[DLQ replay dry-run] Would replay to "${targetTopic}"`
1791
+ );
1792
+ replayed++;
1793
+ } else {
1794
+ await this.producer.send({
1795
+ topic: targetTopic,
1796
+ messages: [{ value, headers: originalHeaders }]
1797
+ });
1798
+ replayed++;
1799
+ }
1800
+ const allDone = Array.from(highWatermarks.entries()).every(
1801
+ ([p, hwm]) => (processedOffsets.get(p) ?? -1) >= hwm - 1
1802
+ );
1803
+ if (allDone) {
1804
+ cleanup();
1805
+ resolve();
1806
+ }
1807
+ }
1808
+ })
1809
+ ).catch((err) => {
1810
+ cleanup();
1811
+ reject(err);
1812
+ });
1813
+ });
1814
+ this.logger.log(
1815
+ `replayDlq: replayed ${replayed}, skipped ${skipped} from "${dlqTopic}"`
1816
+ );
1817
+ return { replayed, skipped };
1818
+ }
1819
+ async resetOffsets(groupId, topic2, position) {
1820
+ const gid = groupId ?? this.defaultGroupId;
1821
+ if (this.runningConsumers.has(gid)) {
1822
+ throw new Error(
1823
+ `resetOffsets: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before resetting offsets.`
1824
+ );
1825
+ }
1826
+ await this.ensureAdminConnected();
1827
+ const partitionOffsets = await this.admin.fetchTopicOffsets(topic2);
1828
+ const partitions = partitionOffsets.map(({ partition, low, high }) => ({
1829
+ partition,
1830
+ offset: position === "earliest" ? low : high
1831
+ }));
1832
+ await this.admin.setOffsets({ groupId: gid, topic: topic2, partitions });
1833
+ this.logger.log(
1834
+ `Offsets reset to ${position} for group "${gid}" on topic "${topic2}"`
1835
+ );
1836
+ }
1837
+ /**
1838
+ * Seek specific topic-partition pairs to explicit offsets for a stopped consumer group.
1839
+ * Throws if the group is still running — call `stopConsumer(groupId)` first.
1840
+ * Assignments are grouped by topic and committed via `admin.setOffsets`.
1841
+ */
1842
+ async seekToOffset(groupId, assignments) {
1843
+ const gid = groupId ?? this.defaultGroupId;
1844
+ if (this.runningConsumers.has(gid)) {
1845
+ throw new Error(
1846
+ `seekToOffset: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before seeking offsets.`
1847
+ );
1848
+ }
1849
+ await this.ensureAdminConnected();
1850
+ const byTopic = /* @__PURE__ */ new Map();
1851
+ for (const { topic: topic2, partition, offset } of assignments) {
1852
+ const list = byTopic.get(topic2) ?? [];
1853
+ list.push({ partition, offset });
1854
+ byTopic.set(topic2, list);
1855
+ }
1856
+ for (const [topic2, partitions] of byTopic) {
1857
+ await this.admin.setOffsets({ groupId: gid, topic: topic2, partitions });
1858
+ this.logger.log(
1859
+ `Offsets set for group "${gid}" on "${topic2}": ${JSON.stringify(partitions)}`
1860
+ );
1861
+ }
1862
+ }
1431
1863
  /**
1432
1864
  * Query consumer group lag per partition.
1433
1865
  * Lag = broker high-watermark − last committed offset.
@@ -1478,14 +1910,31 @@ var KafkaClient = class {
1478
1910
  getClientId() {
1479
1911
  return this.clientId;
1480
1912
  }
1481
- getMetrics() {
1482
- return { ...this._metrics };
1913
+ getMetrics(topic2) {
1914
+ if (topic2 !== void 0) {
1915
+ const m = this._topicMetrics.get(topic2);
1916
+ return m ? { ...m } : { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
1917
+ }
1918
+ const agg = {
1919
+ processedCount: 0,
1920
+ retryCount: 0,
1921
+ dlqCount: 0,
1922
+ dedupCount: 0
1923
+ };
1924
+ for (const m of this._topicMetrics.values()) {
1925
+ agg.processedCount += m.processedCount;
1926
+ agg.retryCount += m.retryCount;
1927
+ agg.dlqCount += m.dlqCount;
1928
+ agg.dedupCount += m.dedupCount;
1929
+ }
1930
+ return agg;
1483
1931
  }
1484
- resetMetrics() {
1485
- this._metrics.processedCount = 0;
1486
- this._metrics.retryCount = 0;
1487
- this._metrics.dlqCount = 0;
1488
- this._metrics.dedupCount = 0;
1932
+ resetMetrics(topic2) {
1933
+ if (topic2 !== void 0) {
1934
+ this._topicMetrics.delete(topic2);
1935
+ return;
1936
+ }
1937
+ this._topicMetrics.clear();
1489
1938
  }
1490
1939
  /** Gracefully disconnect producer, all consumers, and admin. */
1491
1940
  async disconnect(drainTimeoutMs = 3e4) {
@@ -1516,6 +1965,9 @@ var KafkaClient = class {
1516
1965
  this.runningConsumers.clear();
1517
1966
  this.consumerCreationOptions.clear();
1518
1967
  this.companionGroupIds.clear();
1968
+ for (const state of this.circuitStates.values()) clearTimeout(state.timer);
1969
+ this.circuitStates.clear();
1970
+ this.circuitConfigs.clear();
1519
1971
  this.logger.log("All connections closed");
1520
1972
  }
1521
1973
  // ── Graceful shutdown ────────────────────────────────────────────
@@ -1595,29 +2047,112 @@ var KafkaClient = class {
1595
2047
  }
1596
2048
  }
1597
2049
  }
2050
+ metricsFor(topic2) {
2051
+ let m = this._topicMetrics.get(topic2);
2052
+ if (!m) {
2053
+ m = { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
2054
+ this._topicMetrics.set(topic2, m);
2055
+ }
2056
+ return m;
2057
+ }
1598
2058
  notifyRetry(envelope, attempt, maxRetries) {
1599
- this._metrics.retryCount++;
2059
+ this.metricsFor(envelope.topic).retryCount++;
1600
2060
  for (const inst of this.instrumentation) {
1601
2061
  inst.onRetry?.(envelope, attempt, maxRetries);
1602
2062
  }
1603
2063
  }
1604
- notifyDlq(envelope, reason) {
1605
- this._metrics.dlqCount++;
2064
+ notifyDlq(envelope, reason, gid) {
2065
+ this.metricsFor(envelope.topic).dlqCount++;
1606
2066
  for (const inst of this.instrumentation) {
1607
2067
  inst.onDlq?.(envelope, reason);
1608
2068
  }
2069
+ if (!gid) return;
2070
+ const cfg = this.circuitConfigs.get(gid);
2071
+ if (!cfg) return;
2072
+ const threshold = cfg.threshold ?? 5;
2073
+ const recoveryMs = cfg.recoveryMs ?? 3e4;
2074
+ const stateKey = `${gid}:${envelope.topic}:${envelope.partition}`;
2075
+ let state = this.circuitStates.get(stateKey);
2076
+ if (!state) {
2077
+ state = { status: "closed", window: [], successes: 0 };
2078
+ this.circuitStates.set(stateKey, state);
2079
+ }
2080
+ if (state.status === "open") return;
2081
+ const openCircuit = () => {
2082
+ state.status = "open";
2083
+ this.pauseConsumer(gid, [
2084
+ { topic: envelope.topic, partitions: [envelope.partition] }
2085
+ ]);
2086
+ state.timer = setTimeout(() => {
2087
+ state.status = "half-open";
2088
+ state.successes = 0;
2089
+ this.logger.log(
2090
+ `[CircuitBreaker] HALF-OPEN \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
2091
+ );
2092
+ this.resumeConsumer(gid, [
2093
+ { topic: envelope.topic, partitions: [envelope.partition] }
2094
+ ]);
2095
+ }, recoveryMs);
2096
+ };
2097
+ if (state.status === "half-open") {
2098
+ clearTimeout(state.timer);
2099
+ this.logger.warn(
2100
+ `[CircuitBreaker] OPEN (half-open failure) \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
2101
+ );
2102
+ openCircuit();
2103
+ return;
2104
+ }
2105
+ const windowSize = cfg.windowSize ?? Math.max(threshold * 2, 10);
2106
+ state.window = [...state.window, false];
2107
+ if (state.window.length > windowSize) {
2108
+ state.window = state.window.slice(state.window.length - windowSize);
2109
+ }
2110
+ const failures = state.window.filter((v) => !v).length;
2111
+ if (failures >= threshold) {
2112
+ this.logger.warn(
2113
+ `[CircuitBreaker] OPEN \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition} (${failures}/${state.window.length} failures, threshold=${threshold})`
2114
+ );
2115
+ openCircuit();
2116
+ }
1609
2117
  }
1610
2118
  notifyDuplicate(envelope, strategy) {
1611
- this._metrics.dedupCount++;
2119
+ this.metricsFor(envelope.topic).dedupCount++;
1612
2120
  for (const inst of this.instrumentation) {
1613
2121
  inst.onDuplicate?.(envelope, strategy);
1614
2122
  }
1615
2123
  }
1616
- notifyMessage(envelope) {
1617
- this._metrics.processedCount++;
2124
+ notifyMessage(envelope, gid) {
2125
+ this.metricsFor(envelope.topic).processedCount++;
1618
2126
  for (const inst of this.instrumentation) {
1619
2127
  inst.onMessage?.(envelope);
1620
2128
  }
2129
+ if (!gid) return;
2130
+ const cfg = this.circuitConfigs.get(gid);
2131
+ if (!cfg) return;
2132
+ const stateKey = `${gid}:${envelope.topic}:${envelope.partition}`;
2133
+ const state = this.circuitStates.get(stateKey);
2134
+ if (!state) return;
2135
+ const halfOpenSuccesses = cfg.halfOpenSuccesses ?? 1;
2136
+ if (state.status === "half-open") {
2137
+ state.successes++;
2138
+ if (state.successes >= halfOpenSuccesses) {
2139
+ clearTimeout(state.timer);
2140
+ state.timer = void 0;
2141
+ state.status = "closed";
2142
+ state.window = [];
2143
+ state.successes = 0;
2144
+ this.logger.log(
2145
+ `[CircuitBreaker] CLOSED \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
2146
+ );
2147
+ }
2148
+ } else if (state.status === "closed") {
2149
+ const threshold = cfg.threshold ?? 5;
2150
+ const windowSize = cfg.windowSize ?? Math.max(threshold * 2, 10);
2151
+ state.window = [...state.window, true];
2152
+ if (state.window.length > windowSize) {
2153
+ state.window = state.window.slice(state.window.length - windowSize);
2154
+ }
2155
+ }
1621
2156
  }
1622
2157
  /**
1623
2158
  * Start a timer that logs a warning if `fn` hasn't resolved within `timeoutMs`.
@@ -1838,16 +2373,17 @@ var KafkaClient = class {
1838
2373
  logger: this.logger
1839
2374
  };
1840
2375
  }
1841
- get messageDeps() {
2376
+ /** Build MessageHandlerDeps with circuit breaker callbacks bound to the given groupId. */
2377
+ messageDepsFor(gid) {
1842
2378
  return {
1843
2379
  logger: this.logger,
1844
2380
  producer: this.producer,
1845
2381
  instrumentation: this.instrumentation,
1846
2382
  onMessageLost: this.onMessageLost,
1847
2383
  onRetry: this.notifyRetry.bind(this),
1848
- onDlq: this.notifyDlq.bind(this),
2384
+ onDlq: (envelope, reason) => this.notifyDlq(envelope, reason, gid),
1849
2385
  onDuplicate: this.notifyDuplicate.bind(this),
1850
- onMessage: this.notifyMessage.bind(this)
2386
+ onMessage: (envelope) => this.notifyMessage(envelope, gid)
1851
2387
  };
1852
2388
  }
1853
2389
  get retryTopicDeps() {