@drarzter/kafka-client 0.6.6 → 0.6.7

package/README.md

@@ -620,7 +620,7 @@ await kafka.startBatchConsumer(
 | Property/Method | Description |
 | --------------- | ----------- |
 | `partition` | Partition number for this batch |
-| `highWatermark` | Latest offset in the partition (lag indicator) |
+| `highWatermark` | Latest offset in the partition (`string`). `null` when the message is replayed via a retry topic consumer — in that path the broker high-watermark is not available. Guard against `null` before computing lag |
 | `heartbeat()` | Send a heartbeat to keep the consumer session alive — call during long processing loops |
 | `resolveOffset(offset)` | Mark offset as processed (required before `commitOffsetsIfNecessary`) |
 | `commitOffsetsIfNecessary()` | Commit resolved offsets; respects `autoCommit` setting |
@@ -743,6 +743,47 @@ type BeforeConsumeResult =
 
 When multiple instrumentations each provide a `wrap`, they compose in declaration order — the first instrumentation's `wrap` is the outermost.
 
+### Lifecycle event hooks
+
+Three additional hooks fire for specific events in the consume pipeline:
+
+| Hook | When called | Arguments |
+| ---- | ----------- | --------- |
+| `onMessage` | Handler successfully processed a message | `(envelope)` — use as a success counter for error-rate calculations |
+| `onRetry` | A message is queued for another attempt (in-process backoff or routed to a retry topic) | `(envelope, attempt, maxRetries)` |
+| `onDlq` | A message is routed to the dead letter queue | `(envelope, reason)` — reason is `'handler-error'`, `'validation-error'`, or `'lamport-clock-duplicate'` |
+| `onDuplicate` | A duplicate is detected via Lamport Clock | `(envelope, strategy)` — strategy is `'drop'`, `'dlq'`, or `'topic'` |
+
+```typescript
+const myInstrumentation: KafkaInstrumentation = {
+  onMessage(envelope) {
+    metrics.increment('kafka.processed', { topic: envelope.topic });
+  },
+  onRetry(envelope, attempt, maxRetries) {
+    console.warn(`Retrying ${envelope.topic} — attempt ${attempt}/${maxRetries}`);
+  },
+  onDlq(envelope, reason) {
+    alertingSystem.send({ topic: envelope.topic, reason });
+  },
+  onDuplicate(envelope, strategy) {
+    metrics.increment('kafka.duplicate', { topic: envelope.topic, strategy });
+  },
+};
+```
+
+### Built-in metrics
+
+`KafkaClient` maintains lightweight in-process event counters independently of any instrumentation:
+
+```typescript
+const snapshot = kafka.getMetrics();
+// { processedCount: number; retryCount: number; dlqCount: number; dedupCount: number }
+
+kafka.resetMetrics(); // reset all counters to zero
+```
+
+Counters are incremented in the same code paths that fire the corresponding hooks — they are always active regardless of whether any instrumentation is configured.
+
 ## Options reference
 
 ### Send options
@@ -795,6 +836,7 @@ Passed to `KafkaModule.register()` or returned from `registerAsync()` factory:
 | `numPartitions` | `1` | Number of partitions for auto-created topics |
 | `strictSchemas` | `true` | Validate string topic keys against schemas registered via TopicDescriptor |
 | `instrumentation` | `[]` | Client-wide instrumentation hooks (e.g. OTel). Applied to both send and consume paths |
+| `transactionalId` | `${clientId}-tx` | Transactional producer ID for `transaction()` calls. Must be unique per producer instance across the cluster — two instances sharing the same ID will be fenced by Kafka. The client logs a warning when the same ID is registered twice within one process |
 | `onMessageLost` | — | Called when a message is silently dropped without DLQ — use to alert, log to external systems, or trigger fallback logic |
 | `onRebalance` | — | Called on every partition assign/revoke event across all consumers created by this client |
 

package/dist/{chunk-KCUKXR6B.mjs → chunk-ISYOEX4W.mjs}

@@ -318,7 +318,10 @@ async function sendToDlq(topic2, rawMessage, deps, meta) {
     deps.logger.warn(`Message sent to DLQ: ${payload.topic}`);
   } catch (error) {
     const err = toError(error);
-    deps.logger.error(`Failed to send message to DLQ ${payload.topic}:`, err.stack);
+    deps.logger.error(
+      `Failed to send message to DLQ ${payload.topic}:`,
+      err.stack
+    );
     await deps.onMessageLost?.({
       topic: topic2,
       error: err,
@@ -333,14 +336,9 @@ var RETRY_HEADER_MAX_RETRIES = "x-retry-max-retries";
 var RETRY_HEADER_ORIGINAL_TOPIC = "x-retry-original-topic";
 function buildRetryTopicPayload(originalTopic, rawMessages, attempt, maxRetries, delayMs, originalHeaders) {
   const retryTopic = `${originalTopic}.retry.${attempt}`;
+  const STRIP = /* @__PURE__ */ new Set([RETRY_HEADER_ATTEMPT, RETRY_HEADER_AFTER, RETRY_HEADER_MAX_RETRIES, RETRY_HEADER_ORIGINAL_TOPIC]);
   function buildHeaders(hdr) {
-    const {
-      [RETRY_HEADER_ATTEMPT]: _a,
-      [RETRY_HEADER_AFTER]: _b,
-      [RETRY_HEADER_MAX_RETRIES]: _c,
-      [RETRY_HEADER_ORIGINAL_TOPIC]: _d,
-      ...userHeaders
-    } = hdr;
+    const userHeaders = Object.fromEntries(Object.entries(hdr).filter(([k]) => !STRIP.has(k)));
     return {
       ...userHeaders,
       [RETRY_HEADER_ATTEMPT]: String(attempt),
@@ -396,10 +394,18 @@ function buildDuplicateTopicPayload(sourceTopic, rawMessage, destinationTopic, m
     "x-duplicate-incoming-clock": String(meta?.incomingClock ?? 0),
     "x-duplicate-last-processed-clock": String(meta?.lastProcessedClock ?? 0)
   };
-  return { topic: destinationTopic, messages: [{ value: rawMessage, headers }] };
+  return {
+    topic: destinationTopic,
+    messages: [{ value: rawMessage, headers }]
+  };
 }
 async function sendToDuplicatesTopic(sourceTopic, rawMessage, destinationTopic, deps, meta) {
-  const payload = buildDuplicateTopicPayload(sourceTopic, rawMessage, destinationTopic, meta);
+  const payload = buildDuplicateTopicPayload(
+    sourceTopic,
+    rawMessage,
+    destinationTopic,
+    meta
+  );
   try {
     await deps.producer.send(payload);
     deps.logger.warn(`Duplicate message forwarded to ${destinationTopic}`);
@@ -491,7 +497,10 @@ async function executeWithRetry(fn, ctx, deps) {
       interceptors,
       deps.instrumentation
     );
-    if (!error) return;
+    if (!error) {
+      for (const env of envelopes) deps.onMessage?.(env);
+      return;
+    }
     const isLastAttempt = attempt === maxAttempts;
     const reportedError = isLastAttempt && maxAttempts > 1 ? new KafkaRetryExhaustedError(
       topic2,
@@ -516,6 +525,7 @@ async function executeWithRetry(fn, ctx, deps) {
         isBatch ? envelopes.map((e) => e.headers) : envelopes[0]?.headers ?? {},
         deps
       );
+      deps.onRetry?.(envelopes[0], 1, retry.maxRetries);
     } else if (isLastAttempt) {
       if (dlq) {
         for (let i = 0; i < rawMessages.length; i++) {
@@ -524,6 +534,7 @@ async function executeWithRetry(fn, ctx, deps) {
             attempt,
             originalHeaders: envelopes[i]?.headers
           });
+          deps.onDlq?.(envelopes[i] ?? envelopes[0], "handler-error");
         }
       } else {
         await deps.onMessageLost?.({
@@ -535,6 +546,7 @@ async function executeWithRetry(fn, ctx, deps) {
       }
     } else {
       const cap = Math.min(backoffMs * 2 ** (attempt - 1), maxBackoffMs);
+      deps.onRetry?.(envelopes[0], attempt, maxAttempts - 1);
       await sleep(Math.floor(Math.random() * cap));
     }
   }
@@ -558,6 +570,7 @@ async function applyDeduplication(envelope, raw, dedup, dlq, deps) {
     deps.logger.warn(
       `Duplicate message on ${envelope.topic}[${envelope.partition}]: clock=${incomingClock} <= last=${lastProcessedClock} \u2014 strategy=${strategy}`
     );
+    deps.onDuplicate?.(envelope, strategy);
     if (strategy === "dlq" && dlq) {
       const augmentedHeaders = {
         ...envelope.headers,
@@ -761,6 +774,9 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
     producer,
     instrumentation,
     onMessageLost,
+    onRetry,
+    onDlq,
+    onMessage,
     ensureTopic,
     getOrCreateConsumer: getOrCreateConsumer2,
     runningConsumers,
@@ -842,6 +858,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
         instrumentation
       );
       if (!error) {
+        onMessage?.(envelope);
         await consumer.commitOffsets([nextOffset]);
         return;
       }
@@ -874,12 +891,23 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
           await tx.send({ topic: rtTopic, messages: rtMsgs });
           await tx.sendOffsets({
             consumer,
-            topics: [{ topic: nextOffset.topic, partitions: [{ partition: nextOffset.partition, offset: nextOffset.offset }] }]
+            topics: [
+              {
+                topic: nextOffset.topic,
+                partitions: [
+                  {
+                    partition: nextOffset.partition,
+                    offset: nextOffset.offset
+                  }
+                ]
+              }
+            ]
           });
           await tx.commit();
           logger.warn(
             `Message routed to ${rtTopic} (EOS, level ${nextLevel}/${currentMaxRetries})`
           );
+          onRetry?.(envelope, nextLevel, currentMaxRetries);
         } catch (txErr) {
           try {
             await tx.abort();
@@ -907,10 +935,21 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
           await tx.send({ topic: dTopic, messages: dMsgs });
           await tx.sendOffsets({
             consumer,
-            topics: [{ topic: nextOffset.topic, partitions: [{ partition: nextOffset.partition, offset: nextOffset.offset }] }]
+            topics: [
+              {
+                topic: nextOffset.topic,
+                partitions: [
+                  {
+                    partition: nextOffset.partition,
+                    offset: nextOffset.offset
+                  }
+                ]
+              }
+            ]
           });
           await tx.commit();
           logger.warn(`Message sent to DLQ: ${dTopic} (EOS)`);
+          onDlq?.(envelope, "handler-error");
         } catch (txErr) {
           try {
             await tx.abort();
@@ -934,7 +973,12 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
     }
   });
   runningConsumers.set(levelGroupId, "eachMessage");
-  await waitForPartitionAssignment(consumer, levelTopics, logger, assignmentTimeoutMs);
+  await waitForPartitionAssignment(
+    consumer,
+    levelTopics,
+    logger,
+    assignmentTimeoutMs
+  );
   logger.log(
     `Retry level ${level}/${retry.maxRetries} consumer started for: ${originalTopics.join(", ")} (group: ${levelGroupId})`
   );
@@ -964,6 +1008,7 @@ async function startRetryTopicConsumers(originalTopics, originalGroupId, handleM
 
 // src/client/kafka.client/index.ts
 var { Kafka: KafkaClass, logLevel: KafkaLogLevel } = KafkaJS;
+var _activeTransactionalIds = /* @__PURE__ */ new Set();
 var KafkaClient = class {
   kafka;
   producer;
@@ -989,6 +1034,15 @@ var KafkaClient = class {
   instrumentation;
   onMessageLost;
   onRebalance;
+  /** Transactional producer ID — configurable via `KafkaClientOptions.transactionalId`. */
+  txId;
+  /** Internal event counters exposed via `getMetrics()`. */
+  _metrics = {
+    processedCount: 0,
+    retryCount: 0,
+    dlqCount: 0,
+    dedupCount: 0
+  };
   /** Monotonically increasing Lamport clock stamped on every outgoing message. */
   _lamportClock = 0;
   /** Per-groupId deduplication state: `"topic:partition"` → last processed clock. */
@@ -1012,6 +1066,7 @@ var KafkaClient = class {
     this.instrumentation = options?.instrumentation ?? [];
     this.onMessageLost = options?.onMessageLost;
     this.onRebalance = options?.onRebalance;
+    this.txId = options?.transactionalId ?? `${clientId}-tx`;
     this.kafka = new KafkaClass({
       kafkaJS: {
         clientId: this.clientId,
@@ -1048,16 +1103,22 @@ var KafkaClient = class {
   /** Execute multiple sends atomically. Commits on success, aborts on error. */
   async transaction(fn) {
     if (!this.txProducerInitPromise) {
+      if (_activeTransactionalIds.has(this.txId)) {
+        this.logger.warn(
+          `transactionalId "${this.txId}" is already in use by another KafkaClient in this process. Kafka will fence one of the producers. Set a unique \`transactionalId\` (or distinct \`clientId\`) per instance.`
+        );
+      }
       const initPromise = (async () => {
         const p = this.kafka.producer({
           kafkaJS: {
             acks: -1,
             idempotent: true,
-            transactionalId: `${this.clientId}-tx`,
+            transactionalId: this.txId,
             maxInFlightRequests: 1
           }
         });
         await p.connect();
+        _activeTransactionalIds.add(this.txId);
         return p;
       })();
       this.txProducerInitPromise = initPromise.catch((err) => {
@@ -1128,7 +1189,10 @@ var KafkaClient = class {
     const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", options);
     const deps = this.messageDeps;
     const timeoutMs = options.handlerTimeoutMs;
-    const deduplication = this.resolveDeduplicationContext(gid, options.deduplication);
+    const deduplication = this.resolveDeduplicationContext(
+      gid,
+      options.deduplication
+    );
     await consumer.run({
       eachMessage: (payload) => this.trackInFlight(
         () => handleEachMessage(
@@ -1182,7 +1246,10 @@ var KafkaClient = class {
     const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", options);
     const deps = this.messageDeps;
     const timeoutMs = options.handlerTimeoutMs;
-    const deduplication = this.resolveDeduplicationContext(gid, options.deduplication);
+    const deduplication = this.resolveDeduplicationContext(
+      gid,
+      options.deduplication
+    );
     await consumer.run({
       eachBatch: (payload) => this.trackInFlight(
         () => handleEachBatch(
@@ -1209,7 +1276,7 @@ var KafkaClient = class {
       }
       const handleMessageForRetry = (env) => handleBatch([env], {
         partition: env.partition,
-        highWatermark: env.offset,
+        highWatermark: null,
         heartbeat: async () => {
         },
         resolveOffset: () => {
@@ -1277,6 +1344,7 @@ var KafkaClient = class {
               toError(e).message
             )
           );
+          _activeTransactionalIds.delete(txId);
           this.retryTxProducers.delete(txId);
         }
       }
@@ -1352,15 +1420,28 @@ var KafkaClient = class {
   getClientId() {
     return this.clientId;
   }
+  getMetrics() {
+    return { ...this._metrics };
+  }
+  resetMetrics() {
+    this._metrics.processedCount = 0;
+    this._metrics.retryCount = 0;
+    this._metrics.dlqCount = 0;
+    this._metrics.dedupCount = 0;
+  }
   /** Gracefully disconnect producer, all consumers, and admin. */
   async disconnect(drainTimeoutMs = 3e4) {
     await this.waitForDrain(drainTimeoutMs);
     const tasks = [this.producer.disconnect()];
     if (this.txProducer) {
       tasks.push(this.txProducer.disconnect());
+      _activeTransactionalIds.delete(this.txId);
       this.txProducer = void 0;
       this.txProducerInitPromise = void 0;
     }
+    for (const txId of this.retryTxProducers.keys()) {
+      _activeTransactionalIds.delete(txId);
+    }
     for (const p of this.retryTxProducers.values()) {
       tasks.push(p.disconnect());
     }
@@ -1456,6 +1537,30 @@ var KafkaClient = class {
       }
     }
   }
+  notifyRetry(envelope, attempt, maxRetries) {
+    this._metrics.retryCount++;
+    for (const inst of this.instrumentation) {
+      inst.onRetry?.(envelope, attempt, maxRetries);
+    }
+  }
+  notifyDlq(envelope, reason) {
+    this._metrics.dlqCount++;
+    for (const inst of this.instrumentation) {
+      inst.onDlq?.(envelope, reason);
+    }
+  }
+  notifyDuplicate(envelope, strategy) {
+    this._metrics.dedupCount++;
+    for (const inst of this.instrumentation) {
+      inst.onDuplicate?.(envelope, strategy);
+    }
+  }
+  notifyMessage(envelope) {
+    this._metrics.processedCount++;
+    for (const inst of this.instrumentation) {
+      inst.onMessage?.(envelope);
+    }
+  }
   /**
    * Start a timer that logs a warning if `fn` hasn't resolved within `timeoutMs`.
    * The handler itself is not cancelled — the warning is diagnostic only.
@@ -1545,6 +1650,11 @@ var KafkaClient = class {
    * so Kafka can fence stale producers on restart without affecting other levels.
    */
   async createRetryTxProducer(transactionalId) {
+    if (_activeTransactionalIds.has(transactionalId)) {
+      this.logger.warn(
+        `transactionalId "${transactionalId}" is already in use by another KafkaClient in this process. Kafka will fence one of the producers. Set a unique \`transactionalId\` (or distinct \`clientId\`) per instance.`
+      );
+    }
     const p = this.kafka.producer({
       kafkaJS: {
         acks: -1,
@@ -1554,6 +1664,7 @@ var KafkaClient = class {
       }
     });
     await p.connect();
+    _activeTransactionalIds.add(transactionalId);
     this.retryTxProducers.set(transactionalId, p);
     return p;
   }
@@ -1674,7 +1785,11 @@ var KafkaClient = class {
       logger: this.logger,
       producer: this.producer,
       instrumentation: this.instrumentation,
-      onMessageLost: this.onMessageLost
+      onMessageLost: this.onMessageLost,
+      onRetry: this.notifyRetry.bind(this),
+      onDlq: this.notifyDlq.bind(this),
+      onDuplicate: this.notifyDuplicate.bind(this),
+      onMessage: this.notifyMessage.bind(this)
     };
   }
   get retryTopicDeps() {
@@ -1683,6 +1798,9 @@ var KafkaClient = class {
       producer: this.producer,
       instrumentation: this.instrumentation,
       onMessageLost: this.onMessageLost,
+      onRetry: this.notifyRetry.bind(this),
+      onDlq: this.notifyDlq.bind(this),
+      onMessage: this.notifyMessage.bind(this),
       ensureTopic: (t) => this.ensureTopic(t),
       getOrCreateConsumer: (gid, fb, ac) => getOrCreateConsumer(gid, fb, ac, this.consumerOpsDeps),
       runningConsumers: this.runningConsumers,
@@ -1725,4 +1843,4 @@ export {
   KafkaClient,
   topic
 };
-//# sourceMappingURL=chunk-KCUKXR6B.mjs.map
+//# sourceMappingURL=chunk-ISYOEX4W.mjs.map