@drarzter/kafka-client 0.6.6 → 0.6.7

package/dist/core.mjs

@@ -15,7 +15,7 @@ import {
   getEnvelopeContext,
   runWithEnvelopeContext,
   topic
-} from "./chunk-KCUKXR6B.mjs";
+} from "./chunk-ISYOEX4W.mjs";
 import "./chunk-EQQGB2QZ.mjs";
 export {
   HEADER_CORRELATION_ID,

package/dist/index.d.mts

@@ -1,6 +1,6 @@
 export { KafkaClient, KafkaProcessingError, KafkaRetryExhaustedError, KafkaValidationError } from './core.mjs';
-import { T as TopicMapConstraint, C as ClientId, G as GroupId, K as KafkaInstrumentation, a as KafkaClientOptions, S as SchemaLike, b as ConsumerOptions, c as TopicDescriptor, I as IKafkaClient, d as KafkaHealthResult } from './types-CTwLrJVU.mjs';
-export { B as BatchMessageItem, e as BatchMeta, f as BeforeConsumeResult, g as ConsumerHandle, h as ConsumerInterceptor, D as DeduplicationOptions, E as EnvelopeHeaderOptions, i as EventEnvelope, H as HEADER_CORRELATION_ID, j as HEADER_EVENT_ID, k as HEADER_LAMPORT_CLOCK, l as HEADER_SCHEMA_VERSION, m as HEADER_TIMESTAMP, n as HEADER_TRACEPARENT, o as InferSchema, p as KafkaLogger, M as MessageHeaders, q as MessageLostContext, R as RetryOptions, r as SchemaParseContext, s as SendOptions, t as SubscribeRetryOptions, u as TTopicMessageMap, v as TopicsFrom, w as TransactionContext, x as buildEnvelopeHeaders, y as decodeHeaders, z as extractEnvelope, A as getEnvelopeContext, F as runWithEnvelopeContext, J as topic } from './types-CTwLrJVU.mjs';
+import { T as TopicMapConstraint, C as ClientId, G as GroupId, K as KafkaInstrumentation, a as KafkaClientOptions, S as SchemaLike, b as ConsumerOptions, c as TopicDescriptor, I as IKafkaClient, d as KafkaHealthResult } from './types-CqjRm-Cd.mjs';
+export { B as BatchMessageItem, e as BatchMeta, f as BeforeConsumeResult, g as ConsumerHandle, h as ConsumerInterceptor, D as DeduplicationOptions, i as DlqReason, E as EnvelopeHeaderOptions, j as EventEnvelope, H as HEADER_CORRELATION_ID, k as HEADER_EVENT_ID, l as HEADER_LAMPORT_CLOCK, m as HEADER_SCHEMA_VERSION, n as HEADER_TIMESTAMP, o as HEADER_TRACEPARENT, p as InferSchema, q as KafkaLogger, r as KafkaMetrics, M as MessageHeaders, s as MessageLostContext, R as RetryOptions, t as SchemaParseContext, u as SendOptions, v as SubscribeRetryOptions, w as TTopicMessageMap, x as TopicsFrom, y as TransactionContext, z as buildEnvelopeHeaders, A as decodeHeaders, F as extractEnvelope, J as getEnvelopeContext, L as runWithEnvelopeContext, N as topic } from './types-CqjRm-Cd.mjs';
 import { DynamicModule, OnModuleInit } from '@nestjs/common';
 import { DiscoveryService, ModuleRef } from '@nestjs/core';
 

package/dist/index.d.ts

@@ -1,6 +1,6 @@
 export { KafkaClient, KafkaProcessingError, KafkaRetryExhaustedError, KafkaValidationError } from './core.js';
-import { T as TopicMapConstraint, C as ClientId, G as GroupId, K as KafkaInstrumentation, a as KafkaClientOptions, S as SchemaLike, b as ConsumerOptions, c as TopicDescriptor, I as IKafkaClient, d as KafkaHealthResult } from './types-CTwLrJVU.js';
-export { B as BatchMessageItem, e as BatchMeta, f as BeforeConsumeResult, g as ConsumerHandle, h as ConsumerInterceptor, D as DeduplicationOptions, E as EnvelopeHeaderOptions, i as EventEnvelope, H as HEADER_CORRELATION_ID, j as HEADER_EVENT_ID, k as HEADER_LAMPORT_CLOCK, l as HEADER_SCHEMA_VERSION, m as HEADER_TIMESTAMP, n as HEADER_TRACEPARENT, o as InferSchema, p as KafkaLogger, M as MessageHeaders, q as MessageLostContext, R as RetryOptions, r as SchemaParseContext, s as SendOptions, t as SubscribeRetryOptions, u as TTopicMessageMap, v as TopicsFrom, w as TransactionContext, x as buildEnvelopeHeaders, y as decodeHeaders, z as extractEnvelope, A as getEnvelopeContext, F as runWithEnvelopeContext, J as topic } from './types-CTwLrJVU.js';
+import { T as TopicMapConstraint, C as ClientId, G as GroupId, K as KafkaInstrumentation, a as KafkaClientOptions, S as SchemaLike, b as ConsumerOptions, c as TopicDescriptor, I as IKafkaClient, d as KafkaHealthResult } from './types-CqjRm-Cd.js';
+export { B as BatchMessageItem, e as BatchMeta, f as BeforeConsumeResult, g as ConsumerHandle, h as ConsumerInterceptor, D as DeduplicationOptions, i as DlqReason, E as EnvelopeHeaderOptions, j as EventEnvelope, H as HEADER_CORRELATION_ID, k as HEADER_EVENT_ID, l as HEADER_LAMPORT_CLOCK, m as HEADER_SCHEMA_VERSION, n as HEADER_TIMESTAMP, o as HEADER_TRACEPARENT, p as InferSchema, q as KafkaLogger, r as KafkaMetrics, M as MessageHeaders, s as MessageLostContext, R as RetryOptions, t as SchemaParseContext, u as SendOptions, v as SubscribeRetryOptions, w as TTopicMessageMap, x as TopicsFrom, y as TransactionContext, z as buildEnvelopeHeaders, A as decodeHeaders, F as extractEnvelope, J as getEnvelopeContext, L as runWithEnvelopeContext, N as topic } from './types-CqjRm-Cd.js';
 import { DynamicModule, OnModuleInit } from '@nestjs/common';
 import { DiscoveryService, ModuleRef } from '@nestjs/core';
 

package/dist/index.js

@@ -376,7 +376,10 @@ async function sendToDlq(topic2, rawMessage, deps, meta) {
     deps.logger.warn(`Message sent to DLQ: ${payload.topic}`);
   } catch (error) {
     const err = toError(error);
-    deps.logger.error(`Failed to send message to DLQ ${payload.topic}:`, err.stack);
+    deps.logger.error(
+      `Failed to send message to DLQ ${payload.topic}:`,
+      err.stack
+    );
     await deps.onMessageLost?.({
       topic: topic2,
       error: err,
@@ -391,14 +394,9 @@ var RETRY_HEADER_MAX_RETRIES = "x-retry-max-retries";
 var RETRY_HEADER_ORIGINAL_TOPIC = "x-retry-original-topic";
 function buildRetryTopicPayload(originalTopic, rawMessages, attempt, maxRetries, delayMs, originalHeaders) {
   const retryTopic = `${originalTopic}.retry.${attempt}`;
+  const STRIP = /* @__PURE__ */ new Set([RETRY_HEADER_ATTEMPT, RETRY_HEADER_AFTER, RETRY_HEADER_MAX_RETRIES, RETRY_HEADER_ORIGINAL_TOPIC]);
   function buildHeaders(hdr) {
-    const {
-      [RETRY_HEADER_ATTEMPT]: _a,
-      [RETRY_HEADER_AFTER]: _b,
-      [RETRY_HEADER_MAX_RETRIES]: _c,
-      [RETRY_HEADER_ORIGINAL_TOPIC]: _d,
-      ...userHeaders
-    } = hdr;
+    const userHeaders = Object.fromEntries(Object.entries(hdr).filter(([k]) => !STRIP.has(k)));
     return {
       ...userHeaders,
       [RETRY_HEADER_ATTEMPT]: String(attempt),
@@ -454,10 +452,18 @@ function buildDuplicateTopicPayload(sourceTopic, rawMessage, destinationTopic, m
     "x-duplicate-incoming-clock": String(meta?.incomingClock ?? 0),
     "x-duplicate-last-processed-clock": String(meta?.lastProcessedClock ?? 0)
   };
-  return { topic: destinationTopic, messages: [{ value: rawMessage, headers }] };
+  return {
+    topic: destinationTopic,
+    messages: [{ value: rawMessage, headers }]
+  };
 }
 async function sendToDuplicatesTopic(sourceTopic, rawMessage, destinationTopic, deps, meta) {
-  const payload = buildDuplicateTopicPayload(sourceTopic, rawMessage, destinationTopic, meta);
+  const payload = buildDuplicateTopicPayload(
+    sourceTopic,
+    rawMessage,
+    destinationTopic,
+    meta
+  );
   try {
     await deps.producer.send(payload);
     deps.logger.warn(`Duplicate message forwarded to ${destinationTopic}`);
@@ -549,7 +555,10 @@ async function executeWithRetry(fn, ctx, deps) {
       interceptors,
       deps.instrumentation
     );
-    if (!error) return;
+    if (!error) {
+      for (const env of envelopes) deps.onMessage?.(env);
+      return;
+    }
     const isLastAttempt = attempt === maxAttempts;
     const reportedError = isLastAttempt && maxAttempts > 1 ? new KafkaRetryExhaustedError(
       topic2,
@@ -574,6 +583,7 @@ async function executeWithRetry(fn, ctx, deps) {
         isBatch ? envelopes.map((e) => e.headers) : envelopes[0]?.headers ?? {},
         deps
       );
+      deps.onRetry?.(envelopes[0], 1, retry.maxRetries);
     } else if (isLastAttempt) {
       if (dlq) {
         for (let i = 0; i < rawMessages.length; i++) {
@@ -582,6 +592,7 @@ async function executeWithRetry(fn, ctx, deps) {
             attempt,
             originalHeaders: envelopes[i]?.headers
           });
+          deps.onDlq?.(envelopes[i] ?? envelopes[0], "handler-error");
         }
       } else {
         await deps.onMessageLost?.({
@@ -593,6 +604,7 @@ async function executeWithRetry(fn, ctx, deps) {
       }
     } else {
       const cap = Math.min(backoffMs * 2 ** (attempt - 1), maxBackoffMs);
+      deps.onRetry?.(envelopes[0], attempt, maxAttempts - 1);
       await sleep(Math.floor(Math.random() * cap));
     }
   }
@@ -616,6 +628,7 @@ async function applyDeduplication(envelope, raw, dedup, dlq, deps) {
     deps.logger.warn(
       `Duplicate message on ${envelope.topic}[${envelope.partition}]: clock=${incomingClock} <= last=${lastProcessedClock} \u2014 strategy=${strategy}`
     );
+    deps.onDuplicate?.(envelope, strategy);
     if (strategy === "dlq" && dlq) {
       const augmentedHeaders = {
         ...envelope.headers,
@@ -819,6 +832,9 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
     producer,
     instrumentation,
     onMessageLost,
+    onRetry,
+    onDlq,
+    onMessage,
     ensureTopic,
     getOrCreateConsumer: getOrCreateConsumer2,
     runningConsumers,
@@ -900,6 +916,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
         instrumentation
       );
       if (!error) {
+        onMessage?.(envelope);
         await consumer.commitOffsets([nextOffset]);
         return;
       }
@@ -932,12 +949,23 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
           await tx.send({ topic: rtTopic, messages: rtMsgs });
           await tx.sendOffsets({
             consumer,
-            topics: [{ topic: nextOffset.topic, partitions: [{ partition: nextOffset.partition, offset: nextOffset.offset }] }]
+            topics: [
+              {
+                topic: nextOffset.topic,
+                partitions: [
+                  {
+                    partition: nextOffset.partition,
+                    offset: nextOffset.offset
+                  }
+                ]
+              }
+            ]
           });
           await tx.commit();
           logger.warn(
             `Message routed to ${rtTopic} (EOS, level ${nextLevel}/${currentMaxRetries})`
           );
+          onRetry?.(envelope, nextLevel, currentMaxRetries);
         } catch (txErr) {
           try {
             await tx.abort();
@@ -965,10 +993,21 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
           await tx.send({ topic: dTopic, messages: dMsgs });
           await tx.sendOffsets({
             consumer,
-            topics: [{ topic: nextOffset.topic, partitions: [{ partition: nextOffset.partition, offset: nextOffset.offset }] }]
+            topics: [
+              {
+                topic: nextOffset.topic,
+                partitions: [
+                  {
+                    partition: nextOffset.partition,
+                    offset: nextOffset.offset
+                  }
+                ]
+              }
+            ]
           });
           await tx.commit();
           logger.warn(`Message sent to DLQ: ${dTopic} (EOS)`);
+          onDlq?.(envelope, "handler-error");
         } catch (txErr) {
           try {
             await tx.abort();
@@ -992,7 +1031,12 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
     }
   });
   runningConsumers.set(levelGroupId, "eachMessage");
-  await waitForPartitionAssignment(consumer, levelTopics, logger, assignmentTimeoutMs);
+  await waitForPartitionAssignment(
+    consumer,
+    levelTopics,
+    logger,
+    assignmentTimeoutMs
+  );
   logger.log(
     `Retry level ${level}/${retry.maxRetries} consumer started for: ${originalTopics.join(", ")} (group: ${levelGroupId})`
   );
@@ -1022,6 +1066,7 @@ async function startRetryTopicConsumers(originalTopics, originalGroupId, handleM
 
 // src/client/kafka.client/index.ts
 var { Kafka: KafkaClass, logLevel: KafkaLogLevel } = import_kafka_javascript.KafkaJS;
+var _activeTransactionalIds = /* @__PURE__ */ new Set();
 var KafkaClient = class {
   kafka;
   producer;
@@ -1047,6 +1092,15 @@ var KafkaClient = class {
   instrumentation;
   onMessageLost;
   onRebalance;
+  /** Transactional producer ID — configurable via `KafkaClientOptions.transactionalId`. */
+  txId;
+  /** Internal event counters exposed via `getMetrics()`. */
+  _metrics = {
+    processedCount: 0,
+    retryCount: 0,
+    dlqCount: 0,
+    dedupCount: 0
+  };
   /** Monotonically increasing Lamport clock stamped on every outgoing message. */
   _lamportClock = 0;
   /** Per-groupId deduplication state: `"topic:partition"` → last processed clock. */
@@ -1070,6 +1124,7 @@ var KafkaClient = class {
     this.instrumentation = options?.instrumentation ?? [];
     this.onMessageLost = options?.onMessageLost;
     this.onRebalance = options?.onRebalance;
+    this.txId = options?.transactionalId ?? `${clientId}-tx`;
     this.kafka = new KafkaClass({
       kafkaJS: {
         clientId: this.clientId,
@@ -1106,16 +1161,22 @@ var KafkaClient = class {
   /** Execute multiple sends atomically. Commits on success, aborts on error. */
   async transaction(fn) {
     if (!this.txProducerInitPromise) {
+      if (_activeTransactionalIds.has(this.txId)) {
+        this.logger.warn(
+          `transactionalId "${this.txId}" is already in use by another KafkaClient in this process. Kafka will fence one of the producers. Set a unique \`transactionalId\` (or distinct \`clientId\`) per instance.`
+        );
+      }
       const initPromise = (async () => {
         const p = this.kafka.producer({
           kafkaJS: {
             acks: -1,
             idempotent: true,
-            transactionalId: `${this.clientId}-tx`,
+            transactionalId: this.txId,
             maxInFlightRequests: 1
           }
         });
         await p.connect();
+        _activeTransactionalIds.add(this.txId);
         return p;
       })();
       this.txProducerInitPromise = initPromise.catch((err) => {
@@ -1186,7 +1247,10 @@ var KafkaClient = class {
     const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", options);
     const deps = this.messageDeps;
     const timeoutMs = options.handlerTimeoutMs;
-    const deduplication = this.resolveDeduplicationContext(gid, options.deduplication);
+    const deduplication = this.resolveDeduplicationContext(
+      gid,
+      options.deduplication
+    );
     await consumer.run({
       eachMessage: (payload) => this.trackInFlight(
         () => handleEachMessage(
@@ -1240,7 +1304,10 @@ var KafkaClient = class {
     const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", options);
     const deps = this.messageDeps;
     const timeoutMs = options.handlerTimeoutMs;
-    const deduplication = this.resolveDeduplicationContext(gid, options.deduplication);
+    const deduplication = this.resolveDeduplicationContext(
+      gid,
+      options.deduplication
+    );
     await consumer.run({
       eachBatch: (payload) => this.trackInFlight(
         () => handleEachBatch(
@@ -1267,7 +1334,7 @@ var KafkaClient = class {
       }
       const handleMessageForRetry = (env) => handleBatch([env], {
         partition: env.partition,
-        highWatermark: env.offset,
+        highWatermark: null,
         heartbeat: async () => {
         },
         resolveOffset: () => {
@@ -1335,6 +1402,7 @@ var KafkaClient = class {
               toError(e).message
             )
           );
+          _activeTransactionalIds.delete(txId);
           this.retryTxProducers.delete(txId);
         }
       }
@@ -1410,15 +1478,28 @@ var KafkaClient = class {
   getClientId() {
     return this.clientId;
   }
+  getMetrics() {
+    return { ...this._metrics };
+  }
+  resetMetrics() {
+    this._metrics.processedCount = 0;
+    this._metrics.retryCount = 0;
+    this._metrics.dlqCount = 0;
+    this._metrics.dedupCount = 0;
+  }
   /** Gracefully disconnect producer, all consumers, and admin. */
   async disconnect(drainTimeoutMs = 3e4) {
     await this.waitForDrain(drainTimeoutMs);
     const tasks = [this.producer.disconnect()];
     if (this.txProducer) {
       tasks.push(this.txProducer.disconnect());
+      _activeTransactionalIds.delete(this.txId);
       this.txProducer = void 0;
       this.txProducerInitPromise = void 0;
     }
+    for (const txId of this.retryTxProducers.keys()) {
+      _activeTransactionalIds.delete(txId);
+    }
     for (const p of this.retryTxProducers.values()) {
       tasks.push(p.disconnect());
     }
@@ -1514,6 +1595,30 @@ var KafkaClient = class {
       }
     }
   }
+  notifyRetry(envelope, attempt, maxRetries) {
+    this._metrics.retryCount++;
+    for (const inst of this.instrumentation) {
+      inst.onRetry?.(envelope, attempt, maxRetries);
+    }
+  }
+  notifyDlq(envelope, reason) {
+    this._metrics.dlqCount++;
+    for (const inst of this.instrumentation) {
+      inst.onDlq?.(envelope, reason);
+    }
+  }
+  notifyDuplicate(envelope, strategy) {
+    this._metrics.dedupCount++;
+    for (const inst of this.instrumentation) {
+      inst.onDuplicate?.(envelope, strategy);
+    }
+  }
+  notifyMessage(envelope) {
+    this._metrics.processedCount++;
+    for (const inst of this.instrumentation) {
+      inst.onMessage?.(envelope);
+    }
+  }
   /**
    * Start a timer that logs a warning if `fn` hasn't resolved within `timeoutMs`.
    * The handler itself is not cancelled — the warning is diagnostic only.
@@ -1603,6 +1708,11 @@ var KafkaClient = class {
    * so Kafka can fence stale producers on restart without affecting other levels.
    */
   async createRetryTxProducer(transactionalId) {
+    if (_activeTransactionalIds.has(transactionalId)) {
+      this.logger.warn(
+        `transactionalId "${transactionalId}" is already in use by another KafkaClient in this process. Kafka will fence one of the producers. Set a unique \`transactionalId\` (or distinct \`clientId\`) per instance.`
+      );
+    }
     const p = this.kafka.producer({
       kafkaJS: {
         acks: -1,
@@ -1612,6 +1722,7 @@ var KafkaClient = class {
       }
     });
     await p.connect();
+    _activeTransactionalIds.add(transactionalId);
     this.retryTxProducers.set(transactionalId, p);
     return p;
   }
@@ -1732,7 +1843,11 @@ var KafkaClient = class {
       logger: this.logger,
       producer: this.producer,
       instrumentation: this.instrumentation,
-      onMessageLost: this.onMessageLost
+      onMessageLost: this.onMessageLost,
+      onRetry: this.notifyRetry.bind(this),
+      onDlq: this.notifyDlq.bind(this),
+      onDuplicate: this.notifyDuplicate.bind(this),
+      onMessage: this.notifyMessage.bind(this)
     };
   }
   get retryTopicDeps() {
@@ -1741,6 +1856,9 @@ var KafkaClient = class {
       producer: this.producer,
       instrumentation: this.instrumentation,
       onMessageLost: this.onMessageLost,
+      onRetry: this.notifyRetry.bind(this),
+      onDlq: this.notifyDlq.bind(this),
+      onMessage: this.notifyMessage.bind(this),
       ensureTopic: (t) => this.ensureTopic(t),
       getOrCreateConsumer: (gid, fb, ac) => getOrCreateConsumer(gid, fb, ac, this.consumerOpsDeps),
       runningConsumers: this.runningConsumers,