@drarzter/kafka-client 0.6.3 → 0.6.6

package/dist/core.js

@@ -22,6 +22,7 @@ var core_exports = {};
 __export(core_exports, {
   HEADER_CORRELATION_ID: () => HEADER_CORRELATION_ID,
   HEADER_EVENT_ID: () => HEADER_EVENT_ID,
+  HEADER_LAMPORT_CLOCK: () => HEADER_LAMPORT_CLOCK,
   HEADER_SCHEMA_VERSION: () => HEADER_SCHEMA_VERSION,
   HEADER_TIMESTAMP: () => HEADER_TIMESTAMP,
   HEADER_TRACEPARENT: () => HEADER_TRACEPARENT,
@@ -49,6 +50,7 @@ var HEADER_CORRELATION_ID = "x-correlation-id";
 var HEADER_TIMESTAMP = "x-timestamp";
 var HEADER_SCHEMA_VERSION = "x-schema-version";
 var HEADER_TRACEPARENT = "traceparent";
+var HEADER_LAMPORT_CLOCK = "x-lamport-clock";
 var envelopeStorage = new import_node_async_hooks.AsyncLocalStorage();
 function getEnvelopeContext() {
   return envelopeStorage.getStore();
@@ -189,6 +191,9 @@ async function buildSendPayload(topicOrDesc, messages, deps) {
         eventId: m.eventId,
         headers: m.headers
       });
+      if (deps.nextLamportClock) {
+        envelopeHeaders[HEADER_LAMPORT_CLOCK] = String(deps.nextLamportClock());
+      }
       for (const inst of deps.instrumentation) {
         inst.beforeSend?.(topic2, envelopeHeaders);
       }
@@ -326,6 +331,9 @@ async function validateWithSchema(message, raw, topic2, schemaMap, interceptors,
       -1,
       ""
     );
+    for (const inst of deps.instrumentation ?? []) {
+      inst.onConsumeError?.(errorEnvelope, validationError);
+    }
     for (const interceptor of interceptors) {
       await interceptor.onError?.(errorEnvelope, validationError);
     }
@@ -420,6 +428,29 @@ async function sendToRetryTopic(originalTopic, rawMessages, attempt, maxRetries,
     });
   }
 }
+function buildDuplicateTopicPayload(sourceTopic, rawMessage, destinationTopic, meta) {
+  const headers = {
+    ...meta?.originalHeaders ?? {},
+    "x-duplicate-original-topic": sourceTopic,
+    "x-duplicate-detected-at": (/* @__PURE__ */ new Date()).toISOString(),
+    "x-duplicate-reason": "lamport-clock-duplicate",
+    "x-duplicate-incoming-clock": String(meta?.incomingClock ?? 0),
+    "x-duplicate-last-processed-clock": String(meta?.lastProcessedClock ?? 0)
+  };
+  return { topic: destinationTopic, messages: [{ value: rawMessage, headers }] };
+}
+async function sendToDuplicatesTopic(sourceTopic, rawMessage, destinationTopic, deps, meta) {
+  const payload = buildDuplicateTopicPayload(sourceTopic, rawMessage, destinationTopic, meta);
+  try {
+    await deps.producer.send(payload);
+    deps.logger.warn(`Duplicate message forwarded to ${destinationTopic}`);
+  } catch (error) {
+    deps.logger.error(
+      `Failed to forward duplicate to ${destinationTopic}:`,
+      toError(error).stack
+    );
+  }
+}
 async function broadcastToInterceptors(envelopes, interceptors, cb) {
   for (const env of envelopes) {
     for (const interceptor of interceptors) {
@@ -528,13 +559,12 @@ async function executeWithRetry(fn, ctx, deps) {
       );
     } else if (isLastAttempt) {
       if (dlq) {
-        const dlqMeta = {
-          error,
-          attempt,
-          originalHeaders: envelopes[0]?.headers
-        };
-        for (const raw of rawMessages) {
-          await sendToDlq(topic2, raw, deps, dlqMeta);
+        for (let i = 0; i < rawMessages.length; i++) {
+          await sendToDlq(topic2, rawMessages[i], deps, {
+            error,
+            attempt,
+            originalHeaders: envelopes[i]?.headers
+          });
         }
       } else {
         await deps.onMessageLost?.({
@@ -546,12 +576,50 @@ async function executeWithRetry(fn, ctx, deps) {
       }
     } else {
       const cap = Math.min(backoffMs * 2 ** (attempt - 1), maxBackoffMs);
-      await sleep(Math.random() * cap);
+      await sleep(Math.floor(Math.random() * cap));
     }
   }
 }
 
 // src/client/kafka.client/message-handler.ts
+async function applyDeduplication(envelope, raw, dedup, dlq, deps) {
+  const clockRaw = envelope.headers[HEADER_LAMPORT_CLOCK];
+  if (clockRaw === void 0) return false;
+  const incomingClock = Number(clockRaw);
+  if (Number.isNaN(incomingClock)) return false;
+  const stateKey = `${envelope.topic}:${envelope.partition}`;
+  const lastProcessedClock = dedup.state.get(stateKey) ?? -1;
+  if (incomingClock <= lastProcessedClock) {
+    const meta = {
+      incomingClock,
+      lastProcessedClock,
+      originalHeaders: envelope.headers
+    };
+    const strategy = dedup.options.strategy ?? "drop";
+    deps.logger.warn(
+      `Duplicate message on ${envelope.topic}[${envelope.partition}]: clock=${incomingClock} <= last=${lastProcessedClock} \u2014 strategy=${strategy}`
+    );
+    if (strategy === "dlq" && dlq) {
+      const augmentedHeaders = {
+        ...envelope.headers,
+        "x-dlq-reason": "lamport-clock-duplicate",
+        "x-dlq-duplicate-incoming-clock": String(incomingClock),
+        "x-dlq-duplicate-last-processed-clock": String(lastProcessedClock)
+      };
+      await sendToDlq(envelope.topic, raw, deps, {
+        error: new Error("Lamport Clock duplicate detected"),
+        attempt: 0,
+        originalHeaders: augmentedHeaders
+      });
+    } else if (strategy === "topic") {
+      const destination = dedup.options.duplicatesTopic ?? `${envelope.topic}.duplicates`;
+      await sendToDuplicatesTopic(envelope.topic, raw, destination, deps, meta);
+    }
+    return true;
+  }
+  dedup.state.set(stateKey, incomingClock);
+  return false;
+}
 async function parseSingleMessage(message, topic2, partition, schemaMap, interceptors, dlq, deps) {
   if (!message.value) {
     deps.logger.warn(`Received empty message from topic ${topic2}`);
@@ -595,6 +663,16 @@ async function handleEachMessage(payload, opts, deps) {
     deps
   );
   if (envelope === null) return;
+  if (opts.deduplication) {
+    const isDuplicate = await applyDeduplication(
+      envelope,
+      message.value.toString(),
+      opts.deduplication,
+      dlq,
+      deps
+    );
+    if (isDuplicate) return;
+  }
   await executeWithRetry(
     () => {
       const fn = () => runWithEnvelopeContext(
@@ -642,6 +720,17 @@ async function handleEachBatch(payload, opts, deps) {
       deps
     );
     if (envelope === null) continue;
+    if (opts.deduplication) {
+      const raw = message.value.toString();
+      const isDuplicate = await applyDeduplication(
+        envelope,
+        raw,
+        opts.deduplication,
+        dlq,
+        deps
+      );
+      if (isDuplicate) continue;
+    }
     envelopes.push(envelope);
     rawMessages.push(message.value.toString());
   }
@@ -920,7 +1009,9 @@ var KafkaClient = class {
   kafka;
   producer;
   txProducer;
-  retryTxProducers = /* @__PURE__ */ new Set();
+  txProducerInitPromise;
+  /** Maps transactionalId → Producer for each active retry level consumer. */
+  retryTxProducers = /* @__PURE__ */ new Map();
   consumers = /* @__PURE__ */ new Map();
   admin;
   logger;
@@ -928,6 +1019,8 @@ var KafkaClient = class {
   strictSchemasEnabled;
   numPartitions;
   ensuredTopics = /* @__PURE__ */ new Set();
+  /** Pending topic-creation promises keyed by topic name. Prevents duplicate createTopics calls. */
+  ensureTopicPromises = /* @__PURE__ */ new Map();
   defaultGroupId;
   schemaRegistry = /* @__PURE__ */ new Map();
   runningConsumers = /* @__PURE__ */ new Map();
@@ -937,6 +1030,10 @@ var KafkaClient = class {
   instrumentation;
   onMessageLost;
   onRebalance;
+  /** Monotonically increasing Lamport clock stamped on every outgoing message. */
+  _lamportClock = 0;
+  /** Per-groupId deduplication state: `"topic:partition"` → last processed clock. */
+  dedupStates = /* @__PURE__ */ new Map();
   isAdminConnected = false;
   inFlightTotal = 0;
   drainResolvers = [];
@@ -991,18 +1088,25 @@ var KafkaClient = class {
   }
   /** Execute multiple sends atomically. Commits on success, aborts on error. */
   async transaction(fn) {
-    if (!this.txProducer) {
-      const p = this.kafka.producer({
-        kafkaJS: {
-          acks: -1,
-          idempotent: true,
-          transactionalId: `${this.clientId}-tx`,
-          maxInFlightRequests: 1
-        }
+    if (!this.txProducerInitPromise) {
+      const initPromise = (async () => {
+        const p = this.kafka.producer({
+          kafkaJS: {
+            acks: -1,
+            idempotent: true,
+            transactionalId: `${this.clientId}-tx`,
+            maxInFlightRequests: 1
+          }
+        });
+        await p.connect();
+        return p;
+      })();
+      this.txProducerInitPromise = initPromise.catch((err) => {
+        this.txProducerInitPromise = void 0;
+        throw err;
       });
-      await p.connect();
-      this.txProducer = p;
     }
+    this.txProducer = await this.txProducerInitPromise;
     const tx = await this.txProducer.transaction();
     try {
       const ctx = {
@@ -1041,11 +1145,17 @@ var KafkaClient = class {
     }
   }
   // ── Producer lifecycle ───────────────────────────────────────────
-  /** Connect the idempotent producer. Called automatically by `KafkaModule.register()`. */
+  /**
+   * Connect the idempotent producer. Called automatically by `KafkaModule.register()`.
+   * @internal Not part of `IKafkaClient` — use `disconnect()` for full teardown.
+   */
   async connectProducer() {
     await this.producer.connect();
     this.logger.log("Producer connected");
   }
+  /**
+   * @internal Not part of `IKafkaClient` — use `disconnect()` for full teardown.
+   */
   async disconnectProducer() {
     await this.producer.disconnect();
     this.logger.log("Producer disconnected");
@@ -1059,6 +1169,7 @@ var KafkaClient = class {
     const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", options);
     const deps = this.messageDeps;
     const timeoutMs = options.handlerTimeoutMs;
+    const deduplication = this.resolveDeduplicationContext(gid, options.deduplication);
     await consumer.run({
       eachMessage: (payload) => this.trackInFlight(
         () => handleEachMessage(
@@ -1071,7 +1182,8 @@ var KafkaClient = class {
             retry,
             retryTopics: options.retryTopics,
             timeoutMs,
-            wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this)
+            wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this),
+            deduplication
           },
           deps
         )
@@ -1111,6 +1223,7 @@ var KafkaClient = class {
     const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", options);
     const deps = this.messageDeps;
     const timeoutMs = options.handlerTimeoutMs;
+    const deduplication = this.resolveDeduplicationContext(gid, options.deduplication);
     await consumer.run({
       eachBatch: (payload) => this.trackInFlight(
         () => handleEachBatch(
@@ -1123,7 +1236,8 @@ var KafkaClient = class {
             retry,
             retryTopics: options.retryTopics,
             timeoutMs,
-            wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this)
+            wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this),
+            deduplication
           },
           deps
         )
@@ -1169,35 +1283,63 @@ var KafkaClient = class {
         );
         return;
       }
-      await consumer.disconnect().catch(() => {
-      });
+      await consumer.disconnect().catch(
+        (e) => this.logger.warn(
+          `Error disconnecting consumer "${groupId}":`,
+          toError(e).message
+        )
+      );
       this.consumers.delete(groupId);
       this.runningConsumers.delete(groupId);
       this.consumerCreationOptions.delete(groupId);
+      this.dedupStates.delete(groupId);
       this.logger.log(`Consumer disconnected: group "${groupId}"`);
       const companions = this.companionGroupIds.get(groupId) ?? [];
       for (const cGroupId of companions) {
         const cConsumer = this.consumers.get(cGroupId);
         if (cConsumer) {
-          await cConsumer.disconnect().catch(() => {
-          });
+          await cConsumer.disconnect().catch(
+            (e) => this.logger.warn(
+              `Error disconnecting retry consumer "${cGroupId}":`,
+              toError(e).message
+            )
+          );
           this.consumers.delete(cGroupId);
           this.runningConsumers.delete(cGroupId);
           this.consumerCreationOptions.delete(cGroupId);
           this.logger.log(`Retry consumer disconnected: group "${cGroupId}"`);
         }
+        const txId = `${cGroupId}-tx`;
+        const txProducer = this.retryTxProducers.get(txId);
+        if (txProducer) {
+          await txProducer.disconnect().catch(
+            (e) => this.logger.warn(
+              `Error disconnecting retry tx producer "${txId}":`,
+              toError(e).message
+            )
+          );
+          this.retryTxProducers.delete(txId);
+        }
       }
       this.companionGroupIds.delete(groupId);
     } else {
-      const tasks = Array.from(this.consumers.values()).map(
-        (c) => c.disconnect().catch(() => {
-        })
-      );
+      const tasks = [
+        ...Array.from(this.consumers.values()).map(
+          (c) => c.disconnect().catch(() => {
+          })
+        ),
+        ...Array.from(this.retryTxProducers.values()).map(
+          (p) => p.disconnect().catch(() => {
+          })
+        )
+      ];
       await Promise.allSettled(tasks);
       this.consumers.clear();
       this.runningConsumers.clear();
       this.consumerCreationOptions.clear();
       this.companionGroupIds.clear();
+      this.retryTxProducers.clear();
+      this.dedupStates.clear();
       this.logger.log("All consumers disconnected");
     }
   }
@@ -1205,6 +1347,12 @@ var KafkaClient = class {
    * Query consumer group lag per partition.
    * Lag = broker high-watermark − last committed offset.
    * A committed offset of -1 (nothing committed yet) counts as full lag.
+   *
+   * Returns an empty array when the consumer group has never committed any
+   * offsets (freshly created group, `autoCommit: false` with no manual commits,
+   * or group not yet assigned). This is a Kafka protocol limitation:
+   * `fetchOffsets` only returns data for topic-partitions that have at least one
+   * committed offset. Use `checkStatus()` to verify broker connectivity in that case.
    */
   async getConsumerLag(groupId) {
     const gid = groupId ?? this.defaultGroupId;
@@ -1252,8 +1400,9 @@ var KafkaClient = class {
     if (this.txProducer) {
       tasks.push(this.txProducer.disconnect());
       this.txProducer = void 0;
+      this.txProducerInitPromise = void 0;
     }
-    for (const p of this.retryTxProducers) {
+    for (const p of this.retryTxProducers.values()) {
       tasks.push(p.disconnect());
     }
     this.retryTxProducers.clear();
@@ -1272,6 +1421,14 @@ var KafkaClient = class {
     this.logger.log("All connections closed");
   }
   // ── Graceful shutdown ────────────────────────────────────────────
+  /**
+   * NestJS lifecycle hook — called automatically when the host module is torn down.
+   * Drains in-flight handlers and disconnects all producers, consumers, and admin.
+   * `KafkaModule` relies on this method; no separate destroy provider is needed.
+   */
+  async onModuleDestroy() {
+    await this.disconnect();
+  }
   /**
    * Register SIGTERM / SIGINT handlers that drain in-flight messages before
    * disconnecting. Call this once after constructing the client in non-NestJS apps.
@@ -1392,6 +1549,22 @@ var KafkaClient = class {
       );
     }
   }
+  /**
+   * When `deduplication.strategy: 'topic'` and `autoCreateTopics: false`, verify
+   * that every `<topic>.duplicates` destination topic already exists. Throws a
+   * clear error at startup rather than silently dropping duplicates on first hit.
+   */
+  async validateDuplicatesTopicsExist(topicNames, customDestination) {
+    await this.ensureAdminConnected();
+    const existing = new Set(await this.admin.listTopics());
+    const toCheck = customDestination ? [customDestination] : topicNames.map((t) => `${t}.duplicates`);
+    const missing = toCheck.filter((t) => !existing.has(t));
+    if (missing.length > 0) {
+      throw new Error(
+        `deduplication.strategy: 'topic' but the following duplicate-routing topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
+      );
+    }
+  }
   /**
    * Connect the admin client if not already connected.
    * The flag is only set to `true` after a successful connect — if `admin.connect()`
@@ -1422,16 +1595,23 @@ var KafkaClient = class {
       }
     });
     await p.connect();
-    this.retryTxProducers.add(p);
+    this.retryTxProducers.set(transactionalId, p);
     return p;
   }
   async ensureTopic(topic2) {
     if (!this.autoCreateTopicsEnabled || this.ensuredTopics.has(topic2)) return;
-    await this.ensureAdminConnected();
-    await this.admin.createTopics({
-      topics: [{ topic: topic2, numPartitions: this.numPartitions }]
-    });
-    this.ensuredTopics.add(topic2);
+    let p = this.ensureTopicPromises.get(topic2);
+    if (!p) {
+      p = (async () => {
+        await this.ensureAdminConnected();
+        await this.admin.createTopics({
+          topics: [{ topic: topic2, numPartitions: this.numPartitions }]
+        });
+        this.ensuredTopics.add(topic2);
+      })().finally(() => this.ensureTopicPromises.delete(topic2));
+      this.ensureTopicPromises.set(topic2, p);
+    }
+    await p;
   }
   /** Shared consumer setup: groupId check, schema map, connect, subscribe. */
   async setupConsumer(topics, mode, options) {
@@ -1451,6 +1631,12 @@ var KafkaClient = class {
         `Cannot use ${mode} on consumer group "${gid}" \u2014 it is already running with ${oppositeMode}. Use a different groupId for this consumer.`
       );
     }
+    if (existingMode === mode) {
+      const callerName = mode === "eachMessage" ? "startConsumer" : "startBatchConsumer";
+      throw new Error(
+        `${callerName}("${gid}") called twice \u2014 this group is already consuming. Call stopConsumer("${gid}") first or pass a different groupId.`
+      );
+    }
     const consumer = getOrCreateConsumer(
       gid,
       fromBeginning,
@@ -1475,6 +1661,16 @@ var KafkaClient = class {
         await this.validateDlqTopicsExist(topicNames);
       }
     }
+    if (options.deduplication?.strategy === "topic") {
+      const dest = options.deduplication.duplicatesTopic;
+      if (this.autoCreateTopicsEnabled) {
+        for (const t of topicNames) {
+          await this.ensureTopic(dest ?? `${t}.duplicates`);
+        }
+      } else {
+        await this.validateDuplicatesTopicsExist(topicNames, dest);
+      }
+    }
     await consumer.connect();
     await subscribeWithRetry(
       consumer,
@@ -1487,13 +1683,22 @@ var KafkaClient = class {
     );
     return { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry };
   }
+  /** Create or retrieve the deduplication context for a consumer group. */
+  resolveDeduplicationContext(groupId, options) {
+    if (!options) return void 0;
+    if (!this.dedupStates.has(groupId)) {
+      this.dedupStates.set(groupId, /* @__PURE__ */ new Map());
+    }
+    return { options, state: this.dedupStates.get(groupId) };
+  }
   // ── Deps object getters ──────────────────────────────────────────
   get producerOpsDeps() {
     return {
       schemaRegistry: this.schemaRegistry,
       strictSchemasEnabled: this.strictSchemasEnabled,
       instrumentation: this.instrumentation,
-      logger: this.logger
+      logger: this.logger,
+      nextLamportClock: () => ++this._lamportClock
     };
   }
   get consumerOpsDeps() {
@@ -1546,6 +1751,7 @@ function topic(name) {
 0 && (module.exports = {
   HEADER_CORRELATION_ID,
   HEADER_EVENT_ID,
+  HEADER_LAMPORT_CLOCK,
   HEADER_SCHEMA_VERSION,
   HEADER_TIMESTAMP,
   HEADER_TRACEPARENT,