@drarzter/kafka-client 0.6.4 → 0.6.6

package/README.md

@@ -30,6 +30,7 @@ Type-safe Kafka client for Node.js. Framework-agnostic core with a first-class N
 - [Instrumentation](#instrumentation)
 - [Options reference](#options-reference)
 - [Error classes](#error-classes)
+- [Deduplication (Lamport Clock)](#deduplication-lamport-clock)
 - [Retry topic chain](#retry-topic-chain)
 - [stopConsumer](#stopconsumer)
 - [Graceful shutdown](#graceful-shutdown)
@@ -65,6 +66,7 @@ Safe by default. Configurable when you need it. Escape hatches for when you know
 - **Topic descriptors** — `topic()` DX sugar lets you define topics as standalone typed objects instead of string keys
 - **Framework-agnostic** — use standalone or with NestJS (`register()` / `registerAsync()`, DI, lifecycle hooks)
 - **Idempotent producer** — `acks: -1`, `idempotent: true` by default
+- **Lamport Clock deduplication** — every outgoing message is stamped with a monotonically increasing `x-lamport-clock` header; the consumer tracks the last processed value per `topic:partition` and silently drops (or routes to DLQ / a dedicated topic) any message whose clock is not strictly greater than the last seen value
 - **Retry + DLQ** — exponential backoff with full jitter; dead letter queue with error metadata headers (original topic, error message, stack, attempt count)
 - **Batch sending** — send multiple messages in a single request
 - **Batch consuming** — `startBatchConsumer()` for high-throughput `eachBatch` processing
@@ -772,6 +774,8 @@ Options for `sendMessage()` — the third argument:
 | `interceptors` | `[]` | Array of before/after/onError hooks |
 | `retryTopicAssignmentTimeoutMs` | `10000` | Timeout (ms) to wait for each retry level consumer to receive partition assignments after connecting; increase for slow brokers |
 | `handlerTimeoutMs` | — | Log a warning if the handler hasn't resolved within this window (ms) — does not cancel the handler |
+| `deduplication.strategy` | `'drop'` | What to do with duplicate messages: `'drop'` silently discards, `'dlq'` forwards to `{topic}.dlq` (requires `dlq: true`), `'topic'` forwards to `{topic}.duplicates` |
+| `deduplication.duplicatesTopic` | `{topic}.duplicates` | Custom destination for `strategy: 'topic'` |
 | `batch` | `false` | (decorator only) Use `startBatchConsumer` instead of `startConsumer` |
 | `subscribeRetry.retries` | `5` | Max attempts for `consumer.subscribe()` when topic doesn't exist yet |
 | `subscribeRetry.backoffMs` | `5000` | Delay between subscribe retry attempts (ms) |
@@ -880,6 +884,81 @@ const interceptor: ConsumerInterceptor<MyTopics> = {
 };
 ```
 
+## Deduplication (Lamport Clock)
+
+Every outgoing message produced by this library is stamped with a monotonically increasing logical clock — the `x-lamport-clock` header. The counter lives in the `KafkaClient` instance and increments by one per message (including individual messages inside `sendBatch` and `transaction`).
+
+On the consumer side, enable deduplication by passing `deduplication` to `startConsumer` or `startBatchConsumer`. The library checks the incoming clock against the last processed value for that `topic:partition` combination and skips any message whose clock is not strictly greater.
+
+```typescript
+await kafka.startConsumer(['orders.created'], handler, {
+  deduplication: {}, // 'drop' strategy — silently discard duplicates
+});
+```
+
+### How duplicates happen
+
+The most common scenario: a producer service restarts. Its in-memory clock resets to `0`. The consumer already processed messages with clocks `1…N`. All new messages from the restarted producer (clocks `1`, `2`, `3`, …) have clocks ≤ `N` and are treated as duplicates.
+
+```text
+Producer A (running): sends clock 1, 2, 3, 4, 5  → consumer processes all 5
+Producer A (restarts): sends clock 1, 2, 3         → consumer sees 1 ≤ 5 — duplicate!
+```
+
+### Strategies
+
+| Strategy | Behaviour |
+| -------- | --------- |
+| `'drop'` *(default)* | Log a warning and silently discard the message |
+| `'dlq'` | Forward to `{topic}.dlq` with reason metadata headers (`x-dlq-reason`, `x-dlq-duplicate-incoming-clock`, `x-dlq-duplicate-last-processed-clock`). Requires `dlq: true` |
+| `'topic'` | Forward to `{topic}.duplicates` (or `duplicatesTopic` if set) with reason metadata headers (`x-duplicate-reason`, `x-duplicate-incoming-clock`, `x-duplicate-last-processed-clock`, `x-duplicate-detected-at`) |
+
+```typescript
+// Strategy: drop (default)
+await kafka.startConsumer(['orders'], handler, {
+  deduplication: {},
+});
+
+// Strategy: DLQ — inspect duplicates from {topic}.dlq
+await kafka.startConsumer(['orders'], handler, {
+  dlq: true,
+  deduplication: { strategy: 'dlq' },
+});
+
+// Strategy: dedicated topic — consume from {topic}.duplicates
+await kafka.startConsumer(['orders'], handler, {
+  deduplication: { strategy: 'topic' },
+});
+
+// Strategy: custom topic name
+await kafka.startConsumer(['orders'], handler, {
+  deduplication: {
+    strategy: 'topic',
+    duplicatesTopic: 'ops.orders.duplicates',
+  },
+});
+```
+
+### Startup validation
+
+When `autoCreateTopics: false` and `strategy: 'topic'`, `startConsumer` / `startBatchConsumer` validates that the destination topic (`{topic}.duplicates` or `duplicatesTopic`) exists before starting the consumer. A clear error is thrown at startup listing every missing topic, rather than silently failing on the first duplicate.
+
+With `autoCreateTopics: true` the check is skipped — the topic is created automatically instead.
+
+### Backwards compatibility
+
+Messages without an `x-lamport-clock` header pass through unchanged. Producers not using this library are unaffected.
+
+### Limitations
+
+Deduplication state is **in-memory and per-consumer-instance**. Understand what that means:
+
+- **Consumer restart** — state is cleared on restart. The first batch of messages after restart is accepted regardless of their clock values, so duplicates spanning a restart window are not caught.
+- **Multiple consumer instances** (same group, different machines) — each instance tracks its own partition subset. Partitions are reassigned on rebalance, so a rebalance can reset the state for moved partitions.
+- **Cross-session duplicates** — this guards against duplicates from a **producer that restarted within the same consumer session**. For durable, cross-restart deduplication, persist the clock state externally (Redis, database) and implement idempotent handlers.
+
+Use this feature as a lightweight first line of defence — not as a substitute for idempotent business logic.
+
 ## Retry topic chain
 
 > **tl;dr — recommended production setup:**

package/dist/{chunk-7IX4AKQX.mjs → chunk-KCUKXR6B.mjs}

@@ -9,6 +9,7 @@ var HEADER_CORRELATION_ID = "x-correlation-id";
 var HEADER_TIMESTAMP = "x-timestamp";
 var HEADER_SCHEMA_VERSION = "x-schema-version";
 var HEADER_TRACEPARENT = "traceparent";
+var HEADER_LAMPORT_CLOCK = "x-lamport-clock";
 var envelopeStorage = new AsyncLocalStorage();
 function getEnvelopeContext() {
   return envelopeStorage.getStore();
@@ -149,6 +150,9 @@ async function buildSendPayload(topicOrDesc, messages, deps) {
         eventId: m.eventId,
         headers: m.headers
       });
+      if (deps.nextLamportClock) {
+        envelopeHeaders[HEADER_LAMPORT_CLOCK] = String(deps.nextLamportClock());
+      }
       for (const inst of deps.instrumentation) {
         inst.beforeSend?.(topic2, envelopeHeaders);
       }
@@ -286,6 +290,9 @@ async function validateWithSchema(message, raw, topic2, schemaMap, interceptors,
       -1,
       ""
     );
+    for (const inst of deps.instrumentation ?? []) {
+      inst.onConsumeError?.(errorEnvelope, validationError);
+    }
     for (const interceptor of interceptors) {
       await interceptor.onError?.(errorEnvelope, validationError);
     }
@@ -380,6 +387,29 @@ async function sendToRetryTopic(originalTopic, rawMessages, attempt, maxRetries,
     });
   }
 }
+function buildDuplicateTopicPayload(sourceTopic, rawMessage, destinationTopic, meta) {
+  const headers = {
+    ...meta?.originalHeaders ?? {},
+    "x-duplicate-original-topic": sourceTopic,
+    "x-duplicate-detected-at": (/* @__PURE__ */ new Date()).toISOString(),
+    "x-duplicate-reason": "lamport-clock-duplicate",
+    "x-duplicate-incoming-clock": String(meta?.incomingClock ?? 0),
+    "x-duplicate-last-processed-clock": String(meta?.lastProcessedClock ?? 0)
+  };
+  return { topic: destinationTopic, messages: [{ value: rawMessage, headers }] };
+}
+async function sendToDuplicatesTopic(sourceTopic, rawMessage, destinationTopic, deps, meta) {
+  const payload = buildDuplicateTopicPayload(sourceTopic, rawMessage, destinationTopic, meta);
+  try {
+    await deps.producer.send(payload);
+    deps.logger.warn(`Duplicate message forwarded to ${destinationTopic}`);
+  } catch (error) {
+    deps.logger.error(
+      `Failed to forward duplicate to ${destinationTopic}:`,
+      toError(error).stack
+    );
+  }
+}
 async function broadcastToInterceptors(envelopes, interceptors, cb) {
   for (const env of envelopes) {
     for (const interceptor of interceptors) {
@@ -488,13 +518,12 @@ async function executeWithRetry(fn, ctx, deps) {
       );
     } else if (isLastAttempt) {
       if (dlq) {
-        const dlqMeta = {
-          error,
-          attempt,
-          originalHeaders: envelopes[0]?.headers
-        };
-        for (const raw of rawMessages) {
-          await sendToDlq(topic2, raw, deps, dlqMeta);
+        for (let i = 0; i < rawMessages.length; i++) {
+          await sendToDlq(topic2, rawMessages[i], deps, {
+            error,
+            attempt,
+            originalHeaders: envelopes[i]?.headers
+          });
         }
       } else {
         await deps.onMessageLost?.({
@@ -512,6 +541,44 @@ async function executeWithRetry(fn, ctx, deps) {
 }
 
 // src/client/kafka.client/message-handler.ts
+async function applyDeduplication(envelope, raw, dedup, dlq, deps) {
+  const clockRaw = envelope.headers[HEADER_LAMPORT_CLOCK];
+  if (clockRaw === void 0) return false;
+  const incomingClock = Number(clockRaw);
+  if (Number.isNaN(incomingClock)) return false;
+  const stateKey = `${envelope.topic}:${envelope.partition}`;
+  const lastProcessedClock = dedup.state.get(stateKey) ?? -1;
+  if (incomingClock <= lastProcessedClock) {
+    const meta = {
+      incomingClock,
+      lastProcessedClock,
+      originalHeaders: envelope.headers
+    };
+    const strategy = dedup.options.strategy ?? "drop";
+    deps.logger.warn(
+      `Duplicate message on ${envelope.topic}[${envelope.partition}]: clock=${incomingClock} <= last=${lastProcessedClock} \u2014 strategy=${strategy}`
+    );
+    if (strategy === "dlq" && dlq) {
+      const augmentedHeaders = {
+        ...envelope.headers,
+        "x-dlq-reason": "lamport-clock-duplicate",
+        "x-dlq-duplicate-incoming-clock": String(incomingClock),
+        "x-dlq-duplicate-last-processed-clock": String(lastProcessedClock)
+      };
+      await sendToDlq(envelope.topic, raw, deps, {
+        error: new Error("Lamport Clock duplicate detected"),
+        attempt: 0,
+        originalHeaders: augmentedHeaders
+      });
+    } else if (strategy === "topic") {
+      const destination = dedup.options.duplicatesTopic ?? `${envelope.topic}.duplicates`;
+      await sendToDuplicatesTopic(envelope.topic, raw, destination, deps, meta);
+    }
+    return true;
+  }
+  dedup.state.set(stateKey, incomingClock);
+  return false;
+}
 async function parseSingleMessage(message, topic2, partition, schemaMap, interceptors, dlq, deps) {
   if (!message.value) {
     deps.logger.warn(`Received empty message from topic ${topic2}`);
@@ -555,6 +622,16 @@ async function handleEachMessage(payload, opts, deps) {
     deps
   );
   if (envelope === null) return;
+  if (opts.deduplication) {
+    const isDuplicate = await applyDeduplication(
+      envelope,
+      message.value.toString(),
+      opts.deduplication,
+      dlq,
+      deps
+    );
+    if (isDuplicate) return;
+  }
   await executeWithRetry(
     () => {
       const fn = () => runWithEnvelopeContext(
@@ -602,6 +679,17 @@ async function handleEachBatch(payload, opts, deps) {
       deps
     );
     if (envelope === null) continue;
+    if (opts.deduplication) {
+      const raw = message.value.toString();
+      const isDuplicate = await applyDeduplication(
+        envelope,
+        raw,
+        opts.deduplication,
+        dlq,
+        deps
+      );
+      if (isDuplicate) continue;
+    }
     envelopes.push(envelope);
     rawMessages.push(message.value.toString());
   }
@@ -880,6 +968,7 @@ var KafkaClient = class {
   kafka;
   producer;
   txProducer;
+  txProducerInitPromise;
   /** Maps transactionalId → Producer for each active retry level consumer. */
   retryTxProducers = /* @__PURE__ */ new Map();
   consumers = /* @__PURE__ */ new Map();
@@ -889,6 +978,8 @@ var KafkaClient = class {
   strictSchemasEnabled;
   numPartitions;
   ensuredTopics = /* @__PURE__ */ new Set();
+  /** Pending topic-creation promises keyed by topic name. Prevents duplicate createTopics calls. */
+  ensureTopicPromises = /* @__PURE__ */ new Map();
   defaultGroupId;
   schemaRegistry = /* @__PURE__ */ new Map();
   runningConsumers = /* @__PURE__ */ new Map();
@@ -898,6 +989,10 @@ var KafkaClient = class {
   instrumentation;
   onMessageLost;
   onRebalance;
+  /** Monotonically increasing Lamport clock stamped on every outgoing message. */
+  _lamportClock = 0;
+  /** Per-groupId deduplication state: `"topic:partition"` → last processed clock. */
+  dedupStates = /* @__PURE__ */ new Map();
   isAdminConnected = false;
   inFlightTotal = 0;
   drainResolvers = [];
@@ -952,18 +1047,25 @@ var KafkaClient = class {
   }
   /** Execute multiple sends atomically. Commits on success, aborts on error. */
   async transaction(fn) {
-    if (!this.txProducer) {
-      const p = this.kafka.producer({
-        kafkaJS: {
-          acks: -1,
-          idempotent: true,
-          transactionalId: `${this.clientId}-tx`,
-          maxInFlightRequests: 1
-        }
+    if (!this.txProducerInitPromise) {
+      const initPromise = (async () => {
+        const p = this.kafka.producer({
+          kafkaJS: {
+            acks: -1,
+            idempotent: true,
+            transactionalId: `${this.clientId}-tx`,
+            maxInFlightRequests: 1
+          }
+        });
+        await p.connect();
+        return p;
+      })();
+      this.txProducerInitPromise = initPromise.catch((err) => {
+        this.txProducerInitPromise = void 0;
+        throw err;
       });
-      await p.connect();
-      this.txProducer = p;
     }
+    this.txProducer = await this.txProducerInitPromise;
     const tx = await this.txProducer.transaction();
     try {
       const ctx = {
@@ -1002,11 +1104,17 @@ var KafkaClient = class {
     }
   }
   // ── Producer lifecycle ───────────────────────────────────────────
-  /** Connect the idempotent producer. Called automatically by `KafkaModule.register()`. */
+  /**
+   * Connect the idempotent producer. Called automatically by `KafkaModule.register()`.
+   * @internal Not part of `IKafkaClient` — use `disconnect()` for full teardown.
+   */
   async connectProducer() {
     await this.producer.connect();
     this.logger.log("Producer connected");
   }
+  /**
+   * @internal Not part of `IKafkaClient` — use `disconnect()` for full teardown.
+   */
   async disconnectProducer() {
     await this.producer.disconnect();
     this.logger.log("Producer disconnected");
@@ -1020,6 +1128,7 @@ var KafkaClient = class {
     const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", options);
     const deps = this.messageDeps;
     const timeoutMs = options.handlerTimeoutMs;
+    const deduplication = this.resolveDeduplicationContext(gid, options.deduplication);
     await consumer.run({
       eachMessage: (payload) => this.trackInFlight(
         () => handleEachMessage(
@@ -1032,7 +1141,8 @@ var KafkaClient = class {
             retry,
             retryTopics: options.retryTopics,
             timeoutMs,
-            wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this)
+            wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this),
+            deduplication
           },
           deps
         )
@@ -1072,6 +1182,7 @@ var KafkaClient = class {
     const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", options);
     const deps = this.messageDeps;
     const timeoutMs = options.handlerTimeoutMs;
+    const deduplication = this.resolveDeduplicationContext(gid, options.deduplication);
     await consumer.run({
       eachBatch: (payload) => this.trackInFlight(
         () => handleEachBatch(
@@ -1084,7 +1195,8 @@ var KafkaClient = class {
             retry,
             retryTopics: options.retryTopics,
             timeoutMs,
-            wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this)
+            wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this),
+            deduplication
           },
           deps
         )
@@ -1130,18 +1242,27 @@ var KafkaClient = class {
         );
         return;
       }
-      await consumer.disconnect().catch(() => {
-      });
+      await consumer.disconnect().catch(
+        (e) => this.logger.warn(
+          `Error disconnecting consumer "${groupId}":`,
+          toError(e).message
+        )
+      );
       this.consumers.delete(groupId);
       this.runningConsumers.delete(groupId);
       this.consumerCreationOptions.delete(groupId);
+      this.dedupStates.delete(groupId);
       this.logger.log(`Consumer disconnected: group "${groupId}"`);
       const companions = this.companionGroupIds.get(groupId) ?? [];
       for (const cGroupId of companions) {
         const cConsumer = this.consumers.get(cGroupId);
         if (cConsumer) {
-          await cConsumer.disconnect().catch(() => {
-          });
+          await cConsumer.disconnect().catch(
+            (e) => this.logger.warn(
+              `Error disconnecting retry consumer "${cGroupId}":`,
+              toError(e).message
+            )
+          );
           this.consumers.delete(cGroupId);
           this.runningConsumers.delete(cGroupId);
           this.consumerCreationOptions.delete(cGroupId);
@@ -1150,8 +1271,12 @@ var KafkaClient = class {
         const txId = `${cGroupId}-tx`;
         const txProducer = this.retryTxProducers.get(txId);
         if (txProducer) {
-          await txProducer.disconnect().catch(() => {
-          });
+          await txProducer.disconnect().catch(
+            (e) => this.logger.warn(
+              `Error disconnecting retry tx producer "${txId}":`,
+              toError(e).message
+            )
+          );
           this.retryTxProducers.delete(txId);
         }
       }
@@ -1173,6 +1298,7 @@ var KafkaClient = class {
       this.consumerCreationOptions.clear();
       this.companionGroupIds.clear();
       this.retryTxProducers.clear();
+      this.dedupStates.clear();
       this.logger.log("All consumers disconnected");
     }
   }
@@ -1233,6 +1359,7 @@ var KafkaClient = class {
     if (this.txProducer) {
       tasks.push(this.txProducer.disconnect());
       this.txProducer = void 0;
+      this.txProducerInitPromise = void 0;
     }
     for (const p of this.retryTxProducers.values()) {
       tasks.push(p.disconnect());
@@ -1381,6 +1508,22 @@ var KafkaClient = class {
       );
     }
   }
+  /**
+   * When `deduplication.strategy: 'topic'` and `autoCreateTopics: false`, verify
+   * that every `<topic>.duplicates` destination topic already exists. Throws a
+   * clear error at startup rather than silently dropping duplicates on first hit.
+   */
+  async validateDuplicatesTopicsExist(topicNames, customDestination) {
+    await this.ensureAdminConnected();
+    const existing = new Set(await this.admin.listTopics());
+    const toCheck = customDestination ? [customDestination] : topicNames.map((t) => `${t}.duplicates`);
+    const missing = toCheck.filter((t) => !existing.has(t));
+    if (missing.length > 0) {
+      throw new Error(
+        `deduplication.strategy: 'topic' but the following duplicate-routing topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
+      );
+    }
+  }
   /**
    * Connect the admin client if not already connected.
    * The flag is only set to `true` after a successful connect — if `admin.connect()`
@@ -1416,11 +1559,18 @@ var KafkaClient = class {
   }
   async ensureTopic(topic2) {
     if (!this.autoCreateTopicsEnabled || this.ensuredTopics.has(topic2)) return;
-    await this.ensureAdminConnected();
-    await this.admin.createTopics({
-      topics: [{ topic: topic2, numPartitions: this.numPartitions }]
-    });
-    this.ensuredTopics.add(topic2);
+    let p = this.ensureTopicPromises.get(topic2);
+    if (!p) {
+      p = (async () => {
+        await this.ensureAdminConnected();
+        await this.admin.createTopics({
+          topics: [{ topic: topic2, numPartitions: this.numPartitions }]
+        });
+        this.ensuredTopics.add(topic2);
+      })().finally(() => this.ensureTopicPromises.delete(topic2));
+      this.ensureTopicPromises.set(topic2, p);
+    }
+    await p;
   }
   /** Shared consumer setup: groupId check, schema map, connect, subscribe. */
   async setupConsumer(topics, mode, options) {
@@ -1470,6 +1620,16 @@ var KafkaClient = class {
         await this.validateDlqTopicsExist(topicNames);
       }
     }
+    if (options.deduplication?.strategy === "topic") {
+      const dest = options.deduplication.duplicatesTopic;
+      if (this.autoCreateTopicsEnabled) {
+        for (const t of topicNames) {
+          await this.ensureTopic(dest ?? `${t}.duplicates`);
+        }
+      } else {
+        await this.validateDuplicatesTopicsExist(topicNames, dest);
+      }
+    }
     await consumer.connect();
     await subscribeWithRetry(
       consumer,
@@ -1482,13 +1642,22 @@ var KafkaClient = class {
     );
     return { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry };
   }
+  /** Create or retrieve the deduplication context for a consumer group. */
+  resolveDeduplicationContext(groupId, options) {
+    if (!options) return void 0;
+    if (!this.dedupStates.has(groupId)) {
+      this.dedupStates.set(groupId, /* @__PURE__ */ new Map());
+    }
+    return { options, state: this.dedupStates.get(groupId) };
+  }
   // ── Deps object getters ──────────────────────────────────────────
   get producerOpsDeps() {
     return {
       schemaRegistry: this.schemaRegistry,
       strictSchemasEnabled: this.strictSchemasEnabled,
       instrumentation: this.instrumentation,
-      logger: this.logger
+      logger: this.logger,
+      nextLamportClock: () => ++this._lamportClock
     };
   }
   get consumerOpsDeps() {
@@ -1544,6 +1713,7 @@ export {
   HEADER_TIMESTAMP,
   HEADER_SCHEMA_VERSION,
   HEADER_TRACEPARENT,
+  HEADER_LAMPORT_CLOCK,
   getEnvelopeContext,
   runWithEnvelopeContext,
   buildEnvelopeHeaders,
@@ -1555,4 +1725,4 @@ export {
   KafkaClient,
   topic
 };
-//# sourceMappingURL=chunk-7IX4AKQX.mjs.map
+//# sourceMappingURL=chunk-KCUKXR6B.mjs.map