@drarzter/kafka-client 0.7.3 → 0.7.4

package/README.md

@@ -1866,27 +1866,143 @@ Both suites run in CI on every push to `main` and on pull requests.
 
 ```text
 src/
-├── client/                    # Core — 0 framework deps
+├── index.ts                         # Full entrypoint — core + NestJS adapter
+├── core.ts                          # Standalone entrypoint (@drarzter/kafka-client/core)
+├── otel.ts                          # OpenTelemetry entrypoint (@drarzter/kafka-client/otel)
+├── testing.ts                       # Testing entrypoint (@drarzter/kafka-client/testing)
+│
+├── client/                          # Core library — zero framework dependencies
+│   ├── types.ts                     # All public interfaces: KafkaClientOptions, ConsumerOptions,
+│   │                                #   SendOptions, EventEnvelope, ConsumerHandle, BatchMeta,
+│   │                                #   KafkaInstrumentation, ConsumerInterceptor, SchemaLike, …
+│   ├── errors.ts                    # KafkaProcessingError, KafkaRetryExhaustedError, KafkaValidationError
+│   │
+│   ├── message/
+│   │   ├── envelope.ts              # extractEnvelope() — Buffer → EventEnvelope; buildHeaders()
+│   │   └── topic.ts                 # topic() builder → TopicDescriptor; global schema registry;
+│   │                                #   TopicsFrom<T> utility type
+│   │
 │   ├── kafka.client/
-│   │   ├── index.ts           # KafkaClient class
-│   │   ├── admin/             # AdminOps
-│   │   ├── producer/          # payload builders, schema registry
-│   │   ├── consumer/          # consumer ops, handler, retry-topic, DLQ replay, queue, pipeline
-│   │   └── infra/             # CircuitBreakerManager, InFlightTracker, MetricsManager
-│   ├── message/               # EventEnvelope, topic(), headers
-│   ├── __tests__/
-│   │   ├── helpers.ts
-│   │   ├── consumer/          # consumer, batch, retry, dedup, TTL, DLQ replay, …
-│   │   ├── producer/          # producer, transactions, schema, topic
-│   │   ├── admin/             # admin, consumer lag
-│   │   └── infra/             # circuit breaker, errors, instrumentation, OTel, metrics
-│   └── types.ts, errors.ts, …
-├── nest/                      # NestJS adapter — Module, Explorer, decorators, health
-├── testing/                   # Testing utilities — mock client, testcontainer wrapper
-├── core.ts                    # Standalone entrypoint (@drarzter/kafka-client/core)
-├── otel.ts                    # OpenTelemetry entrypoint (@drarzter/kafka-client/otel)
-├── testing.ts                 # Testing entrypoint (@drarzter/kafka-client/testing)
-└── index.ts                   # Full entrypoint — core + NestJS adapter
+│   │   ├── index.ts                 # KafkaClient class — public API, producer/consumer lifecycle,
+│   │   │                            #   Lamport clock, ALS correlation ID, graceful shutdown,
+│   │   │                            #   Lamport clock recovery (clockRecovery option)
+│   │   │
+│   │   ├── admin/
+│   │   │   └── ops.ts               # AdminOps: listConsumerGroups(), describeTopics(),
+│   │   │                            #   deleteRecords(), resetOffsets(), seekToOffset(),
+│   │   │                            #   seekToTimestamp(), getConsumerLag(), ensureTopic()
+│   │   │
+│   │   ├── producer/
+│   │   │   └── ops.ts               # buildPayload() — JSON serialise + schema.parse();
+│   │   │                            #   sendMessage / sendBatch / transaction / sendTombstone;
+│   │   │                            #   schema registry lookup for strictSchemas mode
+│   │   │
+│   │   ├── consumer/
+│   │   │   ├── ops.ts               # setupConsumer() — librdkafka Consumer factory, rebalance
+│   │   │   │                        #   hooks, subscribe with retries, autoCommit config;
+│   │   │   │                        #   startConsumer() / startBatchConsumer() orchestration
+│   │   │   ├── handler.ts           # handleEachMessage() / handleEachBatch() — top-level
+│   │   │   │                        #   eachMessage/eachBatch callbacks wired to pipeline;
+│   │   │   │                        #   EOS main-consumer context for retryTopics mode
+│   │   │   ├── pipeline.ts          # executeWithRetry() — dedup → TTL → interceptors →
+│   │   │   │                        #   handler → retry/DLQ/lost; sendToDlq(); sendToRetryTopic()
+│   │   │   ├── retry-topic.ts       # startRetryTopicConsumers() — spins up N level consumers;
+│   │   │   │                        #   startLevelConsumer() — pause/sleep/resume per partition;
+│   │   │   │                        #   EOS routing via sendOffsetsToTransaction
+│   │   │   ├── subscribe-retry.ts   # subscribeWithRetry() — retries consumer.subscribe() when
+│   │   │   │                        #   topic doesn't exist yet (subscribeRetry option)
+│   │   │   ├── dlq-replay.ts        # replayDlq() — temp consumer reads DLQ up to high-watermark,
+│   │   │   │                        #   strips x-dlq-* headers, re-publishes to original topic
+│   │   │   └── queue.ts             # AsyncQueue — bounded async iterator used by consume();
+│   │   │                            #   backpressure via queueHighWaterMark (pause/resume)
+│   │   │
+│   │   └── infra/
+│   │       ├── circuit-breaker.ts   # CircuitBreakerManager — per groupId:topic:partition state
+│   │       │                        #   machine (CLOSED → OPEN → HALF_OPEN); sliding failure window
+│   │       ├── metrics-manager.ts   # MetricsManager — in-process counters (processed / retry /
+│   │       │                        #   dlq / dedup) per topic; getMetrics() / resetMetrics()
+│   │       └── inflight-tracker.ts  # InFlightTracker — tracks running handlers for graceful
+│   │                                #   shutdown drain (disconnect waits for all to settle)
+│   │
+│   └── __tests__/                   # Unit tests — mocked @confluentinc/kafka-javascript
+│       ├── helpers.ts               # buildMockMessage(), mock setup, spy exports (mockSend, …)
+│       ├── consumer.spec.ts         # Legacy top-level consumer tests
+│       ├── consumer/
+│       │   ├── consumer.spec.ts          # startConsumer() core behaviour
+│       │   ├── batch-consumer.spec.ts    # startBatchConsumer(), BatchMeta, autoCommit
+│       │   ├── consumer-groups.spec.ts   # Multiple groupId, eachMessage/eachBatch conflict guard
+│       │   ├── consumer-handle.spec.ts   # ConsumerHandle.stop()
+│       │   ├── consume-iterator.spec.ts  # consume() iterator, backpressure, break/return
+│       │   ├── retry.spec.ts             # In-process retry, backoff, maxRetries
+│       │   ├── retry-topic.spec.ts       # Retry topic chain, EOS routing, level consumers
+│       │   ├── deduplication.spec.ts     # Lamport clock dedup, strategies (drop/dlq/topic)
+│       │   ├── interceptors.spec.ts      # ConsumerInterceptor before/after/onError hooks
+│       │   ├── dlq-replay.spec.ts        # replayDlq(), dryRun, filter, targetTopic
+│       │   ├── ttl.spec.ts               # messageTtlMs, onTtlExpired, TTL→DLQ routing
+│       │   ├── message-lost.spec.ts      # onMessageLost — handler error, validation, DLQ failure
+│       │   ├── handler-timeout.spec.ts   # handlerTimeoutMs warning
+│       │   └── rebalance.spec.ts         # onRebalance assign/revoke callbacks
+│       ├── producer/
+│       │   ├── producer.spec.ts          # sendMessage(), sendBatch(), sendTombstone(), compression
+│       │   ├── transaction.spec.ts       # transaction(), tx.send(), tx.sendBatch(), rollback
+│       │   ├── schema.spec.ts            # Schema validation on send/consume, strictSchemas
+│       │   └── topic.spec.ts             # topic() descriptor, TopicsFrom, schema registry
+│       ├── admin/
+│       │   ├── admin.spec.ts             # listConsumerGroups(), describeTopics(), deleteRecords(),
+│       │   │                             #   resetOffsets(), seekToOffset(), seekToTimestamp()
+│       │   └── consumer-lag.spec.ts      # getConsumerLag()
+│       └── infra/
+│           ├── circuit-breaker.spec.ts       # CircuitBreaker state machine, getCircuitState()
+│           ├── errors.spec.ts                # Error class hierarchy and properties
+│           ├── instrumentation.spec.ts       # KafkaInstrumentation hooks, wrap/cleanup composition
+│           ├── otel.spec.ts                  # otelInstrumentation(), traceparent propagation
+│           ├── metrics-counters.spec.ts      # getMetrics(), resetMetrics(), per-topic counters
+│           └── metrics-observability.spec.ts # onMessage/onRetry/onDlq/onDuplicate hooks
+│
+├── nest/                            # NestJS adapter — depends on @nestjs/common, reflect-metadata
+│   ├── kafka.module.ts              # KafkaModule.register() / registerAsync(); DynamicModule,
+│   │                                #   isGlobal, named clients; onModuleInit / onModuleDestroy
+│   ├── kafka.explorer.ts            # Auto-discovers @SubscribeTo() methods across all providers
+│   │                                #   at startup and calls startConsumer / startBatchConsumer
+│   ├── kafka.decorator.ts           # @SubscribeTo(topic, options) method decorator;
+│   │                                #   @InjectKafkaClient(name?) parameter decorator
+│   ├── kafka.health.ts              # KafkaHealthIndicator.check() — wraps kafka.checkStatus()
+│   ├── kafka.constants.ts           # DI token constants (KAFKA_CLIENT, KAFKA_OPTIONS)
+│   └── __tests__/
+│       ├── kafka.decorator.spec.ts  # @SubscribeTo / @InjectKafkaClient metadata
+│       ├── kafka.explorer.spec.ts   # Explorer discovery and subscription wiring
+│       └── kafka.health.spec.ts     # KafkaHealthIndicator up/down responses
+│
+├── testing/                         # Testing utilities — no runtime Kafka deps
+│   ├── index.ts                     # Re-exports createMockKafkaClient, KafkaTestContainer
+│   ├── mock-client.ts               # createMockKafkaClient<T>() — jest.fn() on every
+│   │                                #   IKafkaClient method with sensible defaults
+│   ├── test-container.ts            # KafkaTestContainer — thin @testcontainers/kafka wrapper;
+│   │                                #   transaction coordinator warmup, topic pre-creation
+│   └── __tests__/
+│       ├── mock-client.spec.ts      # Mock client method stubs and overrides
+│       └── test-container.spec.ts   # Container start/stop lifecycle
+│
+├── integration/                     # Integration tests — require Docker (testcontainers)
+│   ├── global-setup.ts              # Start shared Kafka container before all suites
+│   ├── global-teardown.ts           # Stop container after all suites
+│   ├── helpers.ts                   # createClient(), waitForMessages(), unique topic names
+│   ├── basic.integration.spec.ts              # Send/receive, headers, batch, fromBeginning
+│   ├── consumer.integration.spec.ts           # startConsumer(), pause/resume, stopConsumer()
+│   ├── transaction.integration.spec.ts        # Atomic sends, rollback on error
+│   ├── retry.integration.spec.ts              # In-process retry, retryTopics chain, DLQ
+│   ├── deduplication.integration.spec.ts      # Lamport clock dedup with real broker
+│   ├── consumer-lag.integration.spec.ts       # getConsumerLag() against real offsets
+│   ├── consumer-handle.integration.spec.ts    # ConsumerHandle.stop() lifecycle
+│   ├── graceful-shutdown.integration.spec.ts  # disconnect() drains in-flight handlers
+│   ├── schema.integration.spec.ts             # Schema validation send+consume round-trip
+│   ├── otel.integration.spec.ts               # OpenTelemetry span propagation end-to-end
+│   └── chaos.integration.spec.ts              # Fault injection — broker restarts, rebalances
+│
+└── __mocks__/
+    └── @confluentinc/
+        └── kafka-javascript.ts      # Manual Jest mock — Kafka, Producer, Consumer stubs;
+                                     #   mockSend, mockTxSend, mockCommit, mockSeek, …
 ```
 
 All exported types and methods have JSDoc comments — your IDE will show inline docs and autocomplete.

package/dist/{chunk-XP7LLRGQ.mjs → chunk-BVWRZTMD.mjs}

@@ -1816,6 +1816,8 @@ var KafkaClient = class _KafkaClient {
   txId;
   /** Monotonically increasing Lamport clock stamped on every outgoing message. */
   _lamportClock = 0;
+  /** Topics to scan for the highest Lamport clock value on `connectProducer()`. */
+  clockRecoveryTopics;
   /** Per-groupId deduplication state: `"topic:partition"` → last processed clock. */
   dedupStates = /* @__PURE__ */ new Map();
   circuitBreaker;
@@ -1851,6 +1853,7 @@ var KafkaClient = class _KafkaClient {
     this.onTtlExpired = options?.onTtlExpired;
     this.onRebalance = options?.onRebalance;
     this.txId = options?.transactionalId ?? `${clientId}-tx`;
+    this.clockRecoveryTopics = options?.clockRecovery?.topics ?? [];
     this.kafka = new KafkaClass({
       kafkaJS: {
         clientId: this.clientId,
@@ -1998,8 +2001,109 @@ var KafkaClient = class _KafkaClient {
    */
   async connectProducer() {
     await this.producer.connect();
+    await this.recoverLamportClock(this.clockRecoveryTopics);
     this.logger.log("Producer connected");
   }
+  /**
+   * Recover the Lamport clock from the last message across the given topics.
+   *
+   * For each topic, fetches partition high-watermarks via admin, creates a
+   * short-lived consumer, seeks every non-empty partition to its last offset
+   * (`highWatermark − 1`), reads one message per partition, and extracts the
+   * maximum `x-lamport-clock` header value. On completion `_lamportClock` is
+   * set to that maximum so the next `++_lamportClock` yields a strictly greater
+   * value than any previously sent clock.
+   *
+   * Topics that are empty or missing are silently skipped.
+   */
+  async recoverLamportClock(topics) {
+    if (topics.length === 0) return;
+    this.logger.log(
+      `Clock recovery: scanning ${topics.length} topic(s) for Lamport clock...`
+    );
+    await this.adminOps.ensureConnected();
+    const partitionsToRead = [];
+    for (const t of topics) {
+      let offsets;
+      try {
+        offsets = await this.adminOps.admin.fetchTopicOffsets(t);
+      } catch {
+        this.logger.warn(
+          `Clock recovery: could not fetch offsets for "${t}", skipping`
+        );
+        continue;
+      }
+      for (const { partition, high, low } of offsets) {
+        if (parseInt(high, 10) > parseInt(low, 10)) {
+          partitionsToRead.push({
+            topic: t,
+            partition,
+            lastOffset: String(parseInt(high, 10) - 1)
+          });
+        }
+      }
+    }
+    if (partitionsToRead.length === 0) {
+      this.logger.log(
+        "Clock recovery: all topics empty \u2014 keeping Lamport clock at 0"
+      );
+      return;
+    }
+    const recoveryGroupId = `${this.clientId}-clock-recovery-${Date.now()}`;
+    let maxClock = -1;
+    await new Promise((resolve, reject) => {
+      const consumer = this.kafka.consumer({
+        kafkaJS: { groupId: recoveryGroupId }
+      });
+      const remaining = new Set(
+        partitionsToRead.map((p) => `${p.topic}:${p.partition}`)
+      );
+      const cleanup = () => {
+        consumer.disconnect().catch(() => {
+        });
+      };
+      consumer.connect().then(async () => {
+        const uniqueTopics = [
+          ...new Set(partitionsToRead.map((p) => p.topic))
+        ];
+        await consumer.subscribe({ topics: uniqueTopics });
+        for (const { topic: t, partition, lastOffset } of partitionsToRead) {
+          consumer.seek({ topic: t, partition, offset: lastOffset });
+        }
+      }).then(
+        () => consumer.run({
+          eachMessage: async ({ topic: t, partition, message }) => {
+            const key = `${t}:${partition}`;
+            if (!remaining.has(key)) return;
+            remaining.delete(key);
+            const clockHeader = message.headers?.[HEADER_LAMPORT_CLOCK];
+            if (clockHeader !== void 0) {
+              const raw = Buffer.isBuffer(clockHeader) ? clockHeader.toString() : String(clockHeader);
+              const clock = Number(raw);
+              if (!isNaN(clock) && clock > maxClock) maxClock = clock;
+            }
+            if (remaining.size === 0) {
+              cleanup();
+              resolve();
+            }
+          }
+        })
+      ).catch((err) => {
+        cleanup();
+        reject(err);
+      });
+    });
+    if (maxClock >= 0) {
+      this._lamportClock = maxClock;
+      this.logger.log(
+        `Clock recovery: Lamport clock restored \u2014 next clock will be ${maxClock + 1}`
+      );
+    } else {
+      this.logger.log(
+        "Clock recovery: no x-lamport-clock headers found \u2014 keeping clock at 0"
+      );
+    }
+  }
   /**
    * @internal Not part of `IKafkaClient` — use `disconnect()` for full teardown.
    */
@@ -2658,4 +2762,4 @@ export {
   KafkaClient,
   topic
 };
-//# sourceMappingURL=chunk-XP7LLRGQ.mjs.map
+//# sourceMappingURL=chunk-BVWRZTMD.mjs.map