@drarzter/kafka-client 0.7.1 → 0.7.3

package/dist/core.d.ts

@@ -1,5 +1,5 @@
-import { T as TopicMapConstraint, I as IKafkaClient, C as ClientId, G as GroupId, a as KafkaClientOptions, c as TopicDescriptor, w as SendOptions, B as BatchMessageItem, A as TransactionContext, l as EventEnvelope, b as ConsumerOptions, h as ConsumerHandle, e as BatchMeta, k as DlqReplayOptions, d as KafkaHealthResult, t as KafkaMetrics } from './types-BEIGjmV6.js';
-export { f as BeforeConsumeResult, g as CircuitBreakerOptions, i as ConsumerInterceptor, D as DeduplicationOptions, j as DlqReason, E as EnvelopeHeaderOptions, H as HEADER_CORRELATION_ID, m as HEADER_EVENT_ID, n as HEADER_LAMPORT_CLOCK, o as HEADER_SCHEMA_VERSION, p as HEADER_TIMESTAMP, q as HEADER_TRACEPARENT, r as InferSchema, K as KafkaInstrumentation, s as KafkaLogger, M as MessageHeaders, u as MessageLostContext, R as RetryOptions, S as SchemaLike, v as SchemaParseContext, x as SubscribeRetryOptions, y as TTopicMessageMap, z as TopicsFrom, F as TtlExpiredContext, J as buildEnvelopeHeaders, L as decodeHeaders, N as extractEnvelope, O as getEnvelopeContext, P as runWithEnvelopeContext, Q as topic } from './types-BEIGjmV6.js';
+import { T as TopicMapConstraint, I as IKafkaClient, C as ClientId, G as GroupId, a as KafkaClientOptions, c as TopicDescriptor, z as SendOptions, M as MessageHeaders, B as BatchMessageItem, f as BatchSendOptions, O as TransactionContext, o as EventEnvelope, b as ConsumerOptions, k as ConsumerHandle, e as BatchMeta, n as DlqReplayOptions, d as KafkaHealthResult, j as ConsumerGroupSummary, J as TopicDescription, w as KafkaMetrics } from './types-4qWrf2aJ.js';
+export { g as BeforeConsumeResult, h as CircuitBreakerOptions, i as CompressionType, l as ConsumerInterceptor, D as DeduplicationOptions, m as DlqReason, E as EnvelopeHeaderOptions, H as HEADER_CORRELATION_ID, p as HEADER_EVENT_ID, q as HEADER_LAMPORT_CLOCK, r as HEADER_SCHEMA_VERSION, s as HEADER_TIMESTAMP, t as HEADER_TRACEPARENT, u as InferSchema, K as KafkaInstrumentation, v as KafkaLogger, x as MessageLostContext, R as RetryOptions, S as SchemaLike, y as SchemaParseContext, A as SubscribeRetryOptions, F as TTopicMessageMap, L as TopicPartitionInfo, N as TopicsFrom, P as TtlExpiredContext, Q as buildEnvelopeHeaders, U as decodeHeaders, V as extractEnvelope, W as getEnvelopeContext, X as runWithEnvelopeContext, Y as topic } from './types-4qWrf2aJ.js';
 
 /**
  * Type-safe Kafka client.
@@ -16,7 +16,6 @@ declare class KafkaClient<T extends TopicMapConstraint<T>> implements IKafkaClie
     /** Maps transactionalId → Producer for each active retry level consumer. */
     private readonly retryTxProducers;
     private readonly consumers;
-    private readonly admin;
     private readonly logger;
     private readonly autoCreateTopicsEnabled;
     private readonly strictSchemasEnabled;
@@ -36,27 +35,54 @@ declare class KafkaClient<T extends TopicMapConstraint<T>> implements IKafkaClie
     private readonly onRebalance;
     /** Transactional producer ID — configurable via `KafkaClientOptions.transactionalId`. */
     private readonly txId;
-    /** Per-topic event counters, lazily created on first event. Aggregated by `getMetrics()`. */
-    private readonly _topicMetrics;
     /** Monotonically increasing Lamport clock stamped on every outgoing message. */
     private _lamportClock;
     /** Per-groupId deduplication state: `"topic:partition"` → last processed clock. */
     private readonly dedupStates;
-    /** Circuit breaker state per `"${gid}:${topic}:${partition}"` key. */
-    private readonly circuitStates;
-    /** Circuit breaker config per groupId, set at startConsumer/startBatchConsumer time. */
-    private readonly circuitConfigs;
-    private isAdminConnected;
-    private inFlightTotal;
-    private readonly drainResolvers;
+    private readonly circuitBreaker;
+    private readonly adminOps;
+    private readonly metrics;
+    private readonly inFlight;
     readonly clientId: ClientId;
+    private readonly _producerOpsDeps;
+    private readonly _consumerOpsDeps;
+    private readonly _retryTopicDeps;
+    /** DLQ header keys added by the pipeline — stripped before re-publishing. */
+    private static readonly DLQ_HEADER_KEYS;
     constructor(clientId: ClientId, groupId: GroupId, brokers: string[], options?: KafkaClientOptions);
-    /** Send a single typed message. Accepts a topic key or a TopicDescriptor. */
+    /**
+     * Send a single typed message. Accepts a topic key or a `TopicDescriptor`.
+     *
+     * @param topic Topic key from the `TopicMessageMap` or a `TopicDescriptor` object.
+     * @param message Message payload — validated against the topic schema when one is registered.
+     * @param options Optional per-send settings: `key`, `headers`, `correlationId`, `compression`, etc.
+     */
     sendMessage<D extends TopicDescriptor<string & keyof T, T[string & keyof T]>>(descriptor: D, message: D["__type"], options?: SendOptions): Promise<void>;
     sendMessage<K extends keyof T>(topic: K, message: T[K], options?: SendOptions): Promise<void>;
-    /** Send multiple typed messages in one call. Accepts a topic key or a TopicDescriptor. */
-    sendBatch<D extends TopicDescriptor<string & keyof T, T[string & keyof T]>>(descriptor: D, messages: Array<BatchMessageItem<D["__type"]>>): Promise<void>;
-    sendBatch<K extends keyof T>(topic: K, messages: Array<BatchMessageItem<T[K]>>): Promise<void>;
+    /**
+     * Send a null-value (tombstone) message. Used with log-compacted topics to signal
+     * that a key's record should be removed during the next compaction cycle.
+     *
+     * Tombstones skip envelope headers, schema validation, and Lamport clock stamping.
+     * Both `beforeSend` and `afterSend` instrumentation hooks are still called so tracing works correctly.
+     *
+     * @param topic Topic name.
+     * @param key Partition key identifying the record to tombstone.
+     * @param headers Optional custom Kafka headers.
+     */
+    sendTombstone(topic: string, key: string, headers?: MessageHeaders): Promise<void>;
+    /**
+     * Send multiple typed messages in a single Kafka produce request. Accepts a topic key or a `TopicDescriptor`.
+     *
+     * Each item in `messages` can carry its own `key`, `headers`, `correlationId`, and `schemaVersion`.
+     * The `key` is used for partition routing — messages with the same key always land on the same partition.
+     *
+     * @param topic Topic key from the `TopicMessageMap` or a `TopicDescriptor` object.
+     * @param messages Array of messages to send.
+     * @param options Optional batch-level settings: `compression` codec.
+     */
+    sendBatch<D extends TopicDescriptor<string & keyof T, T[string & keyof T]>>(descriptor: D, messages: Array<BatchMessageItem<D["__type"]>>, options?: BatchSendOptions): Promise<void>;
+    sendBatch<K extends keyof T>(topic: K, messages: Array<BatchMessageItem<T[K]>>, options?: BatchSendOptions): Promise<void>;
     /** Execute multiple sends atomically. Commits on success, aborts on error. */
     transaction(fn: (ctx: TransactionContext<T>) => Promise<void>): Promise<void>;
     /**
@@ -68,10 +94,36 @@ declare class KafkaClient<T extends TopicMapConstraint<T>> implements IKafkaClie
      * @internal Not part of `IKafkaClient` — use `disconnect()` for full teardown.
      */
     disconnectProducer(): Promise<void>;
-    /** Subscribe to topics and start consuming messages with the given handler. */
+    /**
+     * Subscribe to one or more topics and start consuming messages one at a time.
+     *
+     * Each message is delivered to `handleMessage` as a fully-decoded `EventEnvelope`.
+     * The call blocks until the consumer is connected and the subscription is set up,
+     * then returns a `ConsumerHandle` with a `stop()` method for clean shutdown.
+     *
+     * @param topics Array of topic keys, `TopicDescriptor` objects, or `RegExp` patterns.
+     *   Regex patterns cannot be combined with `retryTopics: true`.
+     * @param handleMessage Async handler called for every message. Throw to trigger retries.
+     * @param options Consumer configuration — `groupId`, `retry`, `dlq`, `circuitBreaker`, etc.
+     * @returns A handle with `{ groupId, stop() }` for managing the consumer lifecycle.
+     */
     startConsumer<K extends Array<keyof T>>(topics: K, handleMessage: (envelope: EventEnvelope<T[K[number]]>) => Promise<void>, options?: ConsumerOptions<T>): Promise<ConsumerHandle>;
     startConsumer<D extends TopicDescriptor<string & keyof T, T[string & keyof T]>>(topics: D[], handleMessage: (envelope: EventEnvelope<D["__type"]>) => Promise<void>, options?: ConsumerOptions<T>): Promise<ConsumerHandle>;
-    /** Subscribe to topics and consume messages in batches. */
+    /**
+     * Subscribe to one or more topics and consume messages in batches.
+     *
+     * `handleBatch` receives an array of decoded `EventEnvelope` objects together with
+     * batch metadata (topic, partition, high-watermark offset). Prefer this over
+     * `startConsumer` when throughput matters more than per-message latency.
+     *
+     * Set `autoCommit: false` in options when the handler calls `resolveOffset()` or
+     * `commitOffsetsIfNecessary()` directly, to avoid offset conflicts.
+     *
+     * @param topics Array of topic keys, `TopicDescriptor` objects, or `RegExp` patterns.
+     * @param handleBatch Async handler called with each batch of decoded messages.
+     * @param options Consumer configuration — `groupId`, `retry`, `dlq`, `autoCommit`, etc.
+     * @returns A handle with `{ groupId, stop() }` for managing the consumer lifecycle.
+     */
     startBatchConsumer<K extends Array<keyof T>>(topics: K, handleBatch: (envelopes: EventEnvelope<T[K[number]]>[], meta: BatchMeta) => Promise<void>, options?: ConsumerOptions<T>): Promise<ConsumerHandle>;
     startBatchConsumer<D extends TopicDescriptor<string & keyof T, T[string & keyof T]>>(topics: D[], handleBatch: (envelopes: EventEnvelope<D["__type"]>[], meta: BatchMeta) => Promise<void>, options?: ConsumerOptions<T>): Promise<ConsumerHandle>;
     /**
@@ -84,11 +136,31 @@ declare class KafkaClient<T extends TopicMapConstraint<T>> implements IKafkaClie
      * }
      */
     consume<K extends keyof T & string>(topic: K, options?: ConsumerOptions<T>): AsyncIterableIterator<EventEnvelope<T[K]>>;
+    /**
+     * Stop all consumers or a specific group.
+     *
+     * If `groupId` is unspecified, all active consumers are stopped.
+     * If `groupId` is specified, only the consumer with that group ID is stopped.
+     *
+     * @throws {Error} if the consumer fails to disconnect.
+     */
     stopConsumer(groupId?: string): Promise<void>;
+    /**
+     * Temporarily stop delivering messages from specific partitions without disconnecting the consumer.
+     *
+     * @param groupId Consumer group to pause. Defaults to the client's default groupId.
+     * @param assignments Topic-partition pairs to pause.
+     */
     pauseConsumer(groupId: string | undefined, assignments: Array<{
         topic: string;
         partitions: number[];
     }>): void;
+    /**
+     * Resume message delivery for previously paused topic-partitions.
+     *
+     * @param {string|undefined} groupId Consumer group to resume. Defaults to the client's default groupId.
+     * @param {Array<{ topic: string; partitions: number[] }>} assignments Topic-partition pairs to resume.
+     */
     resumeConsumer(groupId: string | undefined, assignments: Array<{
         topic: string;
         partitions: number[];
@@ -97,8 +169,17 @@ declare class KafkaClient<T extends TopicMapConstraint<T>> implements IKafkaClie
     private pauseTopicAllPartitions;
     /** Resume all assigned partitions of a topic for a consumer group (used for queue backpressure). */
     private resumeTopicAllPartitions;
-    /** DLQ header keys added by `sendToDlq` — stripped before re-publishing. */
-    private static readonly DLQ_HEADER_KEYS;
+    /**
+     * Re-publish messages from a dead letter queue back to the original topic.
+     *
+     * Messages are consumed from `<topic>.dlq` and re-published to `<topic>`.
+     * The original topic is determined by the `x-dlq-original-topic` header.
+     * The `x-dlq-*` headers are stripped before re-publishing.
+     *
+     * @param topic - The topic to replay from `<topic>.dlq`
+     * @param options - Options for replay
+     * @returns { replayed: number; skipped: number } - counts of re-published vs skipped messages
+     */
     replayDlq(topic: string, options?: DlqReplayOptions): Promise<{
         replayed: number;
         skipped: number;
@@ -114,11 +195,30 @@ declare class KafkaClient<T extends TopicMapConstraint<T>> implements IKafkaClie
         partition: number;
         offset: string;
     }>): Promise<void>;
+    /**
+     * Seek specific topic-partition pairs to the offset nearest to a given timestamp
+     * (in milliseconds) for a stopped consumer group.
+     * Throws if the group is still running — call `stopConsumer(groupId)` first.
+     * Assignments are grouped by topic and committed via `admin.setOffsets`.
+     * If no offset exists at the requested timestamp (e.g. empty partition or
+     * future timestamp), the partition falls back to `-1` (end of topic — new messages only).
+     */
     seekToTimestamp(groupId: string | undefined, assignments: Array<{
         topic: string;
         partition: number;
         timestamp: number;
     }>): Promise<void>;
+    /**
+     * Returns the current circuit breaker state for a specific topic partition.
+     * Returns `undefined` when no circuit state exists — either `circuitBreaker` is not
+     * configured for the group, or the circuit has never been tripped.
+     *
+     * @param topic Topic name.
+     * @param partition Partition index.
+     * @param groupId Consumer group. Defaults to the client's default groupId.
+     *
+     * @returns `{ status, failures, windowSize }` snapshot for a given partition or `undefined` if no state exists.
+     */
     getCircuitState(topic: string, partition: number, groupId?: string): {
         status: "closed" | "open" | "half-open";
         failures: number;
@@ -142,8 +242,41 @@ declare class KafkaClient<T extends TopicMapConstraint<T>> implements IKafkaClie
     }>>;
     /** Check broker connectivity. Never throws — returns a discriminated union. */
     checkStatus(): Promise<KafkaHealthResult>;
+    /**
+     * List all consumer groups known to the broker.
+     * Useful for monitoring which groups are active and their current state.
+     */
+    listConsumerGroups(): Promise<ConsumerGroupSummary[]>;
+    /**
+     * Describe topics — returns partition layout, leader, replicas, and ISR.
+     * @param topics Topic names to describe. Omit to describe all topics.
+     */
+    describeTopics(topics?: string[]): Promise<TopicDescription[]>;
+    /**
+     * Delete records from a topic up to (but not including) the given offsets.
+     * All messages with offsets **before** the given offset are deleted.
+     */
+    deleteRecords(topic: string, partitions: Array<{
+        partition: number;
+        offset: string;
+    }>): Promise<void>;
+    /** Return the client ID provided during `KafkaClient` construction. */
     getClientId(): ClientId;
+    /**
+     * Return a snapshot of internal event counters accumulated since client creation
+     * (or since the last `resetMetrics()` call).
+     *
+     * @param topic Topic name to scope the snapshot to. When omitted, counters are
+     *   aggregated across all topics. If the topic has no recorded events yet, returns
+     *   a zero-valued snapshot.
+     * @returns Read-only `KafkaMetrics` snapshot: `processedCount`, `retryCount`, `dlqCount`, `dedupCount`.
+     */
     getMetrics(topic?: string): Readonly<KafkaMetrics>;
+    /**
+     * Reset internal event counters to zero.
+     *
+     * @param topic Topic name to reset. When omitted, all topics are reset.
+     */
     resetMetrics(topic?: string): void;
     /** Gracefully disconnect producer, all consumers, and admin. */
     disconnect(drainTimeoutMs?: number): Promise<void>;
@@ -159,60 +292,38 @@ declare class KafkaClient<T extends TopicMapConstraint<T>> implements IKafkaClie
      * NestJS apps get drain for free via `onModuleDestroy` → `disconnect()`.
      */
     enableGracefulShutdown(signals?: NodeJS.Signals[], drainTimeoutMs?: number): void;
-    private trackInFlight;
-    private waitForDrain;
     private preparePayload;
-    private notifyAfterSend;
-    private metricsFor;
-    private notifyRetry;
-    private notifyDlq;
-    private notifyDuplicate;
-    private notifyMessage;
     /**
      * Start a timer that logs a warning if `fn` hasn't resolved within `timeoutMs`.
      * The handler itself is not cancelled — the warning is diagnostic only.
      */
     private wrapWithTimeoutWarning;
-    /**
-     * When `retryTopics: true` and `autoCreateTopics: false`, verify that every
-     * `<topic>.retry.<level>` topic already exists. Throws a clear error at startup
-     * rather than silently discovering missing topics on the first handler failure.
-     */
-    private validateRetryTopicsExist;
-    /**
-     * When `autoCreateTopics` is disabled, verify that `<topic>.dlq` exists for every
-     * consumed topic. Throws a clear error at startup rather than silently discovering
-     * missing DLQ topics on the first handler failure.
-     */
-    private validateDlqTopicsExist;
-    /**
-     * When `deduplication.strategy: 'topic'` and `autoCreateTopics: false`, verify
-     * that every `<topic>.duplicates` destination topic already exists. Throws a
-     * clear error at startup rather than silently dropping duplicates on first hit.
-     */
-    private validateDuplicatesTopicsExist;
-    /**
-     * Connect the admin client if not already connected.
-     * The flag is only set to `true` after a successful connect — if `admin.connect()`
-     * throws the flag remains `false` so the next call will retry the connection.
-     */
-    private ensureAdminConnected;
     /**
      * Create and connect a transactional producer for EOS retry routing.
      * Each retry level consumer gets its own producer with a unique `transactionalId`
      * so Kafka can fence stale producers on restart without affecting other levels.
      */
     private createRetryTxProducer;
+    /**
+     * Ensure that a topic exists by creating it if it doesn't already exist.
+     * If `autoCreateTopics` is disabled, returns immediately.
+     * Concurrent calls for the same topic are deduplicated.
+     */
     private ensureTopic;
     /** Shared consumer setup: groupId check, schema map, connect, subscribe. */
     private setupConsumer;
     /** Create or retrieve the deduplication context for a consumer group. */
     private resolveDeduplicationContext;
-    private get producerOpsDeps();
-    private get consumerOpsDeps();
+    /** Guard checks shared by startConsumer and startBatchConsumer. */
+    private validateTopicConsumerOpts;
+    /** Create EOS transactional producer context for atomic main → retry.1 routing. */
+    private makeEosMainContext;
+    /** Start companion retry-level consumers and register them under the main groupId. */
+    private launchRetryChain;
     /** Build MessageHandlerDeps with circuit breaker callbacks bound to the given groupId. */
     private messageDepsFor;
-    private get retryTopicDeps();
+    /** Build the deps object passed to retry topic consumers. */
+    private buildRetryTopicDeps;
 }
 
 /** Error thrown when a consumer message handler fails. */
@@ -241,4 +352,4 @@ declare class KafkaRetryExhaustedError extends KafkaProcessingError {
     });
 }
 
-export { BatchMessageItem, BatchMeta, ClientId, ConsumerHandle, ConsumerOptions, DlqReplayOptions, EventEnvelope, GroupId, IKafkaClient, KafkaClient, KafkaClientOptions, KafkaHealthResult, KafkaMetrics, KafkaProcessingError, KafkaRetryExhaustedError, KafkaValidationError, SendOptions, TopicDescriptor, TopicMapConstraint, TransactionContext };
+export { BatchMessageItem, BatchMeta, BatchSendOptions, ClientId, ConsumerGroupSummary, ConsumerHandle, ConsumerOptions, DlqReplayOptions, EventEnvelope, GroupId, IKafkaClient, KafkaClient, KafkaClientOptions, KafkaHealthResult, KafkaMetrics, KafkaProcessingError, KafkaRetryExhaustedError, KafkaValidationError, MessageHeaders, SendOptions, TopicDescription, TopicDescriptor, TopicMapConstraint, TransactionContext };