@drarzter/kafka-client 0.7.4 → 0.9.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,3 +1,212 @@
1
+ /** A topic-partition pair. */
2
+ type ITopicPartition = {
3
+ topic: string;
4
+ partition: number;
5
+ };
6
+ /** A topic-partition pair with an absolute offset string. */
7
+ type ITopicPartitionOffset = {
8
+ topic: string;
9
+ partition: number;
10
+ offset: string;
11
+ };
12
+ /** Pause / resume assignment shape: one topic + its partition list. */
13
+ type ITopicPartitions = {
14
+ topic: string;
15
+ partitions: number[];
16
+ };
17
+ /** A single message in a produce request. */
18
+ type IProducerMessage = {
19
+ value: string | null;
20
+ key?: string;
21
+ headers?: Record<string, string | Buffer | string[]>;
22
+ };
23
+ /** Produce request payload for one topic. */
24
+ type IProducerRecord = {
25
+ topic: string;
26
+ messages: IProducerMessage[];
27
+ };
28
+ /** Options for creating a producer. */
29
+ type IProducerCreationOptions = {
30
+ /** When set, the producer uses idempotent + exactly-once semantics. */
31
+ transactionalId?: string;
32
+ /** Enable idempotent writes (required for `transactionalId`). */
33
+ idempotent?: boolean;
34
+ };
35
+ /** An open Kafka transaction. */
36
+ interface ITransaction {
37
+ send(record: IProducerRecord): Promise<void>;
38
+ /**
39
+ * Atomically commit offsets for `consumer` as part of this transaction.
40
+ * The `consumer` parameter must be the `IConsumer` whose offsets are being committed.
41
+ */
42
+ sendOffsets(options: {
43
+ consumer: IConsumer;
44
+ topics: Array<{
45
+ topic: string;
46
+ partitions: Array<{
47
+ partition: number;
48
+ offset: string;
49
+ }>;
50
+ }>;
51
+ }): Promise<void>;
52
+ commit(): Promise<void>;
53
+ abort(): Promise<void>;
54
+ }
55
+ /** A Kafka producer. */
56
+ interface IProducer {
57
+ connect(): Promise<void>;
58
+ disconnect(): Promise<void>;
59
+ send(record: IProducerRecord): Promise<void>;
60
+ transaction(): Promise<ITransaction>;
61
+ }
62
+ /** A single message in an `eachMessage` callback. */
63
+ type IMessage = {
64
+ value: Buffer | null;
65
+ /** Header map as returned by librdkafka — values may be arrays. */
66
+ headers: Record<string, any>;
67
+ offset: string;
68
+ key?: Buffer | null;
69
+ };
70
+ /** Payload passed to the `eachMessage` handler. */
71
+ type IEachMessagePayload = {
72
+ topic: string;
73
+ partition: number;
74
+ message: IMessage;
75
+ };
76
+ /** A batch of messages from one topic-partition. */
77
+ type IMessageBatch = {
78
+ topic: string;
79
+ partition: number;
80
+ messages: IMessage[];
81
+ highWatermark?: string;
82
+ };
83
+ /** Payload passed to the `eachBatch` handler. */
84
+ type IEachBatchPayload = {
85
+ batch: IMessageBatch;
86
+ /** Send a heartbeat to the broker to prevent session timeout. */
87
+ heartbeat: () => Promise<void>;
88
+ /** Mark `offset` as processed (without committing). */
89
+ resolveOffset: (offset: string) => void;
90
+ /** Commit if the auto-commit threshold has been reached. */
91
+ commitOffsetsIfNecessary: () => Promise<void>;
92
+ };
93
+ /** Configuration passed to `IConsumer.run()`. */
94
+ type IConsumerRunConfig = {
95
+ eachMessage?: (payload: IEachMessagePayload) => Promise<void>;
96
+ eachBatch?: (payload: IEachBatchPayload) => Promise<void>;
97
+ };
98
+ /** Options for creating a consumer. */
99
+ type IConsumerCreationOptions = {
100
+ groupId: string;
101
+ fromBeginning?: boolean;
102
+ autoCommit?: boolean;
103
+ partitionAssigner?: "cooperative-sticky" | "roundrobin" | "range";
104
+ /** Fired on every partition assign/revoke. */
105
+ onRebalance?: (type: "assign" | "revoke", assignments: ITopicPartition[]) => void;
106
+ };
107
+ /** A Kafka consumer. */
108
+ interface IConsumer {
109
+ connect(): Promise<void>;
110
+ disconnect(): Promise<void>;
111
+ subscribe(options: {
112
+ topics: (string | RegExp)[];
113
+ }): Promise<void>;
114
+ run(config: IConsumerRunConfig): Promise<void>;
115
+ pause(assignments: ITopicPartitions[]): void;
116
+ resume(assignments: ITopicPartitions[]): void;
117
+ /** Seek a partition to an explicit offset. */
118
+ seek(options: ITopicPartitionOffset): void;
119
+ /** Current partition assignment for this consumer. */
120
+ assignment(): ITopicPartition[];
121
+ commitOffsets(offsets: ITopicPartitionOffset[]): Promise<void>;
122
+ /** Stop processing (alias for disconnect in some usages). */
123
+ stop(): Promise<void>;
124
+ }
125
+ /** Low/current/high watermark offsets for one partition. */
126
+ type IPartitionWatermarks = {
127
+ partition: number;
128
+ low: string;
129
+ high: string;
130
+ };
131
+ /** A partition → offset pair. */
132
+ type IPartitionOffset = {
133
+ partition: number;
134
+ offset: string;
135
+ };
136
+ /** Committed offsets for a group's topic. */
137
+ type IGroupTopicOffsets = {
138
+ topic: string;
139
+ partitions: IPartitionOffset[];
140
+ };
141
+ /** A consumer group descriptor. */
142
+ type IGroupDescription = {
143
+ groupId: string;
144
+ state?: string;
145
+ };
146
+ /** Partition metadata. */
147
+ type IPartitionMetadata = {
148
+ partitionId?: number;
149
+ partition?: number;
150
+ leader?: number;
151
+ replicas?: (number | {
152
+ nodeId: number;
153
+ })[];
154
+ isr?: (number | {
155
+ nodeId: number;
156
+ })[];
157
+ };
158
+ /** Topic metadata. */
159
+ type ITopicMetadata = {
160
+ name: string;
161
+ partitions: IPartitionMetadata[];
162
+ };
163
+ /** A Kafka admin client. */
164
+ interface IAdmin {
165
+ connect(): Promise<void>;
166
+ disconnect(): Promise<void>;
167
+ createTopics(options: {
168
+ topics: Array<{
169
+ topic: string;
170
+ numPartitions: number;
171
+ }>;
172
+ }): Promise<void>;
173
+ fetchTopicOffsets(topic: string): Promise<IPartitionWatermarks[]>;
174
+ fetchTopicOffsetsByTimestamp(topic: string, timestamp: number): Promise<IPartitionOffset[]>;
175
+ fetchOffsets(options: {
176
+ groupId: string;
177
+ }): Promise<IGroupTopicOffsets[]>;
178
+ setOffsets(options: {
179
+ groupId: string;
180
+ topic: string;
181
+ partitions: IPartitionOffset[];
182
+ }): Promise<void>;
183
+ listTopics(): Promise<string[]>;
184
+ listGroups(): Promise<{
185
+ groups: IGroupDescription[];
186
+ }>;
187
+ fetchTopicMetadata(options?: {
188
+ topics?: string[];
189
+ }): Promise<{
190
+ topics: ITopicMetadata[];
191
+ }>;
192
+ deleteGroups(groupIds: string[]): Promise<void>;
193
+ deleteTopicRecords(options: {
194
+ topic: string;
195
+ partitions: IPartitionOffset[];
196
+ }): Promise<void>;
197
+ }
198
+ /**
199
+ * Factory that creates connected Kafka primitives.
200
+ * The default implementation wraps `@confluentinc/kafka-javascript` via
201
+ * `ConfluentTransport`. Inject a custom transport (e.g. a fake) via
202
+ * `KafkaClientOptions.transport` for testing or alternative broker support.
203
+ */
204
+ interface KafkaTransport {
205
+ producer(options?: IProducerCreationOptions): IProducer;
206
+ consumer(options: IConsumerCreationOptions): IConsumer;
207
+ admin(): IAdmin;
208
+ }
209
+
1
210
  /**
2
211
  * Context passed as the second argument to `SchemaLike.parse()`.
3
212
  * Enables schema-registry adapters, version-aware migration, and
@@ -115,6 +324,15 @@ declare const HEADER_LAMPORT_CLOCK = "x-lamport-clock";
115
324
  *
116
325
  * On **consume**, the library extracts those headers and assembles
117
326
  * an `EventEnvelope` that is passed to the handler.
327
+ *
328
+ * @example
329
+ * ```ts
330
+ * await kafka.startConsumer(['orders'], async (envelope: EventEnvelope<Order>) => {
331
+ * console.log(envelope.payload.orderId); // typed payload
332
+ * console.log(envelope.correlationId); // auto-propagated
333
+ * console.log(envelope.eventId); // unique message ID
334
+ * });
335
+ * ```
118
336
  */
119
337
  interface EventEnvelope<T> {
120
338
  /** Deserialized + validated message body. */
@@ -142,9 +360,26 @@ interface EnvelopeCtx {
142
360
  correlationId: string;
143
361
  traceparent?: string;
144
362
  }
145
- /** Read the current envelope context (correlationId / traceparent) from ALS. */
363
+ /**
364
+ * Read the current envelope context (correlationId / traceparent) from ALS.
365
+ * Returns `undefined` outside of a Kafka consumer handler.
366
+ * @example
367
+ * ```ts
368
+ * const ctx = getEnvelopeContext();
369
+ * if (ctx) console.log('correlationId:', ctx.correlationId);
370
+ * ```
371
+ */
146
372
  declare function getEnvelopeContext(): EnvelopeCtx | undefined;
147
- /** Execute `fn` inside an envelope context so nested sends inherit correlationId. */
373
+ /**
374
+ * Execute `fn` inside an envelope context so nested sends inherit correlationId.
375
+ * Automatically called by the consumer pipeline — use this in tests or manual flows.
376
+ * @example
377
+ * ```ts
378
+ * await runWithEnvelopeContext({ correlationId: 'abc-123' }, async () => {
379
+ * await kafka.sendMessage('orders.created', payload); // inherits correlationId
380
+ * });
381
+ * ```
382
+ */
148
383
  declare function runWithEnvelopeContext<R>(ctx: EnvelopeCtx, fn: () => R): R;
149
384
  /** Options accepted by `buildEnvelopeHeaders`. */
150
385
  interface EnvelopeHeaderOptions {
@@ -213,7 +448,18 @@ type MessageHeaders = Record<string, string>;
213
448
  * - `'zstd'` — best ratio, slightly slower
214
449
  */
215
450
  type CompressionType = "none" | "gzip" | "snappy" | "lz4" | "zstd";
216
- /** Options for sending a single message. */
451
+ /**
452
+ * Options for sending a single message.
453
+ *
454
+ * @example
455
+ * ```ts
456
+ * await kafka.sendMessage('orders.created', { orderId: '123', amount: 99 }, {
457
+ * key: 'order-123',
458
+ * headers: { 'x-source': 'checkout-service' },
459
+ * compression: 'snappy',
460
+ * });
461
+ * ```
462
+ */
217
463
  interface SendOptions {
218
464
  /** Partition key for message routing. */
219
465
  key?: string;
@@ -232,7 +478,17 @@ interface SendOptions {
232
478
  */
233
479
  compression?: CompressionType;
234
480
  }
235
- /** Shape of each item in a `sendBatch` call. */
481
+ /**
482
+ * Shape of each item in a `sendBatch` call.
483
+ *
484
+ * @example
485
+ * ```ts
486
+ * await kafka.sendBatch('orders.created', [
487
+ * { value: { orderId: '1', amount: 10 }, key: 'order-1' },
488
+ * { value: { orderId: '2', amount: 20 }, key: 'order-2', headers: { 'x-priority': 'high' } },
489
+ * ]);
490
+ * ```
491
+ */
236
492
  interface BatchMessageItem<V> {
237
493
  value: V;
238
494
  /**
@@ -247,7 +503,14 @@ interface BatchMessageItem<V> {
247
503
  schemaVersion?: number;
248
504
  eventId?: string;
249
505
  }
250
- /** Options for a `sendBatch` call (applies to all messages in the batch). */
506
+ /**
507
+ * Options for a `sendBatch` call (applies to all messages in the batch).
508
+ *
509
+ * @example
510
+ * ```ts
511
+ * await kafka.sendBatch('metrics', messages, { compression: 'zstd' });
512
+ * ```
513
+ */
251
514
  interface BatchSendOptions {
252
515
  /**
253
516
  * Compression codec for this batch.
@@ -256,7 +519,19 @@ interface BatchSendOptions {
256
519
  */
257
520
  compression?: CompressionType;
258
521
  }
259
- /** Metadata exposed to batch consumer handlers. */
522
+ /**
523
+ * Metadata exposed to batch consumer handlers.
524
+ *
525
+ * @example
526
+ * ```ts
527
+ * await kafka.startBatchConsumer(['events'], async (envelopes, meta) => {
528
+ * console.log(`Partition ${meta.partition}, HWM ${meta.highWatermark}`);
529
+ * await db.insertMany(envelopes.map(e => e.payload));
530
+ * meta.resolveOffset(envelopes.at(-1)!.offset);
531
+ * await meta.commitOffsetsIfNecessary();
532
+ * }, { autoCommit: false });
533
+ * ```
534
+ */
260
535
  interface BatchMeta {
261
536
  /** Partition number for this batch. */
262
537
  partition: number;
@@ -277,6 +552,14 @@ interface BatchMeta {
277
552
  /**
278
553
  * Options for Lamport Clock-based message deduplication.
279
554
  *
555
+ * @example
556
+ * ```ts
557
+ * await kafka.startConsumer(['payments'], handler, {
558
+ * deduplication: { strategy: 'dlq' },
559
+ * dlq: true,
560
+ * });
561
+ * ```
562
+ *
280
563
  * The producer stamps every outgoing message with a monotonically increasing
281
564
  * `x-lamport-clock` header. The consumer tracks the last processed value per
282
565
  * `topic:partition` and skips any message whose clock is not strictly greater
@@ -311,6 +594,19 @@ interface DeduplicationOptions {
311
594
  /**
312
595
  * Options for the per-partition circuit breaker.
313
596
  *
597
+ * @example
598
+ * ```ts
599
+ * await kafka.startConsumer(['payments'], handler, {
600
+ * circuitBreaker: {
601
+ * threshold: 5,
602
+ * recoveryMs: 30_000,
603
+ * windowSize: 20,
604
+ * halfOpenSuccesses: 2,
605
+ * },
606
+ * dlq: true,
607
+ * });
608
+ * ```
609
+ *
314
610
  * The circuit breaker tracks recent message outcomes in a **sliding window** and
315
611
  * opens (pauses the partition) when too many failures accumulate:
316
612
  *
@@ -343,7 +639,21 @@ interface CircuitBreakerOptions {
343
639
  */
344
640
  halfOpenSuccesses?: number;
345
641
  }
346
- /** Options for configuring a Kafka consumer. */
642
+ /**
643
+ * Options for configuring a Kafka consumer.
644
+ *
645
+ * @example
646
+ * ```ts
647
+ * await kafka.startConsumer(['orders.created'], handler, {
648
+ * groupId: 'billing-service',
649
+ * retry: { maxRetries: 5, backoffMs: 1000 },
650
+ * dlq: true,
651
+ * deduplication: { strategy: 'drop' },
652
+ * circuitBreaker: { threshold: 3, recoveryMs: 60_000 },
653
+ * interceptors: [loggingInterceptor],
654
+ * });
655
+ * ```
656
+ */
347
657
  interface ConsumerOptions<T extends TopicMapConstraint<T> = TTopicMessageMap> {
348
658
  /** Override the default consumer group ID from the constructor. */
349
659
  groupId?: string;
@@ -430,8 +740,32 @@ interface ConsumerOptions<T extends TopicMapConstraint<T> = TTopicMessageMap> {
430
740
  * when both are set. Use this to handle TTL expiry differently per consumer group.
431
741
  */
432
742
  onTtlExpired?: (ctx: TtlExpiredContext) => void | Promise<void>;
743
+ /**
744
+ * Called when a message is silently dropped after all retries are exhausted
745
+ * and `dlq` is not enabled.
746
+ *
747
+ * **Per-consumer override**: takes precedence over `KafkaClientOptions.onMessageLost`
748
+ * when both are set. Use this for consumer-specific alerting or dead-message logging.
749
+ */
750
+ onMessageLost?: (ctx: MessageLostContext) => void | Promise<void>;
751
+ /**
752
+ * Called before each retry attempt (both in-process and retry-topic paths).
753
+ *
754
+ * **Per-consumer override**: fires in addition to any global instrumentation hooks.
755
+ * Use this for per-consumer retry metrics or structured logging.
756
+ */
757
+ onRetry?: (envelope: EventEnvelope<any>, attempt: number, maxRetries: number) => void | Promise<void>;
433
758
  }
434
- /** Configuration for consumer retry behavior. */
759
+ /**
760
+ * Configuration for consumer retry behavior.
761
+ *
762
+ * @example
763
+ * ```ts
764
+ * await kafka.startConsumer(['orders'], handler, {
765
+ * retry: { maxRetries: 5, backoffMs: 500, maxBackoffMs: 10_000 },
766
+ * });
767
+ * ```
768
+ */
435
769
  interface RetryOptions {
436
770
  /** Maximum number of retry attempts before giving up. */
437
771
  maxRetries: number;
@@ -446,6 +780,17 @@ interface RetryOptions {
446
780
  *
447
781
  * Interceptors are per-consumer. For client-wide hooks (e.g. OTel),
448
782
  * use `KafkaInstrumentation` instead.
783
+ *
784
+ * @example
785
+ * ```ts
786
+ * const logger: ConsumerInterceptor = {
787
+ * async before(envelope) { console.log('Processing', envelope.eventId); },
788
+ * async after(envelope) { console.log('Done', envelope.eventId); },
789
+ * async onError(envelope, err) { console.error('Failed', err.message); },
790
+ * };
791
+ *
792
+ * await kafka.startConsumer(['orders'], handler, { interceptors: [logger] });
793
+ * ```
449
794
  */
450
795
  interface ConsumerInterceptor<T extends TopicMapConstraint<T> = TTopicMessageMap> {
451
796
  /** Called before the message handler. */
@@ -465,6 +810,25 @@ interface ConsumerInterceptor<T extends TopicMapConstraint<T> = TTopicMessageMap
465
810
  * async context (e.g. `context.with(spanCtx, fn)` for OpenTelemetry). Multiple
466
811
  * wraps from different instrumentations are composed in declaration order,
467
812
  * so the first instrumentation's wrap is the outermost.
813
+ *
814
+ * @example
815
+ * ```ts
816
+ * // Cleanup-only (legacy form):
817
+ * beforeConsume(envelope) {
818
+ * const timer = startTimer();
819
+ * return () => timer.end();
820
+ * }
821
+ *
822
+ * // Wrap form — run handler inside an OTel span context:
823
+ * beforeConsume(envelope) {
824
+ * const ctx = propagator.extract(ROOT_CONTEXT, envelope.headers);
825
+ * const span = tracer.startSpan(envelope.topic, {}, ctx);
826
+ * return {
827
+ * wrap: (fn) => context.with(trace.setSpan(ctx, span), fn),
828
+ * cleanup: () => span.end(),
829
+ * };
830
+ * }
831
+ * ```
468
832
  */
469
833
  type BeforeConsumeResult = (() => void) | {
470
834
  cleanup?(): void;
@@ -479,7 +843,80 @@ type BeforeConsumeResult = (() => void) | {
479
843
  * - `'ttl-expired'` — message age exceeded `messageTtlMs` before the handler ran.
480
844
  */
481
845
  type DlqReason = "handler-error" | "validation-error" | "lamport-clock-duplicate" | "ttl-expired";
482
- /** Options for `replayDlq`. */
846
+ /**
847
+ * Options for `readSnapshot`.
848
+ *
849
+ * @example
850
+ * ```ts
851
+ * const snapshot = await kafka.readSnapshot('users.state', {
852
+ * schema: UserSchema,
853
+ * onTombstone: (key) => console.log(`Key ${key} was compacted away`),
854
+ * });
855
+ * ```
856
+ */
857
+ interface ReadSnapshotOptions {
858
+ /**
859
+ * Schema to validate each message payload against (Zod, Valibot, ArkType, or any `.parse()` shape).
860
+ * Messages that fail validation are skipped with a warning log — they do not throw.
861
+ */
862
+ schema?: SchemaLike;
863
+ /**
864
+ * Called when a tombstone record (null-value message) is encountered.
865
+ * The corresponding key is removed from the snapshot automatically.
866
+ * Use this for auditing or logging which keys were compacted away.
867
+ */
868
+ onTombstone?: (key: string) => void;
869
+ }
870
+ /** A single partition offset entry stored in a checkpoint record. */
871
+ interface CheckpointEntry {
872
+ topic: string;
873
+ partition: number;
874
+ offset: string;
875
+ }
876
+ /** Result returned by a successful `checkpointOffsets` call. */
877
+ interface CheckpointResult {
878
+ /** Consumer group whose offsets were saved. */
879
+ groupId: string;
880
+ /** Topics included in the checkpoint. */
881
+ topics: string[];
882
+ /** Total number of topic-partition pairs saved. */
883
+ partitionCount: number;
884
+ /** Unix timestamp (ms) when the checkpoint was created. */
885
+ savedAt: number;
886
+ }
887
+ /** Options for `restoreFromCheckpoint`. */
888
+ interface RestoreCheckpointOptions {
889
+ /**
890
+ * Target Unix timestamp (ms). The newest checkpoint whose `savedAt` is **≤ this value**
891
+ * is selected. Defaults to the latest available checkpoint when omitted.
892
+ */
893
+ timestamp?: number;
894
+ }
895
+ /** Result returned by a successful `restoreFromCheckpoint` call. */
896
+ interface CheckpointRestoreResult {
897
+ /** Consumer group that was repositioned. */
898
+ groupId: string;
899
+ /** The committed offsets restored from the checkpoint. */
900
+ offsets: CheckpointEntry[];
901
+ /** Unix timestamp (ms) recorded when the checkpoint was originally saved. */
902
+ restoredAt: number;
903
+ /** Age of the restored checkpoint in milliseconds (now − `restoredAt`). */
904
+ checkpointAge: number;
905
+ }
906
+ /**
907
+ * Options for `replayDlq`.
908
+ *
909
+ * @example
910
+ * ```ts
911
+ * await kafka.replayDlq('orders.created', {
912
+ * targetTopic: 'orders.retry-manual',
913
+ * dryRun: false,
914
+ * filter: (headers, value) =>
915
+ * headers['x-dlq-reason'] === 'handler-error' &&
916
+ * JSON.parse(value).amount > 0,
917
+ * });
918
+ * ```
919
+ */
483
920
  interface DlqReplayOptions {
484
921
  /**
485
922
  * Override the target topic to re-publish to.
@@ -497,6 +934,13 @@ interface DlqReplayOptions {
497
934
  * @param value Raw message value (JSON string).
498
935
  */
499
936
  filter?: (headers: MessageHeaders, value: string) => boolean;
937
+ /**
938
+ * Seek to the earliest available offset before consuming, regardless of any
939
+ * previously committed offsets for the replay consumer group.
940
+ * Default: `true` — full replay of all DLQ messages on every call.
941
+ * Set to `false` to replay only messages added since the previous `replayDlq` call.
942
+ */
943
+ fromBeginning?: boolean;
500
944
  }
501
945
  /**
502
946
  * Snapshot of internal event counters accumulated since client creation
@@ -517,6 +961,24 @@ interface KafkaMetrics {
517
961
  * Use this for cross-cutting concerns like tracing and metrics.
518
962
  *
519
963
  * @see `otelInstrumentation()` from `@drarzter/kafka-client/otel`
964
+ *
965
+ * @example
966
+ * ```ts
967
+ * const tracing: KafkaInstrumentation = {
968
+ * beforeSend(topic, headers) {
969
+ * headers['traceparent'] = getCurrentTraceId();
970
+ * },
971
+ * beforeConsume(envelope) {
972
+ * const span = tracer.startSpan(envelope.topic);
973
+ * return { cleanup: () => span.end() };
974
+ * },
975
+ * onDlq(envelope, reason) {
976
+ * metrics.increment('kafka.dlq', { topic: envelope.topic, reason });
977
+ * },
978
+ * };
979
+ *
980
+ * const kafka = new KafkaClient(config, groupId, { instrumentation: [tracing] });
981
+ * ```
520
982
  */
521
983
  interface KafkaInstrumentation {
522
984
  /** Called before sending — can mutate `headers` (e.g. inject `traceparent`). */
@@ -558,19 +1020,155 @@ interface KafkaInstrumentation {
558
1020
  /** Called when the circuit closes (normal operation restored). */
559
1021
  onCircuitClose?(topic: string, partition: number): void;
560
1022
  }
561
- /** Context passed to the `transaction()` callback with type-safe send methods. */
1023
+ /**
1024
+ * Context passed to the `transaction()` callback with type-safe send methods.
1025
+ *
1026
+ * @example
1027
+ * ```ts
1028
+ * await kafka.transaction(async (tx) => {
1029
+ * await tx.send('inventory.reserved', { itemId: 'a', qty: 1 });
1030
+ * await tx.sendBatch('audit.log', [
1031
+ * { value: { action: 'reserve', itemId: 'a' } },
1032
+ * ]);
1033
+ * });
1034
+ * ```
1035
+ */
562
1036
  interface TransactionContext<T extends TopicMapConstraint<T>> {
563
1037
  send<K extends keyof T>(topic: K, message: T[K], options?: SendOptions): Promise<void>;
564
1038
  send<D extends TopicDescriptor<string & keyof T, T[string & keyof T]>>(descriptor: D, message: D["__type"], options?: SendOptions): Promise<void>;
565
1039
  sendBatch<K extends keyof T>(topic: K, messages: Array<BatchMessageItem<T[K]>>, options?: BatchSendOptions): Promise<void>;
566
1040
  sendBatch<D extends TopicDescriptor<string & keyof T, T[string & keyof T]>>(descriptor: D, messages: Array<BatchMessageItem<D["__type"]>>, options?: BatchSendOptions): Promise<void>;
567
1041
  }
568
- /** Handle returned by `startConsumer` / `startBatchConsumer`. */
1042
+ /**
1043
+ * Transactional context passed to each `startTransactionalConsumer` handler invocation.
1044
+ *
1045
+ * Every call to `send` or `sendBatch` on this object is part of the same Kafka
1046
+ * transaction as the source message offset commit. Either all sends and the offset
1047
+ * commit succeed atomically, or the transaction is aborted and the source message
1048
+ * is redelivered.
1049
+ *
1050
+ * @example
1051
+ * ```ts
1052
+ * await kafka.startTransactionalConsumer(['orders.created'], async (envelope, tx) => {
1053
+ * await tx.send('inventory.reserved', { orderId: envelope.payload.orderId, qty: 1 });
1054
+ * await tx.sendBatch('audit.log', [{ value: { action: 'reserve', orderId: envelope.payload.orderId } }]);
1055
+ * });
1056
+ * ```
1057
+ */
1058
+ interface TransactionalHandlerContext<T extends TopicMapConstraint<T>> {
1059
+ /**
1060
+ * Send a message as part of this message's transaction.
1061
+ * The send is staged — it only becomes visible to consumers after the transaction commits.
1062
+ */
1063
+ send<K extends keyof T>(topic: K, message: T[K], options?: SendOptions): Promise<void>;
1064
+ /**
1065
+ * Send multiple messages as part of this message's transaction.
1066
+ * All messages are staged together and become visible only after commit.
1067
+ */
1068
+ sendBatch<K extends keyof T>(topic: K, messages: Array<BatchMessageItem<T[K]>>, options?: BatchSendOptions): Promise<void>;
1069
+ }
1070
+ /**
1071
+ * Routing configuration for `startRoutedConsumer`.
1072
+ *
1073
+ * Messages are dispatched to the handler whose key matches the value of `header`.
1074
+ * Unmatched messages are forwarded to `fallback` if provided, or silently skipped.
1075
+ *
1076
+ * @example
1077
+ * ```ts
1078
+ * await kafka.startRoutedConsumer(['events'], {
1079
+ * header: 'x-event-type',
1080
+ * routes: {
1081
+ * 'order.created': async (e) => handleOrderCreated(e.payload),
1082
+ * 'order.cancelled': async (e) => handleOrderCancelled(e.payload),
1083
+ * },
1084
+ * fallback: async (e) => logger.warn('Unknown event type', e.headers),
1085
+ * });
1086
+ * ```
1087
+ */
1088
+ interface RoutingOptions<E> {
1089
+ /** Header whose value determines which handler is invoked. */
1090
+ header: string;
1091
+ /**
1092
+ * Map of header value → handler.
1093
+ * The handler with the matching key is called for each message.
1094
+ */
1095
+ routes: Record<string, (envelope: EventEnvelope<E>) => Promise<void>>;
1096
+ /**
1097
+ * Called when no route matches the header value (or the header is absent).
1098
+ * If omitted, unmatched messages are silently skipped.
1099
+ */
1100
+ fallback?: (envelope: EventEnvelope<E>) => Promise<void>;
1101
+ }
1102
+ /**
1103
+ * Metadata passed to the `startWindowConsumer` handler on each flush.
1104
+ *
1105
+ * @example
1106
+ * ```ts
1107
+ * await kafka.startWindowConsumer('events', async (batch, meta) => {
1108
+ * console.log(`Flush: ${batch.length} events, trigger=${meta.trigger}`);
1109
+ * console.log(`Window: ${meta.windowEnd - meta.windowStart} ms`);
1110
+ * await db.insertMany(batch.map(e => e.payload));
1111
+ * }, { maxMessages: 100, maxMs: 5_000 });
1112
+ * ```
1113
+ */
1114
+ interface WindowMeta {
1115
+ /** What triggered this flush: accumulated `maxMessages` messages, or the `maxMs` timer. */
1116
+ trigger: "size" | "time";
1117
+ /** Unix timestamp (ms) of the first message that entered the current window. */
1118
+ windowStart: number;
1119
+ /** Unix timestamp (ms) when the flush was initiated. */
1120
+ windowEnd: number;
1121
+ }
1122
+ /**
1123
+ * Options for `startWindowConsumer`.
1124
+ *
1125
+ * Extends `ConsumerOptions` — all standard consumer settings apply.
1126
+ * `retryTopics`, `retryTopicAssignmentTimeoutMs`, and `queueHighWaterMark`
1127
+ * are excluded: they are incompatible with windowed accumulation.
1128
+ */
1129
+ interface WindowConsumerOptions<T extends TopicMapConstraint<T> = TTopicMessageMap> extends Omit<ConsumerOptions<T>, "retryTopics" | "retryTopicAssignmentTimeoutMs" | "queueHighWaterMark"> {
1130
+ /**
1131
+ * Maximum number of messages to accumulate before flushing.
1132
+ * When the buffer reaches this size the handler is called immediately,
1133
+ * regardless of how much time has elapsed since the first message.
1134
+ */
1135
+ maxMessages: number;
1136
+ /**
1137
+ * Maximum time (ms) to wait after the first message before flushing.
1138
+ * When this timer expires the handler is called with whatever messages
1139
+ * have accumulated so far — even if the buffer is not full.
1140
+ */
1141
+ maxMs: number;
1142
+ }
1143
+ /**
1144
+ * Handle returned by `startConsumer` / `startBatchConsumer`.
1145
+ *
1146
+ * @example
1147
+ * ```ts
1148
+ * const handle = await kafka.startConsumer(['orders'], handler);
1149
+ * // later, on shutdown:
1150
+ * await handle.stop();
1151
+ * ```
1152
+ */
569
1153
  interface ConsumerHandle {
570
1154
  /** The consumer group ID this consumer is running under. */
571
1155
  groupId: string;
572
1156
  /** Stop this consumer. Equivalent to calling `client.stopConsumer(groupId)`. */
573
1157
  stop(): Promise<void>;
1158
+ /**
1159
+ * Resolves once the consumer has received its first partition assignment from
1160
+ * the broker (i.e. it has joined the group and is ready to receive messages).
1161
+ *
1162
+ * Use this in tests and startup probes instead of a fixed `setTimeout` delay:
1163
+ *
1164
+ * @example
1165
+ * ```ts
1166
+ * const handle = await kafka.startConsumer(['orders'], handler);
1167
+ * await handle.ready(); // wait for partition assignment — no fixed sleep needed
1168
+ * await kafka.sendMessage('orders', payload);
1169
+ * ```
1170
+ */
1171
+ ready(): Promise<void>;
574
1172
  }
575
1173
  /** Result returned by `KafkaClient.checkStatus()`. */
576
1174
  type KafkaHealthResult = {
@@ -612,15 +1210,34 @@ interface TopicDescription {
612
1210
  }
613
1211
  /** Interface describing all public methods of the Kafka client. */
614
1212
  interface IKafkaClient<T extends TopicMapConstraint<T>> {
1213
+ /**
1214
+ * @example
1215
+ * ```ts
1216
+ * const status = await kafka.checkStatus();
1217
+ * if (status.status === 'down') console.error(status.error);
1218
+ * ```
1219
+ */
615
1220
  checkStatus(): Promise<KafkaHealthResult>;
616
1221
  /**
617
1222
  * List all consumer groups known to the broker.
618
1223
  * @returns Array of `{ groupId, state }` summaries.
1224
+ *
1225
+ * @example
1226
+ * ```ts
1227
+ * const groups = await kafka.listConsumerGroups();
1228
+ * console.log(groups.map(g => `${g.groupId}: ${g.state}`));
1229
+ * ```
619
1230
  */
620
1231
  listConsumerGroups(): Promise<ConsumerGroupSummary[]>;
621
1232
  /**
622
1233
  * Describe topics — returns partition layout, leader, replicas, and ISR for each topic.
623
1234
  * @param topics Topic names to describe. Omit to describe all topics visible to this client.
1235
+ *
1236
+ * @example
1237
+ * ```ts
1238
+ * const [desc] = await kafka.describeTopics(['orders.created']);
1239
+ * console.log(desc.partitions.length); // number of partitions
1240
+ * ```
624
1241
  */
625
1242
  describeTopics(topics?: string[]): Promise<TopicDescription[]>;
626
1243
  /**
@@ -630,6 +1247,14 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
630
1247
  *
631
1248
  * @param topic Topic name.
632
1249
  * @param partitions Array of `{ partition, offset }` — records before each offset are deleted.
1250
+ *
1251
+ * @example
1252
+ * ```ts
1253
+ * await kafka.deleteRecords('orders.created', [
1254
+ * { partition: 0, offset: '1000' },
1255
+ * { partition: 1, offset: '500' },
1256
+ * ]);
1257
+ * ```
633
1258
  */
634
1259
  deleteRecords(topic: string, partitions: Array<{
635
1260
  partition: number;
@@ -640,12 +1265,30 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
640
1265
  * Lag = (broker high-watermark offset) − (last committed offset).
641
1266
  * - A committed offset of `-1` (no offset committed yet) counts as full lag.
642
1267
  * - Defaults to the client's default `groupId` when none is provided.
1268
+ *
1269
+ * @example
1270
+ * ```ts
1271
+ * const lag = await kafka.getConsumerLag();
1272
+ * const total = lag.reduce((sum, p) => sum + p.lag, 0);
1273
+ * console.log(`Total lag: ${total} messages`);
1274
+ * ```
643
1275
  */
644
1276
  getConsumerLag(groupId?: string): Promise<Array<{
645
1277
  topic: string;
646
1278
  partition: number;
647
1279
  lag: number;
648
1280
  }>>;
1281
+ /**
1282
+ * @example
1283
+ * ```ts
1284
+ * const handle = await kafka.startConsumer(['orders.created'], async (envelope) => {
1285
+ * await processOrder(envelope.payload);
1286
+ * }, { retry: { maxRetries: 3 }, dlq: true });
1287
+ *
1288
+ * // on shutdown:
1289
+ * await handle.stop();
1290
+ * ```
1291
+ */
649
1292
  startConsumer<K extends Array<keyof T>>(topics: K, handleMessage: (envelope: EventEnvelope<T[K[number]]>) => Promise<void>, options?: ConsumerOptions<T>): Promise<ConsumerHandle>;
650
1293
  startConsumer<D extends TopicDescriptor<string & keyof T, T[string & keyof T]>>(topics: D[], handleMessage: (envelope: EventEnvelope<D["__type"]>) => Promise<void>, options?: ConsumerOptions<T>): Promise<ConsumerHandle>;
651
1294
  /**
@@ -654,6 +1297,15 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
654
1297
  * Incompatible with `retryTopics: true`.
655
1298
  */
656
1299
  startConsumer(topics: (string | RegExp)[], handleMessage: (envelope: EventEnvelope<T[keyof T]>) => Promise<void>, options?: ConsumerOptions<T>): Promise<ConsumerHandle>;
1300
+ /**
1301
+ * @example
1302
+ * ```ts
1303
+ * await kafka.startBatchConsumer(['metrics'], async (envelopes, meta) => {
1304
+ * await db.insertMany(envelopes.map(e => e.payload));
1305
+ * meta.resolveOffset(envelopes.at(-1)!.offset);
1306
+ * }, { autoCommit: false });
1307
+ * ```
1308
+ */
657
1309
  startBatchConsumer<K extends Array<keyof T>>(topics: K, handleBatch: (envelopes: EventEnvelope<T[K[number]]>[], meta: BatchMeta) => Promise<void>, options?: ConsumerOptions<T>): Promise<ConsumerHandle>;
658
1310
  startBatchConsumer<D extends TopicDescriptor<string & keyof T, T[string & keyof T]>>(topics: D[], handleBatch: (envelopes: EventEnvelope<D["__type"]>[], meta: BatchMeta) => Promise<void>, options?: ConsumerOptions<T>): Promise<ConsumerHandle>;
659
1311
  /**
@@ -662,12 +1314,101 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
662
1314
  * Incompatible with `retryTopics: true`.
663
1315
  */
664
1316
  startBatchConsumer(topics: (string | RegExp)[], handleBatch: (envelopes: EventEnvelope<T[keyof T]>[], meta: BatchMeta) => Promise<void>, options?: ConsumerOptions<T>): Promise<ConsumerHandle>;
1317
+ /**
1318
+ * Subscribe to a topic and accumulate messages into a window, flushing the handler
1319
+ * when **either** `maxMessages` messages have accumulated **or** `maxMs` milliseconds
1320
+ * have elapsed since the first message in the current window — whichever fires first.
1321
+ *
1322
+ * This is semantically different from `startBatchConsumer`, which delivers
1323
+ * broker-sized batches of unpredictable size. A window consumer gives you control
1324
+ * over both the size and the latency of each batch — useful for micro-batching
1325
+ * writes to a database, aggregating events before processing, or rate-limiting a
1326
+ * downstream API.
1327
+ *
1328
+ * On `handle.stop()`, any messages remaining in the buffer are flushed before the
1329
+ * consumer disconnects, so no messages are lost on clean shutdown.
1330
+ *
1331
+ * @param topic Topic to consume.
1332
+ * @param handler Called with each flushed window and a `WindowMeta` describing the trigger.
1333
+ * @param options Window size/timeout and standard consumer options.
1334
+ *
1335
+ * @example
1336
+ * ```ts
1337
+ * await kafka.startWindowConsumer('events', async (batch, meta) => {
1338
+ * console.log(`Flushing ${batch.length} events (trigger: ${meta.trigger})`);
1339
+ * await db.insertMany(batch.map(e => e.payload));
1340
+ * }, { maxMessages: 100, maxMs: 5_000 });
1341
+ * ```
1342
+ */
1343
+ startWindowConsumer<K extends keyof T & string>(topic: K, handler: (envelopes: EventEnvelope<T[K]>[], meta: WindowMeta) => Promise<void>, options: WindowConsumerOptions<T>): Promise<ConsumerHandle>;
1344
+ /**
1345
+ * Subscribe to one or more topics and dispatch each message to a handler based
1346
+ * on the value of a specific Kafka header. Eliminates boilerplate `if/switch`
1347
+ * statements inside a catch-all handler when one topic carries multiple event types.
1348
+ *
1349
+ * Messages whose header value does not match any route key are forwarded to `fallback`
1350
+ * if provided, or silently skipped otherwise.
1351
+ *
1352
+ * Accepts the same `ConsumerOptions` as `startConsumer` — retry, DLQ, deduplication,
1353
+ * circuit breaker, interceptors, etc. all apply to every route.
1354
+ *
1355
+ * @param topics Array of topic keys or `RegExp` patterns.
1356
+ * @param routing Header name, route map, and optional fallback handler.
1357
+ * @param options Standard consumer options.
1358
+ * @returns A handle with `{ groupId, stop() }`.
1359
+ *
1360
+ * @example
1361
+ * ```ts
1362
+ * await kafka.startRoutedConsumer(['domain.events'], {
1363
+ * header: 'x-event-type',
1364
+ * routes: {
1365
+ * 'order.created': async (e) => handleOrderCreated(e.payload),
1366
+ * 'order.cancelled': async (e) => handleOrderCancelled(e.payload),
1367
+ * },
1368
+ * fallback: async (e) => logger.warn('Unknown event type', e.headers),
1369
+ * });
1370
+ * ```
1371
+ */
1372
+ startRoutedConsumer<K extends Array<keyof T>>(topics: K, routing: RoutingOptions<T[K[number]]>, options?: ConsumerOptions<T>): Promise<ConsumerHandle>;
1373
+ /**
1374
+ * Subscribe to topics and consume messages with **exactly-once semantics** for
1375
+ * read-process-write pipelines.
1376
+ *
1377
+ * Each message is processed inside a Kafka transaction: the handler receives a
1378
+ * `TransactionalHandlerContext` with `send` / `sendBatch` methods that stage
1379
+ * outgoing messages inside the transaction. When the handler resolves, the source
1380
+ * offset commit and all staged sends are committed atomically. A handler error
1381
+ * aborts the transaction and the source message is redelivered — no sends become
1382
+ * visible to downstream consumers.
1383
+ *
1384
+ * Incompatible with `retryTopics: true` — throws at startup if set.
1385
+ * `autoCommit` is always `false` (managed by the transaction).
1386
+ *
1387
+ * @param topics Array of topic keys.
1388
+ * @param handler Called for each message with the decoded envelope and a transaction context.
1389
+ * @param options Standard consumer options (`retry`, `dlq`, `deduplication`, etc.).
1390
+ * @returns A handle with `{ groupId, stop() }`.
1391
+ *
1392
+ * @example
1393
+ * ```ts
1394
+ * await kafka.startTransactionalConsumer(['orders.created'], async (envelope, tx) => {
1395
+ * await tx.send('inventory.reserved', { orderId: envelope.payload.orderId, qty: 1 });
1396
+ * });
1397
+ * ```
1398
+ */
1399
+ startTransactionalConsumer<K extends Array<keyof T>>(topics: K, handler: (envelope: EventEnvelope<T[K[number]]>, tx: TransactionalHandlerContext<T>) => Promise<void>, options?: ConsumerOptions<T>): Promise<ConsumerHandle>;
665
1400
  /**
666
1401
  * Stop consumer(s).
667
1402
  * - `stopConsumer(groupId)` — disconnect and remove the consumer for a specific group.
668
1403
  * - `stopConsumer()` — disconnect and remove all consumers.
669
1404
  */
670
1405
  stopConsumer(groupId?: string): Promise<void>;
1406
+ /**
1407
+ * @example
1408
+ * ```ts
1409
+ * await kafka.sendMessage('orders.created', { orderId: '123', amount: 99 });
1410
+ * ```
1411
+ */
671
1412
  sendMessage<K extends keyof T>(topic: K, message: T[K], options?: SendOptions): Promise<void>;
672
1413
  /**
673
1414
  * Send a null-value (tombstone) message to a topic.
@@ -681,10 +1422,39 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
681
1422
  * @param topic Topic name or descriptor.
682
1423
  * @param key Partition key identifying the record to tombstone.
683
1424
  * @param headers Optional custom Kafka headers.
1425
+ *
1426
+ * @example
1427
+ * ```ts
1428
+ * await kafka.sendTombstone('users.state', 'user-42');
1429
+ * ```
684
1430
  */
685
1431
  sendTombstone(topic: string, key: string, headers?: MessageHeaders): Promise<void>;
1432
+ /**
1433
+ * @example
1434
+ * ```ts
1435
+ * await kafka.sendBatch('orders.created', [
1436
+ * { value: { orderId: '1', amount: 10 }, key: 'order-1' },
1437
+ * { value: { orderId: '2', amount: 20 }, key: 'order-2' },
1438
+ * ]);
1439
+ * ```
1440
+ */
686
1441
  sendBatch<K extends keyof T>(topic: K, messages: Array<BatchMessageItem<T[K]>>, options?: BatchSendOptions): Promise<void>;
1442
+ /**
1443
+ * @example
1444
+ * ```ts
1445
+ * await kafka.transaction(async (tx) => {
1446
+ * await tx.send('orders.created', { orderId: '123', amount: 99 });
1447
+ * await tx.send('inventory.reserved', { itemId: 'a', qty: 1 });
1448
+ * });
1449
+ * ```
1450
+ */
687
1451
  transaction(fn: (ctx: TransactionContext<T>) => Promise<void>): Promise<void>;
1452
+ /**
1453
+ * @example
1454
+ * ```ts
1455
+ * const id = kafka.getClientId(); // e.g. 'my-service'
1456
+ * ```
1457
+ */
688
1458
  getClientId(): ClientId;
689
1459
  /**
690
1460
  * Return a snapshot of internal event counters (retry / DLQ / dedup).
@@ -693,12 +1463,26 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
693
1463
  * if no events have been observed for that topic yet.
694
1464
  *
695
1465
  * Counters accumulate since client creation or the last `resetMetrics()` call.
1466
+ *
1467
+ * @example
1468
+ * ```ts
1469
+ * const { processedCount, dlqCount, retryCount } = kafka.getMetrics();
1470
+ * console.log(`Processed: ${processedCount}, DLQ: ${dlqCount}`);
1471
+ *
1472
+ * const topicMetrics = kafka.getMetrics('orders.created');
1473
+ * ```
696
1474
  */
697
1475
  getMetrics(topic?: string): Readonly<KafkaMetrics>;
698
1476
  /**
699
1477
  * Reset internal event counters to zero.
700
1478
  * - `resetMetrics()` — reset all topics.
701
1479
  * - `resetMetrics(topic)` — reset a single topic only.
1480
+ *
1481
+ * @example
1482
+ * ```ts
1483
+ * kafka.resetMetrics(); // reset all
1484
+ * kafka.resetMetrics('orders.created'); // reset one topic
1485
+ * ```
702
1486
  */
703
1487
  resetMetrics(topic?: string): void;
704
1488
  /**
@@ -709,6 +1493,18 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
709
1493
  * itself is not modified — messages remain there after replay.
710
1494
  *
711
1495
  * @returns `{ replayed, skipped }` — counts of re-published vs skipped messages.
1496
+ *
1497
+ * @example
1498
+ * ```ts
1499
+ * const { replayed, skipped } = await kafka.replayDlq('orders.created');
1500
+ * console.log(`Replayed ${replayed}, skipped ${skipped}`);
1501
+ *
1502
+ * // dry-run with filter:
1503
+ * await kafka.replayDlq('orders.created', {
1504
+ * dryRun: true,
1505
+ * filter: (headers) => headers['x-dlq-reason'] === 'handler-error',
1506
+ * });
1507
+ * ```
712
1508
  */
713
1509
  replayDlq(topic: string, options?: DlqReplayOptions): Promise<{
714
1510
  replayed: number;
@@ -725,6 +1521,12 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
725
1521
  * @param topic Topic to reset.
726
1522
  * @param position `'earliest'` seeks to the first available offset; `'latest'`
727
1523
  * seeks past the last message (consumer will only see new messages).
1524
+ *
1525
+ * @example
1526
+ * ```ts
1527
+ * await kafka.stopConsumer('billing-service');
1528
+ * await kafka.resetOffsets('billing-service', 'orders.created', 'earliest');
1529
+ * ```
728
1530
  */
729
1531
  resetOffsets(groupId: string | undefined, topic: string, position: "earliest" | "latest"): Promise<void>;
730
1532
  /**
@@ -736,6 +1538,14 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
736
1538
  *
737
1539
  * @param groupId Consumer group to seek. Defaults to the client's default groupId.
738
1540
  * @param assignments Array of `{ topic, partition, offset }` tuples.
1541
+ *
1542
+ * @example
1543
+ * ```ts
1544
+ * await kafka.seekToOffset('billing-service', [
1545
+ * { topic: 'orders.created', partition: 0, offset: '1000' },
1546
+ * { topic: 'orders.created', partition: 1, offset: '500' },
1547
+ * ]);
1548
+ * ```
739
1549
  */
740
1550
  seekToOffset(groupId: string | undefined, assignments: Array<{
741
1551
  topic: string;
@@ -752,6 +1562,14 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
752
1562
  *
753
1563
  * @param groupId Consumer group to seek. Defaults to the client's default groupId.
754
1564
  * @param assignments Array of `{ topic, partition, timestamp }` tuples (Unix ms).
1565
+ *
1566
+ * @example
1567
+ * ```ts
1568
+ * const midnight = new Date('2025-01-01').getTime();
1569
+ * await kafka.seekToTimestamp('billing-service', [
1570
+ * { topic: 'orders.created', partition: 0, timestamp: midnight },
1571
+ * ]);
1572
+ * ```
755
1573
  */
756
1574
  seekToTimestamp(groupId: string | undefined, assignments: Array<{
757
1575
  topic: string;
@@ -766,12 +1584,84 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
766
1584
  * @param topic Topic name.
767
1585
  * @param partition Partition index.
768
1586
  * @param groupId Consumer group. Defaults to the client's default groupId.
1587
+ *
1588
+ * @example
1589
+ * ```ts
1590
+ * const state = kafka.getCircuitState('orders.created', 0);
1591
+ * if (state?.status === 'open') console.warn('Circuit open!', state.failures);
1592
+ * ```
769
1593
  */
770
1594
  getCircuitState(topic: string, partition: number, groupId?: string): {
771
1595
  status: "closed" | "open" | "half-open";
772
1596
  failures: number;
773
1597
  windowSize: number;
774
1598
  } | undefined;
1599
+ /**
1600
+ * Read a compacted topic from the beginning to its current high-watermark and
1601
+ * return a `Map<key, EventEnvelope<T>>` with the **latest** value per key.
1602
+ *
1603
+ * Tombstone records (null-value messages) remove the key from the map —
1604
+ * consistent with log-compaction semantics.
1605
+ *
1606
+ * Useful for bootstrapping in-memory state at service startup without an external cache:
1607
+ * ```ts
1608
+ * const orders = await kafka.readSnapshot('orders.state');
1609
+ * // orders.get('order-123') → EventEnvelope with the latest payload for that key
1610
+ * ```
1611
+ *
1612
+ * @param topic Topic to read. Must be a log-compacted topic (or any topic you want a key → latest-value index for).
1613
+ * @param options Optional schema validation and tombstone callback.
1614
+ * @returns Map keyed by the Kafka message key string; value is the last seen `EventEnvelope`.
1615
+ */
1616
+ readSnapshot<K extends keyof T & string>(topic: K, options?: ReadSnapshotOptions): Promise<Map<string, EventEnvelope<T[K]>>>;
1617
+ /**
1618
+ * Snapshot the current committed offsets of a consumer group into a Kafka topic.
1619
+ *
1620
+ * Each call appends a new checkpoint record keyed by `groupId`. The checkpoint topic
1621
+ * acts as an append-only audit log — use a non-compacted topic to retain history.
1622
+ * Use `restoreFromCheckpoint` to rewind the group to any saved point in time.
1623
+ *
1624
+ * Requires `connectProducer()` to have been called.
1625
+ *
1626
+ * @param groupId Consumer group whose offsets to checkpoint. Defaults to the client's default group.
1627
+ * @param checkpointTopic Topic where checkpoint records are written.
1628
+ * @returns Summary of the saved checkpoint.
1629
+ *
1630
+ * @example
1631
+ * ```ts
1632
+ * const result = await kafka.checkpointOffsets(undefined, 'checkpoints');
1633
+ * console.log(`Saved ${result.partitionCount} offsets at ${result.savedAt}`);
1634
+ * ```
1635
+ */
1636
+ checkpointOffsets(groupId: string | undefined, checkpointTopic: string): Promise<CheckpointResult>;
1637
+ /**
1638
+ * Restore a consumer group's committed offsets from the nearest checkpoint stored in `checkpointTopic`.
1639
+ *
1640
+ * Reads all checkpoint records for the group, selects the newest checkpoint whose `savedAt`
1641
+ * timestamp is ≤ `options.timestamp` (or the latest checkpoint if no timestamp is given),
1642
+ * then calls `admin.setOffsets` for every topic-partition in that checkpoint.
1643
+ *
1644
+ * **The consumer group must be stopped before calling this method** — throws if any consumer
1645
+ * in the group is currently running.
1646
+ *
1647
+ * @param groupId Consumer group to reposition. Defaults to the client's default group.
1648
+ * @param checkpointTopic Topic where checkpoints were written by `checkpointOffsets`.
1649
+ * @param options.timestamp Target Unix ms. Omit to restore the latest checkpoint.
1650
+ * @throws If no checkpoint exists for the group, or if the group is still running.
1651
+ *
1652
+ * @example
1653
+ * ```ts
1654
+ * await kafka.stopConsumer('billing-service');
1655
+ * const result = await kafka.restoreFromCheckpoint(undefined, 'checkpoints');
1656
+ * console.log(`Restored from checkpoint saved ${result.checkpointAge}ms ago`);
1657
+ *
1658
+ * // restore to the state before a specific deployment:
1659
+ * await kafka.restoreFromCheckpoint(undefined, 'checkpoints', {
1660
+ * timestamp: new Date('2025-06-01T00:00:00Z').getTime(),
1661
+ * });
1662
+ * ```
1663
+ */
1664
+ restoreFromCheckpoint(groupId: string | undefined, checkpointTopic: string, options?: RestoreCheckpointOptions): Promise<CheckpointRestoreResult>;
775
1665
  /**
776
1666
  * Consume messages as an async iterator. Useful for scripts, migrations, and
777
1667
  * one-off processing where the full `startConsumer` lifecycle is unnecessary.
@@ -799,6 +1689,13 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
799
1689
  *
800
1690
  * @param groupId Consumer group to pause. Defaults to the client's default groupId.
801
1691
  * @param assignments Topic-partition pairs to pause.
1692
+ *
1693
+ * @example
1694
+ * ```ts
1695
+ * kafka.pauseConsumer(undefined, [{ topic: 'orders.created', partitions: [0, 1] }]);
1696
+ * // ... do some backpressure work ...
1697
+ * kafka.resumeConsumer(undefined, [{ topic: 'orders.created', partitions: [0, 1] }]);
1698
+ * ```
802
1699
  */
803
1700
  pauseConsumer(groupId: string | undefined, assignments: Array<{
804
1701
  topic: string;
@@ -809,6 +1706,11 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
809
1706
  *
810
1707
  * @param groupId Consumer group to resume. Defaults to the client's default groupId.
811
1708
  * @param assignments Topic-partition pairs to resume.
1709
+ *
1710
+ * @example
1711
+ * ```ts
1712
+ * kafka.resumeConsumer(undefined, [{ topic: 'orders.created', partitions: [0, 1] }]);
1713
+ * ```
812
1714
  */
813
1715
  resumeConsumer(groupId: string | undefined, assignments: Array<{
814
1716
  topic: string;
@@ -817,12 +1719,22 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
817
1719
  /**
818
1720
  * Drain in-flight handlers, then disconnect all producers, consumers, and admin.
819
1721
  * @param drainTimeoutMs Max ms to wait for in-flight handlers (default 30 000).
1722
+ *
1723
+ * @example
1724
+ * ```ts
1725
+ * await kafka.disconnect();
1726
+ * ```
820
1727
  */
821
1728
  disconnect(drainTimeoutMs?: number): Promise<void>;
822
1729
  /**
823
1730
  * Register SIGTERM / SIGINT signal handlers that drain in-flight messages before
824
1731
  * disconnecting. Call once after constructing the client in non-NestJS apps.
825
1732
  * NestJS apps get drain automatically via `onModuleDestroy` → `disconnect()`.
1733
+ *
1734
+ * @example
1735
+ * ```ts
1736
+ * kafka.enableGracefulShutdown(['SIGTERM', 'SIGINT'], 30_000);
1737
+ * ```
826
1738
  */
827
1739
  enableGracefulShutdown(signals?: NodeJS.Signals[], drainTimeoutMs?: number): void;
828
1740
  }
@@ -831,6 +1743,22 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
831
1743
  * Compatible with NestJS Logger, console, winston, pino, or any custom logger.
832
1744
  *
833
1745
  * `debug` is optional — omit it to suppress debug output in production.
1746
+ *
1747
+ * @example
1748
+ * ```ts
1749
+ * // Pass a NestJS logger:
1750
+ * const kafka = new KafkaClient(config, groupId, { logger: this.logger });
1751
+ *
1752
+ * // Or a minimal pino wrapper:
1753
+ * const kafka = new KafkaClient(config, groupId, {
1754
+ * logger: {
1755
+ * log: (msg) => pino.info(msg),
1756
+ * warn: (msg, ...a) => pino.warn(msg, ...a),
1757
+ * error: (msg, ...a) => pino.error(msg, ...a),
1758
+ * debug: (msg, ...a) => pino.debug(msg, ...a),
1759
+ * },
1760
+ * });
1761
+ * ```
834
1762
  */
835
1763
  interface KafkaLogger {
836
1764
  log(message: string): void;
@@ -841,6 +1769,15 @@ interface KafkaLogger {
841
1769
  /**
842
1770
  * Context passed to `onTtlExpired` when a message is dropped because it
843
1771
  * exceeded `messageTtlMs` and `dlq` is not enabled.
1772
+ *
1773
+ * @example
1774
+ * ```ts
1775
+ * const kafka = new KafkaClient(config, groupId, {
1776
+ * onTtlExpired: ({ topic, ageMs, messageTtlMs }) => {
1777
+ * console.warn(`Message on ${topic} expired: age=${ageMs}ms, ttl=${messageTtlMs}ms`);
1778
+ * },
1779
+ * });
1780
+ * ```
844
1781
  */
845
1782
  interface TtlExpiredContext {
846
1783
  /** Topic the message was consumed from. */
@@ -855,6 +1792,15 @@ interface TtlExpiredContext {
855
1792
  /**
856
1793
  * Context passed to `onMessageLost` when a message is silently dropped
857
1794
  * (handler threw and `dlq` is not enabled).
1795
+ *
1796
+ * @example
1797
+ * ```ts
1798
+ * const kafka = new KafkaClient(config, groupId, {
1799
+ * onMessageLost: ({ topic, error, attempt }) => {
1800
+ * alerting.fire('kafka.message-lost', { topic, error: error.message, attempt });
1801
+ * },
1802
+ * });
1803
+ * ```
858
1804
  */
859
1805
  interface MessageLostContext {
860
1806
  /** Topic the message was consumed from. */
@@ -866,7 +1812,20 @@ interface MessageLostContext {
866
1812
  /** Original Kafka message headers (correlationId, traceparent, etc.). */
867
1813
  headers: MessageHeaders;
868
1814
  }
869
- /** Options for `KafkaClient` constructor. */
1815
+ /**
1816
+ * Options for `KafkaClient` constructor.
1817
+ *
1818
+ * @example
1819
+ * ```ts
1820
+ * const kafka = new KafkaClient(kafkaConfig, 'my-service', {
1821
+ * transactionalId: `my-service-tx-${replicaIndex}`,
1822
+ * lagThrottle: { maxLag: 10_000, pollIntervalMs: 3_000 },
1823
+ * clockRecovery: { topics: ['orders.created'] },
1824
+ * onMessageLost: (ctx) => alerting.fire('kafka.message-lost', ctx),
1825
+ * instrumentation: [otelInstrumentation()],
1826
+ * });
1827
+ * ```
1828
+ */
870
1829
  interface KafkaClientOptions {
871
1830
  /** Auto-create topics via admin before the first `sendMessage`, `sendBatch`, or `transaction` for each topic. Useful for development — not recommended in production. */
872
1831
  autoCreateTopics?: boolean;
@@ -930,8 +1889,59 @@ interface KafkaClientOptions {
930
1889
  /** Topic names to scan for the highest Lamport clock. */
931
1890
  topics: string[];
932
1891
  };
1892
+ /**
1893
+ * Delay `sendMessage` / `sendBatch` / `sendTombstone` when the observed lag of a
1894
+ * consumer group exceeds `maxLag`. Resumes immediately when lag drops below the threshold.
1895
+ *
1896
+ * Lag is polled via `getConsumerLag()` every `pollIntervalMs` in the background;
1897
+ * no admin call is made on each individual send.
1898
+ *
1899
+ * When `maxWaitMs` is exceeded the send is unblocked with a warning — this is
1900
+ * best-effort throttling, not hard back-pressure.
1901
+ *
1902
+ * Requires `connectProducer()` to have been called to start the polling loop.
1903
+ */
1904
+ lagThrottle?: {
1905
+ /** Consumer group whose lag is monitored. Defaults to the client's default group. */
1906
+ groupId?: string;
1907
+ /** Lag threshold (number of messages) above which sends are delayed. */
1908
+ maxLag: number;
1909
+ /** How often to poll `getConsumerLag()`. Default: `5000` ms. */
1910
+ pollIntervalMs?: number;
1911
+ /**
1912
+ * Maximum time (ms) a send will wait while throttled before proceeding anyway.
1913
+ * Default: `30_000` ms.
1914
+ */
1915
+ maxWaitMs?: number;
1916
+ };
1917
+ /**
1918
+ * Custom transport implementation.
1919
+ *
1920
+ * By default `KafkaClient` uses `ConfluentTransport` which wraps
1921
+ * `@confluentinc/kafka-javascript` (librdkafka). Inject a different
1922
+ * `KafkaTransport` to target an alternative broker library, or to supply
1923
+ * a deterministic fake in unit tests without mocking the confluentinc module.
1924
+ *
1925
+ * @example
1926
+ * ```ts
1927
+ * // In tests — no jest.mock() needed
1928
+ * const kafka = new KafkaClient('svc', 'grp', [], {
1929
+ * transport: new FakeTransport(),
1930
+ * });
1931
+ * ```
1932
+ */
1933
+ transport?: KafkaTransport;
933
1934
  }
934
- /** Options for consumer subscribe retry when topic doesn't exist yet. */
1935
+ /**
1936
+ * Options for consumer subscribe retry when topic doesn't exist yet.
1937
+ *
1938
+ * @example
1939
+ * ```ts
1940
+ * await kafka.startConsumer(['orders.created'], handler, {
1941
+ * subscribeRetry: { retries: 10, backoffMs: 2_000 },
1942
+ * });
1943
+ * ```
1944
+ */
935
1945
  interface SubscribeRetryOptions {
936
1946
  /** Maximum number of subscribe attempts. Default: `5`. */
937
1947
  retries?: number;
@@ -939,4 +1949,4 @@ interface SubscribeRetryOptions {
939
1949
  backoffMs?: number;
940
1950
  }
941
1951
 
942
- export { type SubscribeRetryOptions as A, type BatchMessageItem as B, type ClientId as C, type DeduplicationOptions as D, type EnvelopeHeaderOptions as E, type TTopicMessageMap as F, type GroupId as G, HEADER_CORRELATION_ID as H, type IKafkaClient as I, type TopicDescription as J, type KafkaInstrumentation as K, type TopicPartitionInfo as L, type MessageHeaders as M, type TopicsFrom as N, type TransactionContext as O, type TtlExpiredContext as P, buildEnvelopeHeaders as Q, type RetryOptions as R, type SchemaLike as S, type TopicMapConstraint as T, decodeHeaders as U, extractEnvelope as V, getEnvelopeContext as W, runWithEnvelopeContext as X, topic as Y, type KafkaClientOptions as a, type ConsumerOptions as b, type TopicDescriptor as c, type KafkaHealthResult as d, type BatchMeta as e, type BatchSendOptions as f, type BeforeConsumeResult as g, type CircuitBreakerOptions as h, type CompressionType as i, type ConsumerGroupSummary as j, type ConsumerHandle as k, type ConsumerInterceptor as l, type DlqReason as m, type DlqReplayOptions as n, type EventEnvelope as o, HEADER_EVENT_ID as p, HEADER_LAMPORT_CLOCK as q, HEADER_SCHEMA_VERSION as r, HEADER_TIMESTAMP as s, HEADER_TRACEPARENT as t, type InferSchema as u, type KafkaLogger as v, type KafkaMetrics as w, type MessageLostContext as x, type SchemaParseContext as y, type SendOptions as z };
1952
+ export { type KafkaMetrics as $, type CheckpointRestoreResult as A, type BatchMessageItem as B, type ClientId as C, type CheckpointResult as D, type CircuitBreakerOptions as E, type CompressionType as F, type GroupId as G, type ConsumerGroupSummary as H, type IKafkaClient as I, type ConsumerHandle as J, type KafkaInstrumentation as K, type ConsumerInterceptor as L, type DeduplicationOptions as M, type DlqReason as N, type DlqReplayOptions as O, type EnvelopeHeaderOptions as P, type EventEnvelope as Q, HEADER_CORRELATION_ID as R, type SchemaLike as S, type TopicMapConstraint as T, HEADER_EVENT_ID as U, HEADER_LAMPORT_CLOCK as V, HEADER_SCHEMA_VERSION as W, HEADER_TIMESTAMP as X, HEADER_TRACEPARENT as Y, type InferSchema as Z, type KafkaLogger as _, type IAdmin as a, type MessageHeaders as a0, type MessageLostContext as a1, type ReadSnapshotOptions as a2, type RestoreCheckpointOptions as a3, type RetryOptions as a4, type RoutingOptions as a5, type SchemaParseContext as a6, type SendOptions as a7, type SubscribeRetryOptions as a8, type TTopicMessageMap as a9, type TopicDescription as aa, type TopicPartitionInfo as ab, type TopicsFrom as ac, type TransactionContext as ad, type TransactionalHandlerContext as ae, type TtlExpiredContext as af, type WindowConsumerOptions as ag, type WindowMeta as ah, buildEnvelopeHeaders as ai, decodeHeaders as aj, extractEnvelope as ak, getEnvelopeContext as al, runWithEnvelopeContext as am, topic as an, type IPartitionWatermarks as b, type IGroupTopicOffsets as c, type IPartitionOffset as d, type IGroupDescription as e, type ITopicMetadata as f, type IConsumer as g, type IConsumerCreationOptions as h, type IConsumerRunConfig as i, type ITopicPartitions as j, type ITopicPartitionOffset as k, type ITopicPartition as l, type IMessage as m, type IProducer as n, type IProducerRecord as o, type ITransaction as p, type IProducerCreationOptions as q, type KafkaTransport as r, type KafkaClientOptions as s, type ConsumerOptions as t, type TopicDescriptor as u, type KafkaHealthResult as v, type BatchMeta as w, type BatchSendOptions as x, type BeforeConsumeResult as y, type CheckpointEntry as z };