@drarzter/kafka-client 0.7.4 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +282 -5
- package/dist/{chunk-BVWRZTMD.mjs → chunk-Z2DOJQRI.mjs} +2948 -2134
- package/dist/chunk-Z2DOJQRI.mjs.map +1 -0
- package/dist/core.d.mts +100 -279
- package/dist/core.d.ts +100 -279
- package/dist/core.js +2947 -2133
- package/dist/core.js.map +1 -1
- package/dist/core.mjs +1 -1
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +2945 -2131
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1 -1
- package/dist/otel.d.mts +1 -1
- package/dist/otel.d.ts +1 -1
- package/dist/testing.d.mts +215 -2
- package/dist/testing.d.ts +215 -2
- package/dist/testing.js +298 -2
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +293 -2
- package/dist/testing.mjs.map +1 -1
- package/dist/{types-Db7qSbZP.d.mts → types-4XNxkici.d.mts} +1024 -14
- package/dist/{types-Db7qSbZP.d.ts → types-4XNxkici.d.ts} +1024 -14
- package/package.json +1 -1
- package/dist/chunk-BVWRZTMD.mjs.map +0 -1
|
@@ -1,3 +1,212 @@
|
|
|
1
|
+
/** A topic-partition pair. */
|
|
2
|
+
type ITopicPartition = {
|
|
3
|
+
topic: string;
|
|
4
|
+
partition: number;
|
|
5
|
+
};
|
|
6
|
+
/** A topic-partition pair with an absolute offset string. */
|
|
7
|
+
type ITopicPartitionOffset = {
|
|
8
|
+
topic: string;
|
|
9
|
+
partition: number;
|
|
10
|
+
offset: string;
|
|
11
|
+
};
|
|
12
|
+
/** Pause / resume assignment shape: one topic + its partition list. */
|
|
13
|
+
type ITopicPartitions = {
|
|
14
|
+
topic: string;
|
|
15
|
+
partitions: number[];
|
|
16
|
+
};
|
|
17
|
+
/** A single message in a produce request. */
|
|
18
|
+
type IProducerMessage = {
|
|
19
|
+
value: string | null;
|
|
20
|
+
key?: string;
|
|
21
|
+
headers?: Record<string, string | Buffer | string[]>;
|
|
22
|
+
};
|
|
23
|
+
/** Produce request payload for one topic. */
|
|
24
|
+
type IProducerRecord = {
|
|
25
|
+
topic: string;
|
|
26
|
+
messages: IProducerMessage[];
|
|
27
|
+
};
|
|
28
|
+
/** Options for creating a producer. */
|
|
29
|
+
type IProducerCreationOptions = {
|
|
30
|
+
/** When set, the producer uses idempotent + exactly-once semantics. */
|
|
31
|
+
transactionalId?: string;
|
|
32
|
+
/** Enable idempotent writes (required for `transactionalId`). */
|
|
33
|
+
idempotent?: boolean;
|
|
34
|
+
};
|
|
35
|
+
/** An open Kafka transaction. */
|
|
36
|
+
interface ITransaction {
|
|
37
|
+
send(record: IProducerRecord): Promise<void>;
|
|
38
|
+
/**
|
|
39
|
+
* Atomically commit offsets for `consumer` as part of this transaction.
|
|
40
|
+
* The `consumer` parameter must be the `IConsumer` whose offsets are being committed.
|
|
41
|
+
*/
|
|
42
|
+
sendOffsets(options: {
|
|
43
|
+
consumer: IConsumer;
|
|
44
|
+
topics: Array<{
|
|
45
|
+
topic: string;
|
|
46
|
+
partitions: Array<{
|
|
47
|
+
partition: number;
|
|
48
|
+
offset: string;
|
|
49
|
+
}>;
|
|
50
|
+
}>;
|
|
51
|
+
}): Promise<void>;
|
|
52
|
+
commit(): Promise<void>;
|
|
53
|
+
abort(): Promise<void>;
|
|
54
|
+
}
|
|
55
|
+
/** A Kafka producer. */
|
|
56
|
+
interface IProducer {
|
|
57
|
+
connect(): Promise<void>;
|
|
58
|
+
disconnect(): Promise<void>;
|
|
59
|
+
send(record: IProducerRecord): Promise<void>;
|
|
60
|
+
transaction(): Promise<ITransaction>;
|
|
61
|
+
}
|
|
62
|
+
/** A single message in an `eachMessage` callback. */
|
|
63
|
+
type IMessage = {
|
|
64
|
+
value: Buffer | null;
|
|
65
|
+
/** Header map as returned by librdkafka — values may be arrays. */
|
|
66
|
+
headers: Record<string, any>;
|
|
67
|
+
offset: string;
|
|
68
|
+
key?: Buffer | null;
|
|
69
|
+
};
|
|
70
|
+
/** Payload passed to the `eachMessage` handler. */
|
|
71
|
+
type IEachMessagePayload = {
|
|
72
|
+
topic: string;
|
|
73
|
+
partition: number;
|
|
74
|
+
message: IMessage;
|
|
75
|
+
};
|
|
76
|
+
/** A batch of messages from one topic-partition. */
|
|
77
|
+
type IMessageBatch = {
|
|
78
|
+
topic: string;
|
|
79
|
+
partition: number;
|
|
80
|
+
messages: IMessage[];
|
|
81
|
+
highWatermark?: string;
|
|
82
|
+
};
|
|
83
|
+
/** Payload passed to the `eachBatch` handler. */
|
|
84
|
+
type IEachBatchPayload = {
|
|
85
|
+
batch: IMessageBatch;
|
|
86
|
+
/** Send a heartbeat to the broker to prevent session timeout. */
|
|
87
|
+
heartbeat: () => Promise<void>;
|
|
88
|
+
/** Mark `offset` as processed (without committing). */
|
|
89
|
+
resolveOffset: (offset: string) => void;
|
|
90
|
+
/** Commit if the auto-commit threshold has been reached. */
|
|
91
|
+
commitOffsetsIfNecessary: () => Promise<void>;
|
|
92
|
+
};
|
|
93
|
+
/** Configuration passed to `IConsumer.run()`. */
|
|
94
|
+
type IConsumerRunConfig = {
|
|
95
|
+
eachMessage?: (payload: IEachMessagePayload) => Promise<void>;
|
|
96
|
+
eachBatch?: (payload: IEachBatchPayload) => Promise<void>;
|
|
97
|
+
};
|
|
98
|
+
/** Options for creating a consumer. */
|
|
99
|
+
type IConsumerCreationOptions = {
|
|
100
|
+
groupId: string;
|
|
101
|
+
fromBeginning?: boolean;
|
|
102
|
+
autoCommit?: boolean;
|
|
103
|
+
partitionAssigner?: "cooperative-sticky" | "roundrobin" | "range";
|
|
104
|
+
/** Fired on every partition assign/revoke. */
|
|
105
|
+
onRebalance?: (type: "assign" | "revoke", assignments: ITopicPartition[]) => void;
|
|
106
|
+
};
|
|
107
|
+
/** A Kafka consumer. */
|
|
108
|
+
interface IConsumer {
|
|
109
|
+
connect(): Promise<void>;
|
|
110
|
+
disconnect(): Promise<void>;
|
|
111
|
+
subscribe(options: {
|
|
112
|
+
topics: (string | RegExp)[];
|
|
113
|
+
}): Promise<void>;
|
|
114
|
+
run(config: IConsumerRunConfig): Promise<void>;
|
|
115
|
+
pause(assignments: ITopicPartitions[]): void;
|
|
116
|
+
resume(assignments: ITopicPartitions[]): void;
|
|
117
|
+
/** Seek a partition to an explicit offset. */
|
|
118
|
+
seek(options: ITopicPartitionOffset): void;
|
|
119
|
+
/** Current partition assignment for this consumer. */
|
|
120
|
+
assignment(): ITopicPartition[];
|
|
121
|
+
commitOffsets(offsets: ITopicPartitionOffset[]): Promise<void>;
|
|
122
|
+
/** Stop processing (alias for disconnect in some usages). */
|
|
123
|
+
stop(): Promise<void>;
|
|
124
|
+
}
|
|
125
|
+
/** Low/current/high watermark offsets for one partition. */
|
|
126
|
+
type IPartitionWatermarks = {
|
|
127
|
+
partition: number;
|
|
128
|
+
low: string;
|
|
129
|
+
high: string;
|
|
130
|
+
};
|
|
131
|
+
/** A partition → offset pair. */
|
|
132
|
+
type IPartitionOffset = {
|
|
133
|
+
partition: number;
|
|
134
|
+
offset: string;
|
|
135
|
+
};
|
|
136
|
+
/** Committed offsets for a group's topic. */
|
|
137
|
+
type IGroupTopicOffsets = {
|
|
138
|
+
topic: string;
|
|
139
|
+
partitions: IPartitionOffset[];
|
|
140
|
+
};
|
|
141
|
+
/** A consumer group descriptor. */
|
|
142
|
+
type IGroupDescription = {
|
|
143
|
+
groupId: string;
|
|
144
|
+
state?: string;
|
|
145
|
+
};
|
|
146
|
+
/** Partition metadata. */
|
|
147
|
+
type IPartitionMetadata = {
|
|
148
|
+
partitionId?: number;
|
|
149
|
+
partition?: number;
|
|
150
|
+
leader?: number;
|
|
151
|
+
replicas?: (number | {
|
|
152
|
+
nodeId: number;
|
|
153
|
+
})[];
|
|
154
|
+
isr?: (number | {
|
|
155
|
+
nodeId: number;
|
|
156
|
+
})[];
|
|
157
|
+
};
|
|
158
|
+
/** Topic metadata. */
|
|
159
|
+
type ITopicMetadata = {
|
|
160
|
+
name: string;
|
|
161
|
+
partitions: IPartitionMetadata[];
|
|
162
|
+
};
|
|
163
|
+
/** A Kafka admin client. */
|
|
164
|
+
interface IAdmin {
|
|
165
|
+
connect(): Promise<void>;
|
|
166
|
+
disconnect(): Promise<void>;
|
|
167
|
+
createTopics(options: {
|
|
168
|
+
topics: Array<{
|
|
169
|
+
topic: string;
|
|
170
|
+
numPartitions: number;
|
|
171
|
+
}>;
|
|
172
|
+
}): Promise<void>;
|
|
173
|
+
fetchTopicOffsets(topic: string): Promise<IPartitionWatermarks[]>;
|
|
174
|
+
fetchTopicOffsetsByTimestamp(topic: string, timestamp: number): Promise<IPartitionOffset[]>;
|
|
175
|
+
fetchOffsets(options: {
|
|
176
|
+
groupId: string;
|
|
177
|
+
}): Promise<IGroupTopicOffsets[]>;
|
|
178
|
+
setOffsets(options: {
|
|
179
|
+
groupId: string;
|
|
180
|
+
topic: string;
|
|
181
|
+
partitions: IPartitionOffset[];
|
|
182
|
+
}): Promise<void>;
|
|
183
|
+
listTopics(): Promise<string[]>;
|
|
184
|
+
listGroups(): Promise<{
|
|
185
|
+
groups: IGroupDescription[];
|
|
186
|
+
}>;
|
|
187
|
+
fetchTopicMetadata(options?: {
|
|
188
|
+
topics?: string[];
|
|
189
|
+
}): Promise<{
|
|
190
|
+
topics: ITopicMetadata[];
|
|
191
|
+
}>;
|
|
192
|
+
deleteGroups(groupIds: string[]): Promise<void>;
|
|
193
|
+
deleteTopicRecords(options: {
|
|
194
|
+
topic: string;
|
|
195
|
+
partitions: IPartitionOffset[];
|
|
196
|
+
}): Promise<void>;
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Factory that creates connected Kafka primitives.
|
|
200
|
+
* The default implementation wraps `@confluentinc/kafka-javascript` via
|
|
201
|
+
* `ConfluentTransport`. Inject a custom transport (e.g. a fake) via
|
|
202
|
+
* `KafkaClientOptions.transport` for testing or alternative broker support.
|
|
203
|
+
*/
|
|
204
|
+
interface KafkaTransport {
|
|
205
|
+
producer(options?: IProducerCreationOptions): IProducer;
|
|
206
|
+
consumer(options: IConsumerCreationOptions): IConsumer;
|
|
207
|
+
admin(): IAdmin;
|
|
208
|
+
}
|
|
209
|
+
|
|
1
210
|
/**
|
|
2
211
|
* Context passed as the second argument to `SchemaLike.parse()`.
|
|
3
212
|
* Enables schema-registry adapters, version-aware migration, and
|
|
@@ -115,6 +324,15 @@ declare const HEADER_LAMPORT_CLOCK = "x-lamport-clock";
|
|
|
115
324
|
*
|
|
116
325
|
* On **consume**, the library extracts those headers and assembles
|
|
117
326
|
* an `EventEnvelope` that is passed to the handler.
|
|
327
|
+
*
|
|
328
|
+
* @example
|
|
329
|
+
* ```ts
|
|
330
|
+
* await kafka.startConsumer(['orders'], async (envelope: EventEnvelope<Order>) => {
|
|
331
|
+
* console.log(envelope.payload.orderId); // typed payload
|
|
332
|
+
* console.log(envelope.correlationId); // auto-propagated
|
|
333
|
+
* console.log(envelope.eventId); // unique message ID
|
|
334
|
+
* });
|
|
335
|
+
* ```
|
|
118
336
|
*/
|
|
119
337
|
interface EventEnvelope<T> {
|
|
120
338
|
/** Deserialized + validated message body. */
|
|
@@ -142,9 +360,26 @@ interface EnvelopeCtx {
|
|
|
142
360
|
correlationId: string;
|
|
143
361
|
traceparent?: string;
|
|
144
362
|
}
|
|
145
|
-
/**
|
|
363
|
+
/**
|
|
364
|
+
* Read the current envelope context (correlationId / traceparent) from ALS.
|
|
365
|
+
* Returns `undefined` outside of a Kafka consumer handler.
|
|
366
|
+
* @example
|
|
367
|
+
* ```ts
|
|
368
|
+
* const ctx = getEnvelopeContext();
|
|
369
|
+
* if (ctx) console.log('correlationId:', ctx.correlationId);
|
|
370
|
+
* ```
|
|
371
|
+
*/
|
|
146
372
|
declare function getEnvelopeContext(): EnvelopeCtx | undefined;
|
|
147
|
-
/**
|
|
373
|
+
/**
|
|
374
|
+
* Execute `fn` inside an envelope context so nested sends inherit correlationId.
|
|
375
|
+
* Automatically called by the consumer pipeline — use this in tests or manual flows.
|
|
376
|
+
* @example
|
|
377
|
+
* ```ts
|
|
378
|
+
* await runWithEnvelopeContext({ correlationId: 'abc-123' }, async () => {
|
|
379
|
+
* await kafka.sendMessage('orders.created', payload); // inherits correlationId
|
|
380
|
+
* });
|
|
381
|
+
* ```
|
|
382
|
+
*/
|
|
148
383
|
declare function runWithEnvelopeContext<R>(ctx: EnvelopeCtx, fn: () => R): R;
|
|
149
384
|
/** Options accepted by `buildEnvelopeHeaders`. */
|
|
150
385
|
interface EnvelopeHeaderOptions {
|
|
@@ -213,7 +448,18 @@ type MessageHeaders = Record<string, string>;
|
|
|
213
448
|
* - `'zstd'` — best ratio, slightly slower
|
|
214
449
|
*/
|
|
215
450
|
type CompressionType = "none" | "gzip" | "snappy" | "lz4" | "zstd";
|
|
216
|
-
/**
|
|
451
|
+
/**
|
|
452
|
+
* Options for sending a single message.
|
|
453
|
+
*
|
|
454
|
+
* @example
|
|
455
|
+
* ```ts
|
|
456
|
+
* await kafka.sendMessage('orders.created', { orderId: '123', amount: 99 }, {
|
|
457
|
+
* key: 'order-123',
|
|
458
|
+
* headers: { 'x-source': 'checkout-service' },
|
|
459
|
+
* compression: 'snappy',
|
|
460
|
+
* });
|
|
461
|
+
* ```
|
|
462
|
+
*/
|
|
217
463
|
interface SendOptions {
|
|
218
464
|
/** Partition key for message routing. */
|
|
219
465
|
key?: string;
|
|
@@ -232,7 +478,17 @@ interface SendOptions {
|
|
|
232
478
|
*/
|
|
233
479
|
compression?: CompressionType;
|
|
234
480
|
}
|
|
235
|
-
/**
|
|
481
|
+
/**
|
|
482
|
+
* Shape of each item in a `sendBatch` call.
|
|
483
|
+
*
|
|
484
|
+
* @example
|
|
485
|
+
* ```ts
|
|
486
|
+
* await kafka.sendBatch('orders.created', [
|
|
487
|
+
* { value: { orderId: '1', amount: 10 }, key: 'order-1' },
|
|
488
|
+
* { value: { orderId: '2', amount: 20 }, key: 'order-2', headers: { 'x-priority': 'high' } },
|
|
489
|
+
* ]);
|
|
490
|
+
* ```
|
|
491
|
+
*/
|
|
236
492
|
interface BatchMessageItem<V> {
|
|
237
493
|
value: V;
|
|
238
494
|
/**
|
|
@@ -247,7 +503,14 @@ interface BatchMessageItem<V> {
|
|
|
247
503
|
schemaVersion?: number;
|
|
248
504
|
eventId?: string;
|
|
249
505
|
}
|
|
250
|
-
/**
|
|
506
|
+
/**
|
|
507
|
+
* Options for a `sendBatch` call (applies to all messages in the batch).
|
|
508
|
+
*
|
|
509
|
+
* @example
|
|
510
|
+
* ```ts
|
|
511
|
+
* await kafka.sendBatch('metrics', messages, { compression: 'zstd' });
|
|
512
|
+
* ```
|
|
513
|
+
*/
|
|
251
514
|
interface BatchSendOptions {
|
|
252
515
|
/**
|
|
253
516
|
* Compression codec for this batch.
|
|
@@ -256,7 +519,19 @@ interface BatchSendOptions {
|
|
|
256
519
|
*/
|
|
257
520
|
compression?: CompressionType;
|
|
258
521
|
}
|
|
259
|
-
/**
|
|
522
|
+
/**
|
|
523
|
+
* Metadata exposed to batch consumer handlers.
|
|
524
|
+
*
|
|
525
|
+
* @example
|
|
526
|
+
* ```ts
|
|
527
|
+
* await kafka.startBatchConsumer(['events'], async (envelopes, meta) => {
|
|
528
|
+
* console.log(`Partition ${meta.partition}, HWM ${meta.highWatermark}`);
|
|
529
|
+
* await db.insertMany(envelopes.map(e => e.payload));
|
|
530
|
+
* meta.resolveOffset(envelopes.at(-1)!.offset);
|
|
531
|
+
* await meta.commitOffsetsIfNecessary();
|
|
532
|
+
* }, { autoCommit: false });
|
|
533
|
+
* ```
|
|
534
|
+
*/
|
|
260
535
|
interface BatchMeta {
|
|
261
536
|
/** Partition number for this batch. */
|
|
262
537
|
partition: number;
|
|
@@ -277,6 +552,14 @@ interface BatchMeta {
|
|
|
277
552
|
/**
|
|
278
553
|
* Options for Lamport Clock-based message deduplication.
|
|
279
554
|
*
|
|
555
|
+
* @example
|
|
556
|
+
* ```ts
|
|
557
|
+
* await kafka.startConsumer(['payments'], handler, {
|
|
558
|
+
* deduplication: { strategy: 'dlq' },
|
|
559
|
+
* dlq: true,
|
|
560
|
+
* });
|
|
561
|
+
* ```
|
|
562
|
+
*
|
|
280
563
|
* The producer stamps every outgoing message with a monotonically increasing
|
|
281
564
|
* `x-lamport-clock` header. The consumer tracks the last processed value per
|
|
282
565
|
* `topic:partition` and skips any message whose clock is not strictly greater
|
|
@@ -311,6 +594,19 @@ interface DeduplicationOptions {
|
|
|
311
594
|
/**
|
|
312
595
|
* Options for the per-partition circuit breaker.
|
|
313
596
|
*
|
|
597
|
+
* @example
|
|
598
|
+
* ```ts
|
|
599
|
+
* await kafka.startConsumer(['payments'], handler, {
|
|
600
|
+
* circuitBreaker: {
|
|
601
|
+
* threshold: 5,
|
|
602
|
+
* recoveryMs: 30_000,
|
|
603
|
+
* windowSize: 20,
|
|
604
|
+
* halfOpenSuccesses: 2,
|
|
605
|
+
* },
|
|
606
|
+
* dlq: true,
|
|
607
|
+
* });
|
|
608
|
+
* ```
|
|
609
|
+
*
|
|
314
610
|
* The circuit breaker tracks recent message outcomes in a **sliding window** and
|
|
315
611
|
* opens (pauses the partition) when too many failures accumulate:
|
|
316
612
|
*
|
|
@@ -343,7 +639,21 @@ interface CircuitBreakerOptions {
|
|
|
343
639
|
*/
|
|
344
640
|
halfOpenSuccesses?: number;
|
|
345
641
|
}
|
|
346
|
-
/**
|
|
642
|
+
/**
|
|
643
|
+
* Options for configuring a Kafka consumer.
|
|
644
|
+
*
|
|
645
|
+
* @example
|
|
646
|
+
* ```ts
|
|
647
|
+
* await kafka.startConsumer(['orders.created'], handler, {
|
|
648
|
+
* groupId: 'billing-service',
|
|
649
|
+
* retry: { maxRetries: 5, backoffMs: 1000 },
|
|
650
|
+
* dlq: true,
|
|
651
|
+
* deduplication: { strategy: 'drop' },
|
|
652
|
+
* circuitBreaker: { threshold: 3, recoveryMs: 60_000 },
|
|
653
|
+
* interceptors: [loggingInterceptor],
|
|
654
|
+
* });
|
|
655
|
+
* ```
|
|
656
|
+
*/
|
|
347
657
|
interface ConsumerOptions<T extends TopicMapConstraint<T> = TTopicMessageMap> {
|
|
348
658
|
/** Override the default consumer group ID from the constructor. */
|
|
349
659
|
groupId?: string;
|
|
@@ -430,8 +740,32 @@ interface ConsumerOptions<T extends TopicMapConstraint<T> = TTopicMessageMap> {
|
|
|
430
740
|
* when both are set. Use this to handle TTL expiry differently per consumer group.
|
|
431
741
|
*/
|
|
432
742
|
onTtlExpired?: (ctx: TtlExpiredContext) => void | Promise<void>;
|
|
743
|
+
/**
|
|
744
|
+
* Called when a message is silently dropped after all retries are exhausted
|
|
745
|
+
* and `dlq` is not enabled.
|
|
746
|
+
*
|
|
747
|
+
* **Per-consumer override**: takes precedence over `KafkaClientOptions.onMessageLost`
|
|
748
|
+
* when both are set. Use this for consumer-specific alerting or dead-message logging.
|
|
749
|
+
*/
|
|
750
|
+
onMessageLost?: (ctx: MessageLostContext) => void | Promise<void>;
|
|
751
|
+
/**
|
|
752
|
+
* Called before each retry attempt (both in-process and retry-topic paths).
|
|
753
|
+
*
|
|
754
|
+
* **Per-consumer override**: fires in addition to any global instrumentation hooks.
|
|
755
|
+
* Use this for per-consumer retry metrics or structured logging.
|
|
756
|
+
*/
|
|
757
|
+
onRetry?: (envelope: EventEnvelope<any>, attempt: number, maxRetries: number) => void | Promise<void>;
|
|
433
758
|
}
|
|
434
|
-
/**
|
|
759
|
+
/**
|
|
760
|
+
* Configuration for consumer retry behavior.
|
|
761
|
+
*
|
|
762
|
+
* @example
|
|
763
|
+
* ```ts
|
|
764
|
+
* await kafka.startConsumer(['orders'], handler, {
|
|
765
|
+
* retry: { maxRetries: 5, backoffMs: 500, maxBackoffMs: 10_000 },
|
|
766
|
+
* });
|
|
767
|
+
* ```
|
|
768
|
+
*/
|
|
435
769
|
interface RetryOptions {
|
|
436
770
|
/** Maximum number of retry attempts before giving up. */
|
|
437
771
|
maxRetries: number;
|
|
@@ -446,6 +780,17 @@ interface RetryOptions {
|
|
|
446
780
|
*
|
|
447
781
|
* Interceptors are per-consumer. For client-wide hooks (e.g. OTel),
|
|
448
782
|
* use `KafkaInstrumentation` instead.
|
|
783
|
+
*
|
|
784
|
+
* @example
|
|
785
|
+
* ```ts
|
|
786
|
+
* const logger: ConsumerInterceptor = {
|
|
787
|
+
* async before(envelope) { console.log('Processing', envelope.eventId); },
|
|
788
|
+
* async after(envelope) { console.log('Done', envelope.eventId); },
|
|
789
|
+
* async onError(envelope, err) { console.error('Failed', err.message); },
|
|
790
|
+
* };
|
|
791
|
+
*
|
|
792
|
+
* await kafka.startConsumer(['orders'], handler, { interceptors: [logger] });
|
|
793
|
+
* ```
|
|
449
794
|
*/
|
|
450
795
|
interface ConsumerInterceptor<T extends TopicMapConstraint<T> = TTopicMessageMap> {
|
|
451
796
|
/** Called before the message handler. */
|
|
@@ -465,6 +810,25 @@ interface ConsumerInterceptor<T extends TopicMapConstraint<T> = TTopicMessageMap
|
|
|
465
810
|
* async context (e.g. `context.with(spanCtx, fn)` for OpenTelemetry). Multiple
|
|
466
811
|
* wraps from different instrumentations are composed in declaration order,
|
|
467
812
|
* so the first instrumentation's wrap is the outermost.
|
|
813
|
+
*
|
|
814
|
+
* @example
|
|
815
|
+
* ```ts
|
|
816
|
+
* // Cleanup-only (legacy form):
|
|
817
|
+
* beforeConsume(envelope) {
|
|
818
|
+
* const timer = startTimer();
|
|
819
|
+
* return () => timer.end();
|
|
820
|
+
* }
|
|
821
|
+
*
|
|
822
|
+
* // Wrap form — run handler inside an OTel span context:
|
|
823
|
+
* beforeConsume(envelope) {
|
|
824
|
+
* const ctx = propagator.extract(ROOT_CONTEXT, envelope.headers);
|
|
825
|
+
* const span = tracer.startSpan(envelope.topic, {}, ctx);
|
|
826
|
+
* return {
|
|
827
|
+
* wrap: (fn) => context.with(trace.setSpan(ctx, span), fn),
|
|
828
|
+
* cleanup: () => span.end(),
|
|
829
|
+
* };
|
|
830
|
+
* }
|
|
831
|
+
* ```
|
|
468
832
|
*/
|
|
469
833
|
type BeforeConsumeResult = (() => void) | {
|
|
470
834
|
cleanup?(): void;
|
|
@@ -479,7 +843,80 @@ type BeforeConsumeResult = (() => void) | {
|
|
|
479
843
|
* - `'ttl-expired'` — message age exceeded `messageTtlMs` before the handler ran.
|
|
480
844
|
*/
|
|
481
845
|
type DlqReason = "handler-error" | "validation-error" | "lamport-clock-duplicate" | "ttl-expired";
|
|
482
|
-
/**
|
|
846
|
+
/**
|
|
847
|
+
* Options for `readSnapshot`.
|
|
848
|
+
*
|
|
849
|
+
* @example
|
|
850
|
+
* ```ts
|
|
851
|
+
* const snapshot = await kafka.readSnapshot('users.state', {
|
|
852
|
+
* schema: UserSchema,
|
|
853
|
+
* onTombstone: (key) => console.log(`Key ${key} was compacted away`),
|
|
854
|
+
* });
|
|
855
|
+
* ```
|
|
856
|
+
*/
|
|
857
|
+
interface ReadSnapshotOptions {
|
|
858
|
+
/**
|
|
859
|
+
* Schema to validate each message payload against (Zod, Valibot, ArkType, or any `.parse()` shape).
|
|
860
|
+
* Messages that fail validation are skipped with a warning log — they do not throw.
|
|
861
|
+
*/
|
|
862
|
+
schema?: SchemaLike;
|
|
863
|
+
/**
|
|
864
|
+
* Called when a tombstone record (null-value message) is encountered.
|
|
865
|
+
* The corresponding key is removed from the snapshot automatically.
|
|
866
|
+
* Use this for auditing or logging which keys were compacted away.
|
|
867
|
+
*/
|
|
868
|
+
onTombstone?: (key: string) => void;
|
|
869
|
+
}
|
|
870
|
+
/** A single partition offset entry stored in a checkpoint record. */
|
|
871
|
+
interface CheckpointEntry {
|
|
872
|
+
topic: string;
|
|
873
|
+
partition: number;
|
|
874
|
+
offset: string;
|
|
875
|
+
}
|
|
876
|
+
/** Result returned by a successful `checkpointOffsets` call. */
|
|
877
|
+
interface CheckpointResult {
|
|
878
|
+
/** Consumer group whose offsets were saved. */
|
|
879
|
+
groupId: string;
|
|
880
|
+
/** Topics included in the checkpoint. */
|
|
881
|
+
topics: string[];
|
|
882
|
+
/** Total number of topic-partition pairs saved. */
|
|
883
|
+
partitionCount: number;
|
|
884
|
+
/** Unix timestamp (ms) when the checkpoint was created. */
|
|
885
|
+
savedAt: number;
|
|
886
|
+
}
|
|
887
|
+
/** Options for `restoreFromCheckpoint`. */
|
|
888
|
+
interface RestoreCheckpointOptions {
|
|
889
|
+
/**
|
|
890
|
+
* Target Unix timestamp (ms). The newest checkpoint whose `savedAt` is **≤ this value**
|
|
891
|
+
* is selected. Defaults to the latest available checkpoint when omitted.
|
|
892
|
+
*/
|
|
893
|
+
timestamp?: number;
|
|
894
|
+
}
|
|
895
|
+
/** Result returned by a successful `restoreFromCheckpoint` call. */
|
|
896
|
+
interface CheckpointRestoreResult {
|
|
897
|
+
/** Consumer group that was repositioned. */
|
|
898
|
+
groupId: string;
|
|
899
|
+
/** The committed offsets restored from the checkpoint. */
|
|
900
|
+
offsets: CheckpointEntry[];
|
|
901
|
+
/** Unix timestamp (ms) recorded when the checkpoint was originally saved. */
|
|
902
|
+
restoredAt: number;
|
|
903
|
+
/** Age of the restored checkpoint in milliseconds (now − `restoredAt`). */
|
|
904
|
+
checkpointAge: number;
|
|
905
|
+
}
|
|
906
|
+
/**
|
|
907
|
+
* Options for `replayDlq`.
|
|
908
|
+
*
|
|
909
|
+
* @example
|
|
910
|
+
* ```ts
|
|
911
|
+
* await kafka.replayDlq('orders.created', {
|
|
912
|
+
* targetTopic: 'orders.retry-manual',
|
|
913
|
+
* dryRun: false,
|
|
914
|
+
* filter: (headers, value) =>
|
|
915
|
+
* headers['x-dlq-reason'] === 'handler-error' &&
|
|
916
|
+
* JSON.parse(value).amount > 0,
|
|
917
|
+
* });
|
|
918
|
+
* ```
|
|
919
|
+
*/
|
|
483
920
|
interface DlqReplayOptions {
|
|
484
921
|
/**
|
|
485
922
|
* Override the target topic to re-publish to.
|
|
@@ -497,6 +934,13 @@ interface DlqReplayOptions {
|
|
|
497
934
|
* @param value Raw message value (JSON string).
|
|
498
935
|
*/
|
|
499
936
|
filter?: (headers: MessageHeaders, value: string) => boolean;
|
|
937
|
+
/**
|
|
938
|
+
* Seek to the earliest available offset before consuming, regardless of any
|
|
939
|
+
* previously committed offsets for the replay consumer group.
|
|
940
|
+
* Default: `true` — full replay of all DLQ messages on every call.
|
|
941
|
+
* Set to `false` to replay only messages added since the previous `replayDlq` call.
|
|
942
|
+
*/
|
|
943
|
+
fromBeginning?: boolean;
|
|
500
944
|
}
|
|
501
945
|
/**
|
|
502
946
|
* Snapshot of internal event counters accumulated since client creation
|
|
@@ -517,6 +961,24 @@ interface KafkaMetrics {
|
|
|
517
961
|
* Use this for cross-cutting concerns like tracing and metrics.
|
|
518
962
|
*
|
|
519
963
|
* @see `otelInstrumentation()` from `@drarzter/kafka-client/otel`
|
|
964
|
+
*
|
|
965
|
+
* @example
|
|
966
|
+
* ```ts
|
|
967
|
+
* const tracing: KafkaInstrumentation = {
|
|
968
|
+
* beforeSend(topic, headers) {
|
|
969
|
+
* headers['traceparent'] = getCurrentTraceId();
|
|
970
|
+
* },
|
|
971
|
+
* beforeConsume(envelope) {
|
|
972
|
+
* const span = tracer.startSpan(envelope.topic);
|
|
973
|
+
* return { cleanup: () => span.end() };
|
|
974
|
+
* },
|
|
975
|
+
* onDlq(envelope, reason) {
|
|
976
|
+
* metrics.increment('kafka.dlq', { topic: envelope.topic, reason });
|
|
977
|
+
* },
|
|
978
|
+
* };
|
|
979
|
+
*
|
|
980
|
+
* const kafka = new KafkaClient(config, groupId, { instrumentation: [tracing] });
|
|
981
|
+
* ```
|
|
520
982
|
*/
|
|
521
983
|
interface KafkaInstrumentation {
|
|
522
984
|
/** Called before sending — can mutate `headers` (e.g. inject `traceparent`). */
|
|
@@ -558,19 +1020,155 @@ interface KafkaInstrumentation {
|
|
|
558
1020
|
/** Called when the circuit closes (normal operation restored). */
|
|
559
1021
|
onCircuitClose?(topic: string, partition: number): void;
|
|
560
1022
|
}
|
|
561
|
-
/**
|
|
1023
|
+
/**
|
|
1024
|
+
* Context passed to the `transaction()` callback with type-safe send methods.
|
|
1025
|
+
*
|
|
1026
|
+
* @example
|
|
1027
|
+
* ```ts
|
|
1028
|
+
* await kafka.transaction(async (tx) => {
|
|
1029
|
+
* await tx.send('inventory.reserved', { itemId: 'a', qty: 1 });
|
|
1030
|
+
* await tx.sendBatch('audit.log', [
|
|
1031
|
+
* { value: { action: 'reserve', itemId: 'a' } },
|
|
1032
|
+
* ]);
|
|
1033
|
+
* });
|
|
1034
|
+
* ```
|
|
1035
|
+
*/
|
|
562
1036
|
interface TransactionContext<T extends TopicMapConstraint<T>> {
|
|
563
1037
|
send<K extends keyof T>(topic: K, message: T[K], options?: SendOptions): Promise<void>;
|
|
564
1038
|
send<D extends TopicDescriptor<string & keyof T, T[string & keyof T]>>(descriptor: D, message: D["__type"], options?: SendOptions): Promise<void>;
|
|
565
1039
|
sendBatch<K extends keyof T>(topic: K, messages: Array<BatchMessageItem<T[K]>>, options?: BatchSendOptions): Promise<void>;
|
|
566
1040
|
sendBatch<D extends TopicDescriptor<string & keyof T, T[string & keyof T]>>(descriptor: D, messages: Array<BatchMessageItem<D["__type"]>>, options?: BatchSendOptions): Promise<void>;
|
|
567
1041
|
}
|
|
568
|
-
/**
|
|
1042
|
+
/**
|
|
1043
|
+
* Transactional context passed to each `startTransactionalConsumer` handler invocation.
|
|
1044
|
+
*
|
|
1045
|
+
* Every call to `send` or `sendBatch` on this object is part of the same Kafka
|
|
1046
|
+
* transaction as the source message offset commit. Either all sends and the offset
|
|
1047
|
+
* commit succeed atomically, or the transaction is aborted and the source message
|
|
1048
|
+
* is redelivered.
|
|
1049
|
+
*
|
|
1050
|
+
* @example
|
|
1051
|
+
* ```ts
|
|
1052
|
+
* await kafka.startTransactionalConsumer(['orders.created'], async (envelope, tx) => {
|
|
1053
|
+
* await tx.send('inventory.reserved', { orderId: envelope.payload.orderId, qty: 1 });
|
|
1054
|
+
* await tx.sendBatch('audit.log', [{ value: { action: 'reserve', orderId: envelope.payload.orderId } }]);
|
|
1055
|
+
* });
|
|
1056
|
+
* ```
|
|
1057
|
+
*/
|
|
1058
|
+
interface TransactionalHandlerContext<T extends TopicMapConstraint<T>> {
|
|
1059
|
+
/**
|
|
1060
|
+
* Send a message as part of this message's transaction.
|
|
1061
|
+
* The send is staged — it only becomes visible to consumers after the transaction commits.
|
|
1062
|
+
*/
|
|
1063
|
+
send<K extends keyof T>(topic: K, message: T[K], options?: SendOptions): Promise<void>;
|
|
1064
|
+
/**
|
|
1065
|
+
* Send multiple messages as part of this message's transaction.
|
|
1066
|
+
* All messages are staged together and become visible only after commit.
|
|
1067
|
+
*/
|
|
1068
|
+
sendBatch<K extends keyof T>(topic: K, messages: Array<BatchMessageItem<T[K]>>, options?: BatchSendOptions): Promise<void>;
|
|
1069
|
+
}
|
|
1070
|
+
/**
|
|
1071
|
+
* Routing configuration for `startRoutedConsumer`.
|
|
1072
|
+
*
|
|
1073
|
+
* Messages are dispatched to the handler whose key matches the value of `header`.
|
|
1074
|
+
* Unmatched messages are forwarded to `fallback` if provided, or silently skipped.
|
|
1075
|
+
*
|
|
1076
|
+
* @example
|
|
1077
|
+
* ```ts
|
|
1078
|
+
* await kafka.startRoutedConsumer(['events'], {
|
|
1079
|
+
* header: 'x-event-type',
|
|
1080
|
+
* routes: {
|
|
1081
|
+
* 'order.created': async (e) => handleOrderCreated(e.payload),
|
|
1082
|
+
* 'order.cancelled': async (e) => handleOrderCancelled(e.payload),
|
|
1083
|
+
* },
|
|
1084
|
+
* fallback: async (e) => logger.warn('Unknown event type', e.headers),
|
|
1085
|
+
* });
|
|
1086
|
+
* ```
|
|
1087
|
+
*/
|
|
1088
|
+
interface RoutingOptions<E> {
|
|
1089
|
+
/** Header whose value determines which handler is invoked. */
|
|
1090
|
+
header: string;
|
|
1091
|
+
/**
|
|
1092
|
+
* Map of header value → handler.
|
|
1093
|
+
* The handler with the matching key is called for each message.
|
|
1094
|
+
*/
|
|
1095
|
+
routes: Record<string, (envelope: EventEnvelope<E>) => Promise<void>>;
|
|
1096
|
+
/**
|
|
1097
|
+
* Called when no route matches the header value (or the header is absent).
|
|
1098
|
+
* If omitted, unmatched messages are silently skipped.
|
|
1099
|
+
*/
|
|
1100
|
+
fallback?: (envelope: EventEnvelope<E>) => Promise<void>;
|
|
1101
|
+
}
|
|
1102
|
+
/**
|
|
1103
|
+
* Metadata passed to the `startWindowConsumer` handler on each flush.
|
|
1104
|
+
*
|
|
1105
|
+
* @example
|
|
1106
|
+
* ```ts
|
|
1107
|
+
* await kafka.startWindowConsumer('events', async (batch, meta) => {
|
|
1108
|
+
* console.log(`Flush: ${batch.length} events, trigger=${meta.trigger}`);
|
|
1109
|
+
* console.log(`Window: ${meta.windowEnd - meta.windowStart} ms`);
|
|
1110
|
+
* await db.insertMany(batch.map(e => e.payload));
|
|
1111
|
+
* }, { maxMessages: 100, maxMs: 5_000 });
|
|
1112
|
+
* ```
|
|
1113
|
+
*/
|
|
1114
|
+
interface WindowMeta {
|
|
1115
|
+
/** What triggered this flush: accumulated `maxMessages` messages, or the `maxMs` timer. */
|
|
1116
|
+
trigger: "size" | "time";
|
|
1117
|
+
/** Unix timestamp (ms) of the first message that entered the current window. */
|
|
1118
|
+
windowStart: number;
|
|
1119
|
+
/** Unix timestamp (ms) when the flush was initiated. */
|
|
1120
|
+
windowEnd: number;
|
|
1121
|
+
}
|
|
1122
|
+
/**
|
|
1123
|
+
* Options for `startWindowConsumer`.
|
|
1124
|
+
*
|
|
1125
|
+
* Extends `ConsumerOptions` — all standard consumer settings apply.
|
|
1126
|
+
* `retryTopics`, `retryTopicAssignmentTimeoutMs`, and `queueHighWaterMark`
|
|
1127
|
+
* are excluded: they are incompatible with windowed accumulation.
|
|
1128
|
+
*/
|
|
1129
|
+
interface WindowConsumerOptions<T extends TopicMapConstraint<T> = TTopicMessageMap> extends Omit<ConsumerOptions<T>, "retryTopics" | "retryTopicAssignmentTimeoutMs" | "queueHighWaterMark"> {
|
|
1130
|
+
/**
|
|
1131
|
+
* Maximum number of messages to accumulate before flushing.
|
|
1132
|
+
* When the buffer reaches this size the handler is called immediately,
|
|
1133
|
+
* regardless of how much time has elapsed since the first message.
|
|
1134
|
+
*/
|
|
1135
|
+
maxMessages: number;
|
|
1136
|
+
/**
|
|
1137
|
+
* Maximum time (ms) to wait after the first message before flushing.
|
|
1138
|
+
* When this timer expires the handler is called with whatever messages
|
|
1139
|
+
* have accumulated so far — even if the buffer is not full.
|
|
1140
|
+
*/
|
|
1141
|
+
maxMs: number;
|
|
1142
|
+
}
|
|
1143
|
+
/**
|
|
1144
|
+
* Handle returned by `startConsumer` / `startBatchConsumer`.
|
|
1145
|
+
*
|
|
1146
|
+
* @example
|
|
1147
|
+
* ```ts
|
|
1148
|
+
* const handle = await kafka.startConsumer(['orders'], handler);
|
|
1149
|
+
* // later, on shutdown:
|
|
1150
|
+
* await handle.stop();
|
|
1151
|
+
* ```
|
|
1152
|
+
*/
|
|
569
1153
|
interface ConsumerHandle {
|
|
570
1154
|
/** The consumer group ID this consumer is running under. */
|
|
571
1155
|
groupId: string;
|
|
572
1156
|
/** Stop this consumer. Equivalent to calling `client.stopConsumer(groupId)`. */
|
|
573
1157
|
stop(): Promise<void>;
|
|
1158
|
+
/**
|
|
1159
|
+
* Resolves once the consumer has received its first partition assignment from
|
|
1160
|
+
* the broker (i.e. it has joined the group and is ready to receive messages).
|
|
1161
|
+
*
|
|
1162
|
+
* Use this in tests and startup probes instead of a fixed `setTimeout` delay:
|
|
1163
|
+
*
|
|
1164
|
+
* @example
|
|
1165
|
+
* ```ts
|
|
1166
|
+
* const handle = await kafka.startConsumer(['orders'], handler);
|
|
1167
|
+
* await handle.ready(); // wait for partition assignment — no fixed sleep needed
|
|
1168
|
+
* await kafka.sendMessage('orders', payload);
|
|
1169
|
+
* ```
|
|
1170
|
+
*/
|
|
1171
|
+
ready(): Promise<void>;
|
|
574
1172
|
}
|
|
575
1173
|
/** Result returned by `KafkaClient.checkStatus()`. */
|
|
576
1174
|
type KafkaHealthResult = {
|
|
@@ -612,15 +1210,34 @@ interface TopicDescription {
|
|
|
612
1210
|
}
|
|
613
1211
|
/** Interface describing all public methods of the Kafka client. */
|
|
614
1212
|
interface IKafkaClient<T extends TopicMapConstraint<T>> {
|
|
1213
|
+
/**
|
|
1214
|
+
* @example
|
|
1215
|
+
* ```ts
|
|
1216
|
+
* const status = await kafka.checkStatus();
|
|
1217
|
+
* if (status.status === 'down') console.error(status.error);
|
|
1218
|
+
* ```
|
|
1219
|
+
*/
|
|
615
1220
|
checkStatus(): Promise<KafkaHealthResult>;
|
|
616
1221
|
/**
|
|
617
1222
|
* List all consumer groups known to the broker.
|
|
618
1223
|
* @returns Array of `{ groupId, state }` summaries.
|
|
1224
|
+
*
|
|
1225
|
+
* @example
|
|
1226
|
+
* ```ts
|
|
1227
|
+
* const groups = await kafka.listConsumerGroups();
|
|
1228
|
+
* console.log(groups.map(g => `${g.groupId}: ${g.state}`));
|
|
1229
|
+
* ```
|
|
619
1230
|
*/
|
|
620
1231
|
listConsumerGroups(): Promise<ConsumerGroupSummary[]>;
|
|
621
1232
|
/**
|
|
622
1233
|
* Describe topics — returns partition layout, leader, replicas, and ISR for each topic.
|
|
623
1234
|
* @param topics Topic names to describe. Omit to describe all topics visible to this client.
|
|
1235
|
+
*
|
|
1236
|
+
* @example
|
|
1237
|
+
* ```ts
|
|
1238
|
+
* const [desc] = await kafka.describeTopics(['orders.created']);
|
|
1239
|
+
* console.log(desc.partitions.length); // number of partitions
|
|
1240
|
+
* ```
|
|
624
1241
|
*/
|
|
625
1242
|
describeTopics(topics?: string[]): Promise<TopicDescription[]>;
|
|
626
1243
|
/**
|
|
@@ -630,6 +1247,14 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
|
|
|
630
1247
|
*
|
|
631
1248
|
* @param topic Topic name.
|
|
632
1249
|
* @param partitions Array of `{ partition, offset }` — records before each offset are deleted.
|
|
1250
|
+
*
|
|
1251
|
+
* @example
|
|
1252
|
+
* ```ts
|
|
1253
|
+
* await kafka.deleteRecords('orders.created', [
|
|
1254
|
+
* { partition: 0, offset: '1000' },
|
|
1255
|
+
* { partition: 1, offset: '500' },
|
|
1256
|
+
* ]);
|
|
1257
|
+
* ```
|
|
633
1258
|
*/
|
|
634
1259
|
deleteRecords(topic: string, partitions: Array<{
|
|
635
1260
|
partition: number;
|
|
@@ -640,12 +1265,30 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
|
|
|
640
1265
|
* Lag = (broker high-watermark offset) − (last committed offset).
|
|
641
1266
|
* - A committed offset of `-1` (no offset committed yet) counts as full lag.
|
|
642
1267
|
* - Defaults to the client's default `groupId` when none is provided.
|
|
1268
|
+
*
|
|
1269
|
+
* @example
|
|
1270
|
+
* ```ts
|
|
1271
|
+
* const lag = await kafka.getConsumerLag();
|
|
1272
|
+
* const total = lag.reduce((sum, p) => sum + p.lag, 0);
|
|
1273
|
+
* console.log(`Total lag: ${total} messages`);
|
|
1274
|
+
* ```
|
|
643
1275
|
*/
|
|
644
1276
|
getConsumerLag(groupId?: string): Promise<Array<{
|
|
645
1277
|
topic: string;
|
|
646
1278
|
partition: number;
|
|
647
1279
|
lag: number;
|
|
648
1280
|
}>>;
|
|
1281
|
+
/**
|
|
1282
|
+
* @example
|
|
1283
|
+
* ```ts
|
|
1284
|
+
* const handle = await kafka.startConsumer(['orders.created'], async (envelope) => {
|
|
1285
|
+
* await processOrder(envelope.payload);
|
|
1286
|
+
* }, { retry: { maxRetries: 3 }, dlq: true });
|
|
1287
|
+
*
|
|
1288
|
+
* // on shutdown:
|
|
1289
|
+
* await handle.stop();
|
|
1290
|
+
* ```
|
|
1291
|
+
*/
|
|
649
1292
|
startConsumer<K extends Array<keyof T>>(topics: K, handleMessage: (envelope: EventEnvelope<T[K[number]]>) => Promise<void>, options?: ConsumerOptions<T>): Promise<ConsumerHandle>;
|
|
650
1293
|
startConsumer<D extends TopicDescriptor<string & keyof T, T[string & keyof T]>>(topics: D[], handleMessage: (envelope: EventEnvelope<D["__type"]>) => Promise<void>, options?: ConsumerOptions<T>): Promise<ConsumerHandle>;
|
|
651
1294
|
/**
|
|
@@ -654,6 +1297,15 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
|
|
|
654
1297
|
* Incompatible with `retryTopics: true`.
|
|
655
1298
|
*/
|
|
656
1299
|
startConsumer(topics: (string | RegExp)[], handleMessage: (envelope: EventEnvelope<T[keyof T]>) => Promise<void>, options?: ConsumerOptions<T>): Promise<ConsumerHandle>;
|
|
1300
|
+
/**
|
|
1301
|
+
* @example
|
|
1302
|
+
* ```ts
|
|
1303
|
+
* await kafka.startBatchConsumer(['metrics'], async (envelopes, meta) => {
|
|
1304
|
+
* await db.insertMany(envelopes.map(e => e.payload));
|
|
1305
|
+
* meta.resolveOffset(envelopes.at(-1)!.offset);
|
|
1306
|
+
* }, { autoCommit: false });
|
|
1307
|
+
* ```
|
|
1308
|
+
*/
|
|
657
1309
|
startBatchConsumer<K extends Array<keyof T>>(topics: K, handleBatch: (envelopes: EventEnvelope<T[K[number]]>[], meta: BatchMeta) => Promise<void>, options?: ConsumerOptions<T>): Promise<ConsumerHandle>;
|
|
658
1310
|
startBatchConsumer<D extends TopicDescriptor<string & keyof T, T[string & keyof T]>>(topics: D[], handleBatch: (envelopes: EventEnvelope<D["__type"]>[], meta: BatchMeta) => Promise<void>, options?: ConsumerOptions<T>): Promise<ConsumerHandle>;
|
|
659
1311
|
/**
|
|
@@ -662,12 +1314,101 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
|
|
|
662
1314
|
* Incompatible with `retryTopics: true`.
|
|
663
1315
|
*/
|
|
664
1316
|
startBatchConsumer(topics: (string | RegExp)[], handleBatch: (envelopes: EventEnvelope<T[keyof T]>[], meta: BatchMeta) => Promise<void>, options?: ConsumerOptions<T>): Promise<ConsumerHandle>;
|
|
1317
|
+
/**
|
|
1318
|
+
* Subscribe to a topic and accumulate messages into a window, flushing the handler
|
|
1319
|
+
* when **either** `maxMessages` messages have accumulated **or** `maxMs` milliseconds
|
|
1320
|
+
* have elapsed since the first message in the current window — whichever fires first.
|
|
1321
|
+
*
|
|
1322
|
+
* This is semantically different from `startBatchConsumer`, which delivers
|
|
1323
|
+
* broker-sized batches of unpredictable size. A window consumer gives you control
|
|
1324
|
+
* over both the size and the latency of each batch — useful for micro-batching
|
|
1325
|
+
* writes to a database, aggregating events before processing, or rate-limiting a
|
|
1326
|
+
* downstream API.
|
|
1327
|
+
*
|
|
1328
|
+
* On `handle.stop()`, any messages remaining in the buffer are flushed before the
|
|
1329
|
+
* consumer disconnects, so no messages are lost on clean shutdown.
|
|
1330
|
+
*
|
|
1331
|
+
* @param topic Topic to consume.
|
|
1332
|
+
* @param handler Called with each flushed window and a `WindowMeta` describing the trigger.
|
|
1333
|
+
* @param options Window size/timeout and standard consumer options.
|
|
1334
|
+
*
|
|
1335
|
+
* @example
|
|
1336
|
+
* ```ts
|
|
1337
|
+
* await kafka.startWindowConsumer('events', async (batch, meta) => {
|
|
1338
|
+
* console.log(`Flushing ${batch.length} events (trigger: ${meta.trigger})`);
|
|
1339
|
+
* await db.insertMany(batch.map(e => e.payload));
|
|
1340
|
+
* }, { maxMessages: 100, maxMs: 5_000 });
|
|
1341
|
+
* ```
|
|
1342
|
+
*/
|
|
1343
|
+
startWindowConsumer<K extends keyof T & string>(topic: K, handler: (envelopes: EventEnvelope<T[K]>[], meta: WindowMeta) => Promise<void>, options: WindowConsumerOptions<T>): Promise<ConsumerHandle>;
|
|
1344
|
+
/**
|
|
1345
|
+
* Subscribe to one or more topics and dispatch each message to a handler based
|
|
1346
|
+
* on the value of a specific Kafka header. Eliminates boilerplate `if/switch`
|
|
1347
|
+
* statements inside a catch-all handler when one topic carries multiple event types.
|
|
1348
|
+
*
|
|
1349
|
+
* Messages whose header value does not match any route key are forwarded to `fallback`
|
|
1350
|
+
* if provided, or silently skipped otherwise.
|
|
1351
|
+
*
|
|
1352
|
+
* Accepts the same `ConsumerOptions` as `startConsumer` — retry, DLQ, deduplication,
|
|
1353
|
+
* circuit breaker, interceptors, etc. all apply to every route.
|
|
1354
|
+
*
|
|
1355
|
+
* @param topics Array of topic keys or `RegExp` patterns.
|
|
1356
|
+
* @param routing Header name, route map, and optional fallback handler.
|
|
1357
|
+
* @param options Standard consumer options.
|
|
1358
|
+
* @returns A handle with `{ groupId, stop() }`.
|
|
1359
|
+
*
|
|
1360
|
+
* @example
|
|
1361
|
+
* ```ts
|
|
1362
|
+
* await kafka.startRoutedConsumer(['domain.events'], {
|
|
1363
|
+
* header: 'x-event-type',
|
|
1364
|
+
* routes: {
|
|
1365
|
+
* 'order.created': async (e) => handleOrderCreated(e.payload),
|
|
1366
|
+
* 'order.cancelled': async (e) => handleOrderCancelled(e.payload),
|
|
1367
|
+
* },
|
|
1368
|
+
* fallback: async (e) => logger.warn('Unknown event type', e.headers),
|
|
1369
|
+
* });
|
|
1370
|
+
* ```
|
|
1371
|
+
*/
|
|
1372
|
+
startRoutedConsumer<K extends Array<keyof T>>(topics: K, routing: RoutingOptions<T[K[number]]>, options?: ConsumerOptions<T>): Promise<ConsumerHandle>;
|
|
1373
|
+
/**
|
|
1374
|
+
* Subscribe to topics and consume messages with **exactly-once semantics** for
|
|
1375
|
+
* read-process-write pipelines.
|
|
1376
|
+
*
|
|
1377
|
+
* Each message is processed inside a Kafka transaction: the handler receives a
|
|
1378
|
+
* `TransactionalHandlerContext` with `send` / `sendBatch` methods that stage
|
|
1379
|
+
* outgoing messages inside the transaction. When the handler resolves, the source
|
|
1380
|
+
* offset commit and all staged sends are committed atomically. A handler error
|
|
1381
|
+
* aborts the transaction and the source message is redelivered — no sends become
|
|
1382
|
+
* visible to downstream consumers.
|
|
1383
|
+
*
|
|
1384
|
+
* Incompatible with `retryTopics: true` — throws at startup if set.
|
|
1385
|
+
* `autoCommit` is always `false` (managed by the transaction).
|
|
1386
|
+
*
|
|
1387
|
+
* @param topics Array of topic keys.
|
|
1388
|
+
* @param handler Called for each message with the decoded envelope and a transaction context.
|
|
1389
|
+
* @param options Standard consumer options (`retry`, `dlq`, `deduplication`, etc.).
|
|
1390
|
+
* @returns A handle with `{ groupId, stop() }`.
|
|
1391
|
+
*
|
|
1392
|
+
* @example
|
|
1393
|
+
* ```ts
|
|
1394
|
+
* await kafka.startTransactionalConsumer(['orders.created'], async (envelope, tx) => {
|
|
1395
|
+
* await tx.send('inventory.reserved', { orderId: envelope.payload.orderId, qty: 1 });
|
|
1396
|
+
* });
|
|
1397
|
+
* ```
|
|
1398
|
+
*/
|
|
1399
|
+
startTransactionalConsumer<K extends Array<keyof T>>(topics: K, handler: (envelope: EventEnvelope<T[K[number]]>, tx: TransactionalHandlerContext<T>) => Promise<void>, options?: ConsumerOptions<T>): Promise<ConsumerHandle>;
|
|
665
1400
|
/**
|
|
666
1401
|
* Stop consumer(s).
|
|
667
1402
|
* - `stopConsumer(groupId)` — disconnect and remove the consumer for a specific group.
|
|
668
1403
|
* - `stopConsumer()` — disconnect and remove all consumers.
|
|
669
1404
|
*/
|
|
670
1405
|
stopConsumer(groupId?: string): Promise<void>;
|
|
1406
|
+
/**
|
|
1407
|
+
* @example
|
|
1408
|
+
* ```ts
|
|
1409
|
+
* await kafka.sendMessage('orders.created', { orderId: '123', amount: 99 });
|
|
1410
|
+
* ```
|
|
1411
|
+
*/
|
|
671
1412
|
sendMessage<K extends keyof T>(topic: K, message: T[K], options?: SendOptions): Promise<void>;
|
|
672
1413
|
/**
|
|
673
1414
|
* Send a null-value (tombstone) message to a topic.
|
|
@@ -681,10 +1422,39 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
|
|
|
681
1422
|
* @param topic Topic name or descriptor.
|
|
682
1423
|
* @param key Partition key identifying the record to tombstone.
|
|
683
1424
|
* @param headers Optional custom Kafka headers.
|
|
1425
|
+
*
|
|
1426
|
+
* @example
|
|
1427
|
+
* ```ts
|
|
1428
|
+
* await kafka.sendTombstone('users.state', 'user-42');
|
|
1429
|
+
* ```
|
|
684
1430
|
*/
|
|
685
1431
|
sendTombstone(topic: string, key: string, headers?: MessageHeaders): Promise<void>;
|
|
1432
|
+
/**
|
|
1433
|
+
* @example
|
|
1434
|
+
* ```ts
|
|
1435
|
+
* await kafka.sendBatch('orders.created', [
|
|
1436
|
+
* { value: { orderId: '1', amount: 10 }, key: 'order-1' },
|
|
1437
|
+
* { value: { orderId: '2', amount: 20 }, key: 'order-2' },
|
|
1438
|
+
* ]);
|
|
1439
|
+
* ```
|
|
1440
|
+
*/
|
|
686
1441
|
sendBatch<K extends keyof T>(topic: K, messages: Array<BatchMessageItem<T[K]>>, options?: BatchSendOptions): Promise<void>;
|
|
1442
|
+
/**
|
|
1443
|
+
* @example
|
|
1444
|
+
* ```ts
|
|
1445
|
+
* await kafka.transaction(async (tx) => {
|
|
1446
|
+
* await tx.send('orders.created', { orderId: '123', amount: 99 });
|
|
1447
|
+
* await tx.send('inventory.reserved', { itemId: 'a', qty: 1 });
|
|
1448
|
+
* });
|
|
1449
|
+
* ```
|
|
1450
|
+
*/
|
|
687
1451
|
transaction(fn: (ctx: TransactionContext<T>) => Promise<void>): Promise<void>;
|
|
1452
|
+
/**
|
|
1453
|
+
* @example
|
|
1454
|
+
* ```ts
|
|
1455
|
+
* const id = kafka.getClientId(); // e.g. 'my-service'
|
|
1456
|
+
* ```
|
|
1457
|
+
*/
|
|
688
1458
|
getClientId(): ClientId;
|
|
689
1459
|
/**
|
|
690
1460
|
* Return a snapshot of internal event counters (retry / DLQ / dedup).
|
|
@@ -693,12 +1463,26 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
|
|
|
693
1463
|
* if no events have been observed for that topic yet.
|
|
694
1464
|
*
|
|
695
1465
|
* Counters accumulate since client creation or the last `resetMetrics()` call.
|
|
1466
|
+
*
|
|
1467
|
+
* @example
|
|
1468
|
+
* ```ts
|
|
1469
|
+
* const { processedCount, dlqCount, retryCount } = kafka.getMetrics();
|
|
1470
|
+
* console.log(`Processed: ${processedCount}, DLQ: ${dlqCount}`);
|
|
1471
|
+
*
|
|
1472
|
+
* const topicMetrics = kafka.getMetrics('orders.created');
|
|
1473
|
+
* ```
|
|
696
1474
|
*/
|
|
697
1475
|
getMetrics(topic?: string): Readonly<KafkaMetrics>;
|
|
698
1476
|
/**
|
|
699
1477
|
* Reset internal event counters to zero.
|
|
700
1478
|
* - `resetMetrics()` — reset all topics.
|
|
701
1479
|
* - `resetMetrics(topic)` — reset a single topic only.
|
|
1480
|
+
*
|
|
1481
|
+
* @example
|
|
1482
|
+
* ```ts
|
|
1483
|
+
* kafka.resetMetrics(); // reset all
|
|
1484
|
+
* kafka.resetMetrics('orders.created'); // reset one topic
|
|
1485
|
+
* ```
|
|
702
1486
|
*/
|
|
703
1487
|
resetMetrics(topic?: string): void;
|
|
704
1488
|
/**
|
|
@@ -709,6 +1493,18 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
|
|
|
709
1493
|
* itself is not modified — messages remain there after replay.
|
|
710
1494
|
*
|
|
711
1495
|
* @returns `{ replayed, skipped }` — counts of re-published vs skipped messages.
|
|
1496
|
+
*
|
|
1497
|
+
* @example
|
|
1498
|
+
* ```ts
|
|
1499
|
+
* const { replayed, skipped } = await kafka.replayDlq('orders.created');
|
|
1500
|
+
* console.log(`Replayed ${replayed}, skipped ${skipped}`);
|
|
1501
|
+
*
|
|
1502
|
+
* // dry-run with filter:
|
|
1503
|
+
* await kafka.replayDlq('orders.created', {
|
|
1504
|
+
* dryRun: true,
|
|
1505
|
+
* filter: (headers) => headers['x-dlq-reason'] === 'handler-error',
|
|
1506
|
+
* });
|
|
1507
|
+
* ```
|
|
712
1508
|
*/
|
|
713
1509
|
replayDlq(topic: string, options?: DlqReplayOptions): Promise<{
|
|
714
1510
|
replayed: number;
|
|
@@ -725,6 +1521,12 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
|
|
|
725
1521
|
* @param topic Topic to reset.
|
|
726
1522
|
* @param position `'earliest'` seeks to the first available offset; `'latest'`
|
|
727
1523
|
* seeks past the last message (consumer will only see new messages).
|
|
1524
|
+
*
|
|
1525
|
+
* @example
|
|
1526
|
+
* ```ts
|
|
1527
|
+
* await kafka.stopConsumer('billing-service');
|
|
1528
|
+
* await kafka.resetOffsets('billing-service', 'orders.created', 'earliest');
|
|
1529
|
+
* ```
|
|
728
1530
|
*/
|
|
729
1531
|
resetOffsets(groupId: string | undefined, topic: string, position: "earliest" | "latest"): Promise<void>;
|
|
730
1532
|
/**
|
|
@@ -736,6 +1538,14 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
|
|
|
736
1538
|
*
|
|
737
1539
|
* @param groupId Consumer group to seek. Defaults to the client's default groupId.
|
|
738
1540
|
* @param assignments Array of `{ topic, partition, offset }` tuples.
|
|
1541
|
+
*
|
|
1542
|
+
* @example
|
|
1543
|
+
* ```ts
|
|
1544
|
+
* await kafka.seekToOffset('billing-service', [
|
|
1545
|
+
* { topic: 'orders.created', partition: 0, offset: '1000' },
|
|
1546
|
+
* { topic: 'orders.created', partition: 1, offset: '500' },
|
|
1547
|
+
* ]);
|
|
1548
|
+
* ```
|
|
739
1549
|
*/
|
|
740
1550
|
seekToOffset(groupId: string | undefined, assignments: Array<{
|
|
741
1551
|
topic: string;
|
|
@@ -752,6 +1562,14 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
|
|
|
752
1562
|
*
|
|
753
1563
|
* @param groupId Consumer group to seek. Defaults to the client's default groupId.
|
|
754
1564
|
* @param assignments Array of `{ topic, partition, timestamp }` tuples (Unix ms).
|
|
1565
|
+
*
|
|
1566
|
+
* @example
|
|
1567
|
+
* ```ts
|
|
1568
|
+
* const midnight = new Date('2025-01-01').getTime();
|
|
1569
|
+
* await kafka.seekToTimestamp('billing-service', [
|
|
1570
|
+
* { topic: 'orders.created', partition: 0, timestamp: midnight },
|
|
1571
|
+
* ]);
|
|
1572
|
+
* ```
|
|
755
1573
|
*/
|
|
756
1574
|
seekToTimestamp(groupId: string | undefined, assignments: Array<{
|
|
757
1575
|
topic: string;
|
|
@@ -766,12 +1584,84 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
|
|
|
766
1584
|
* @param topic Topic name.
|
|
767
1585
|
* @param partition Partition index.
|
|
768
1586
|
* @param groupId Consumer group. Defaults to the client's default groupId.
|
|
1587
|
+
*
|
|
1588
|
+
* @example
|
|
1589
|
+
* ```ts
|
|
1590
|
+
* const state = kafka.getCircuitState('orders.created', 0);
|
|
1591
|
+
* if (state?.status === 'open') console.warn('Circuit open!', state.failures);
|
|
1592
|
+
* ```
|
|
769
1593
|
*/
|
|
770
1594
|
getCircuitState(topic: string, partition: number, groupId?: string): {
|
|
771
1595
|
status: "closed" | "open" | "half-open";
|
|
772
1596
|
failures: number;
|
|
773
1597
|
windowSize: number;
|
|
774
1598
|
} | undefined;
|
|
1599
|
+
/**
|
|
1600
|
+
* Read a compacted topic from the beginning to its current high-watermark and
|
|
1601
|
+
* return a `Map<key, EventEnvelope<T>>` with the **latest** value per key.
|
|
1602
|
+
*
|
|
1603
|
+
* Tombstone records (null-value messages) remove the key from the map —
|
|
1604
|
+
* consistent with log-compaction semantics.
|
|
1605
|
+
*
|
|
1606
|
+
* Useful for bootstrapping in-memory state at service startup without an external cache:
|
|
1607
|
+
* ```ts
|
|
1608
|
+
* const orders = await kafka.readSnapshot('orders.state');
|
|
1609
|
+
* // orders.get('order-123') → EventEnvelope with the latest payload for that key
|
|
1610
|
+
* ```
|
|
1611
|
+
*
|
|
1612
|
+
* @param topic Topic to read. Must be a log-compacted topic (or any topic you want a key → latest-value index for).
|
|
1613
|
+
* @param options Optional schema validation and tombstone callback.
|
|
1614
|
+
* @returns Map keyed by the Kafka message key string; value is the last seen `EventEnvelope`.
|
|
1615
|
+
*/
|
|
1616
|
+
readSnapshot<K extends keyof T & string>(topic: K, options?: ReadSnapshotOptions): Promise<Map<string, EventEnvelope<T[K]>>>;
|
|
1617
|
+
/**
|
|
1618
|
+
* Snapshot the current committed offsets of a consumer group into a Kafka topic.
|
|
1619
|
+
*
|
|
1620
|
+
* Each call appends a new checkpoint record keyed by `groupId`. The checkpoint topic
|
|
1621
|
+
* acts as an append-only audit log — use a non-compacted topic to retain history.
|
|
1622
|
+
* Use `restoreFromCheckpoint` to rewind the group to any saved point in time.
|
|
1623
|
+
*
|
|
1624
|
+
* Requires `connectProducer()` to have been called.
|
|
1625
|
+
*
|
|
1626
|
+
* @param groupId Consumer group whose offsets to checkpoint. Defaults to the client's default group.
|
|
1627
|
+
* @param checkpointTopic Topic where checkpoint records are written.
|
|
1628
|
+
* @returns Summary of the saved checkpoint.
|
|
1629
|
+
*
|
|
1630
|
+
* @example
|
|
1631
|
+
* ```ts
|
|
1632
|
+
* const result = await kafka.checkpointOffsets(undefined, 'checkpoints');
|
|
1633
|
+
* console.log(`Saved ${result.partitionCount} offsets at ${result.savedAt}`);
|
|
1634
|
+
* ```
|
|
1635
|
+
*/
|
|
1636
|
+
checkpointOffsets(groupId: string | undefined, checkpointTopic: string): Promise<CheckpointResult>;
|
|
1637
|
+
/**
|
|
1638
|
+
* Restore a consumer group's committed offsets from the nearest checkpoint stored in `checkpointTopic`.
|
|
1639
|
+
*
|
|
1640
|
+
* Reads all checkpoint records for the group, selects the newest checkpoint whose `savedAt`
|
|
1641
|
+
* timestamp is ≤ `options.timestamp` (or the latest checkpoint if no timestamp is given),
|
|
1642
|
+
* then calls `admin.setOffsets` for every topic-partition in that checkpoint.
|
|
1643
|
+
*
|
|
1644
|
+
* **The consumer group must be stopped before calling this method** — throws if any consumer
|
|
1645
|
+
* in the group is currently running.
|
|
1646
|
+
*
|
|
1647
|
+
* @param groupId Consumer group to reposition. Defaults to the client's default group.
|
|
1648
|
+
* @param checkpointTopic Topic where checkpoints were written by `checkpointOffsets`.
|
|
1649
|
+
* @param options.timestamp Target Unix ms. Omit to restore the latest checkpoint.
|
|
1650
|
+
* @throws If no checkpoint exists for the group, or if the group is still running.
|
|
1651
|
+
*
|
|
1652
|
+
* @example
|
|
1653
|
+
* ```ts
|
|
1654
|
+
* await kafka.stopConsumer('billing-service');
|
|
1655
|
+
* const result = await kafka.restoreFromCheckpoint(undefined, 'checkpoints');
|
|
1656
|
+
* console.log(`Restored from checkpoint saved ${result.checkpointAge}ms ago`);
|
|
1657
|
+
*
|
|
1658
|
+
* // restore to the state before a specific deployment:
|
|
1659
|
+
* await kafka.restoreFromCheckpoint(undefined, 'checkpoints', {
|
|
1660
|
+
* timestamp: new Date('2025-06-01T00:00:00Z').getTime(),
|
|
1661
|
+
* });
|
|
1662
|
+
* ```
|
|
1663
|
+
*/
|
|
1664
|
+
restoreFromCheckpoint(groupId: string | undefined, checkpointTopic: string, options?: RestoreCheckpointOptions): Promise<CheckpointRestoreResult>;
|
|
775
1665
|
/**
|
|
776
1666
|
* Consume messages as an async iterator. Useful for scripts, migrations, and
|
|
777
1667
|
* one-off processing where the full `startConsumer` lifecycle is unnecessary.
|
|
@@ -799,6 +1689,13 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
|
|
|
799
1689
|
*
|
|
800
1690
|
* @param groupId Consumer group to pause. Defaults to the client's default groupId.
|
|
801
1691
|
* @param assignments Topic-partition pairs to pause.
|
|
1692
|
+
*
|
|
1693
|
+
* @example
|
|
1694
|
+
* ```ts
|
|
1695
|
+
* kafka.pauseConsumer(undefined, [{ topic: 'orders.created', partitions: [0, 1] }]);
|
|
1696
|
+
* // ... do some backpressure work ...
|
|
1697
|
+
* kafka.resumeConsumer(undefined, [{ topic: 'orders.created', partitions: [0, 1] }]);
|
|
1698
|
+
* ```
|
|
802
1699
|
*/
|
|
803
1700
|
pauseConsumer(groupId: string | undefined, assignments: Array<{
|
|
804
1701
|
topic: string;
|
|
@@ -809,6 +1706,11 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
|
|
|
809
1706
|
*
|
|
810
1707
|
* @param groupId Consumer group to resume. Defaults to the client's default groupId.
|
|
811
1708
|
* @param assignments Topic-partition pairs to resume.
|
|
1709
|
+
*
|
|
1710
|
+
* @example
|
|
1711
|
+
* ```ts
|
|
1712
|
+
* kafka.resumeConsumer(undefined, [{ topic: 'orders.created', partitions: [0, 1] }]);
|
|
1713
|
+
* ```
|
|
812
1714
|
*/
|
|
813
1715
|
resumeConsumer(groupId: string | undefined, assignments: Array<{
|
|
814
1716
|
topic: string;
|
|
@@ -817,12 +1719,22 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
|
|
|
817
1719
|
/**
|
|
818
1720
|
* Drain in-flight handlers, then disconnect all producers, consumers, and admin.
|
|
819
1721
|
* @param drainTimeoutMs Max ms to wait for in-flight handlers (default 30 000).
|
|
1722
|
+
*
|
|
1723
|
+
* @example
|
|
1724
|
+
* ```ts
|
|
1725
|
+
* await kafka.disconnect();
|
|
1726
|
+
* ```
|
|
820
1727
|
*/
|
|
821
1728
|
disconnect(drainTimeoutMs?: number): Promise<void>;
|
|
822
1729
|
/**
|
|
823
1730
|
* Register SIGTERM / SIGINT signal handlers that drain in-flight messages before
|
|
824
1731
|
* disconnecting. Call once after constructing the client in non-NestJS apps.
|
|
825
1732
|
* NestJS apps get drain automatically via `onModuleDestroy` → `disconnect()`.
|
|
1733
|
+
*
|
|
1734
|
+
* @example
|
|
1735
|
+
* ```ts
|
|
1736
|
+
* kafka.enableGracefulShutdown(['SIGTERM', 'SIGINT'], 30_000);
|
|
1737
|
+
* ```
|
|
826
1738
|
*/
|
|
827
1739
|
enableGracefulShutdown(signals?: NodeJS.Signals[], drainTimeoutMs?: number): void;
|
|
828
1740
|
}
|
|
@@ -831,6 +1743,22 @@ interface IKafkaClient<T extends TopicMapConstraint<T>> {
|
|
|
831
1743
|
* Compatible with NestJS Logger, console, winston, pino, or any custom logger.
|
|
832
1744
|
*
|
|
833
1745
|
* `debug` is optional — omit it to suppress debug output in production.
|
|
1746
|
+
*
|
|
1747
|
+
* @example
|
|
1748
|
+
* ```ts
|
|
1749
|
+
* // Pass a NestJS logger:
|
|
1750
|
+
* const kafka = new KafkaClient(config, groupId, { logger: this.logger });
|
|
1751
|
+
*
|
|
1752
|
+
* // Or a minimal pino wrapper:
|
|
1753
|
+
* const kafka = new KafkaClient(config, groupId, {
|
|
1754
|
+
* logger: {
|
|
1755
|
+
* log: (msg) => pino.info(msg),
|
|
1756
|
+
* warn: (msg, ...a) => pino.warn(msg, ...a),
|
|
1757
|
+
* error: (msg, ...a) => pino.error(msg, ...a),
|
|
1758
|
+
* debug: (msg, ...a) => pino.debug(msg, ...a),
|
|
1759
|
+
* },
|
|
1760
|
+
* });
|
|
1761
|
+
* ```
|
|
834
1762
|
*/
|
|
835
1763
|
interface KafkaLogger {
|
|
836
1764
|
log(message: string): void;
|
|
@@ -841,6 +1769,15 @@ interface KafkaLogger {
|
|
|
841
1769
|
/**
|
|
842
1770
|
* Context passed to `onTtlExpired` when a message is dropped because it
|
|
843
1771
|
* exceeded `messageTtlMs` and `dlq` is not enabled.
|
|
1772
|
+
*
|
|
1773
|
+
* @example
|
|
1774
|
+
* ```ts
|
|
1775
|
+
* const kafka = new KafkaClient(config, groupId, {
|
|
1776
|
+
* onTtlExpired: ({ topic, ageMs, messageTtlMs }) => {
|
|
1777
|
+
* console.warn(`Message on ${topic} expired: age=${ageMs}ms, ttl=${messageTtlMs}ms`);
|
|
1778
|
+
* },
|
|
1779
|
+
* });
|
|
1780
|
+
* ```
|
|
844
1781
|
*/
|
|
845
1782
|
interface TtlExpiredContext {
|
|
846
1783
|
/** Topic the message was consumed from. */
|
|
@@ -855,6 +1792,15 @@ interface TtlExpiredContext {
|
|
|
855
1792
|
/**
|
|
856
1793
|
* Context passed to `onMessageLost` when a message is silently dropped
|
|
857
1794
|
* (handler threw and `dlq` is not enabled).
|
|
1795
|
+
*
|
|
1796
|
+
* @example
|
|
1797
|
+
* ```ts
|
|
1798
|
+
* const kafka = new KafkaClient(config, groupId, {
|
|
1799
|
+
* onMessageLost: ({ topic, error, attempt }) => {
|
|
1800
|
+
* alerting.fire('kafka.message-lost', { topic, error: error.message, attempt });
|
|
1801
|
+
* },
|
|
1802
|
+
* });
|
|
1803
|
+
* ```
|
|
858
1804
|
*/
|
|
859
1805
|
interface MessageLostContext {
|
|
860
1806
|
/** Topic the message was consumed from. */
|
|
@@ -866,7 +1812,20 @@ interface MessageLostContext {
|
|
|
866
1812
|
/** Original Kafka message headers (correlationId, traceparent, etc.). */
|
|
867
1813
|
headers: MessageHeaders;
|
|
868
1814
|
}
|
|
869
|
-
/**
|
|
1815
|
+
/**
|
|
1816
|
+
* Options for `KafkaClient` constructor.
|
|
1817
|
+
*
|
|
1818
|
+
* @example
|
|
1819
|
+
* ```ts
|
|
1820
|
+
* const kafka = new KafkaClient(kafkaConfig, 'my-service', {
|
|
1821
|
+
* transactionalId: `my-service-tx-${replicaIndex}`,
|
|
1822
|
+
* lagThrottle: { maxLag: 10_000, pollIntervalMs: 3_000 },
|
|
1823
|
+
* clockRecovery: { topics: ['orders.created'] },
|
|
1824
|
+
* onMessageLost: (ctx) => alerting.fire('kafka.message-lost', ctx),
|
|
1825
|
+
* instrumentation: [otelInstrumentation()],
|
|
1826
|
+
* });
|
|
1827
|
+
* ```
|
|
1828
|
+
*/
|
|
870
1829
|
interface KafkaClientOptions {
|
|
871
1830
|
/** Auto-create topics via admin before the first `sendMessage`, `sendBatch`, or `transaction` for each topic. Useful for development — not recommended in production. */
|
|
872
1831
|
autoCreateTopics?: boolean;
|
|
@@ -930,8 +1889,59 @@ interface KafkaClientOptions {
|
|
|
930
1889
|
/** Topic names to scan for the highest Lamport clock. */
|
|
931
1890
|
topics: string[];
|
|
932
1891
|
};
|
|
1892
|
+
/**
|
|
1893
|
+
* Delay `sendMessage` / `sendBatch` / `sendTombstone` when the observed lag of a
|
|
1894
|
+
* consumer group exceeds `maxLag`. Resumes immediately when lag drops below the threshold.
|
|
1895
|
+
*
|
|
1896
|
+
* Lag is polled via `getConsumerLag()` every `pollIntervalMs` in the background;
|
|
1897
|
+
* no admin call is made on each individual send.
|
|
1898
|
+
*
|
|
1899
|
+
* When `maxWaitMs` is exceeded the send is unblocked with a warning — this is
|
|
1900
|
+
* best-effort throttling, not hard back-pressure.
|
|
1901
|
+
*
|
|
1902
|
+
* Requires `connectProducer()` to have been called to start the polling loop.
|
|
1903
|
+
*/
|
|
1904
|
+
lagThrottle?: {
|
|
1905
|
+
/** Consumer group whose lag is monitored. Defaults to the client's default group. */
|
|
1906
|
+
groupId?: string;
|
|
1907
|
+
/** Lag threshold (number of messages) above which sends are delayed. */
|
|
1908
|
+
maxLag: number;
|
|
1909
|
+
/** How often to poll `getConsumerLag()`. Default: `5000` ms. */
|
|
1910
|
+
pollIntervalMs?: number;
|
|
1911
|
+
/**
|
|
1912
|
+
* Maximum time (ms) a send will wait while throttled before proceeding anyway.
|
|
1913
|
+
* Default: `30_000` ms.
|
|
1914
|
+
*/
|
|
1915
|
+
maxWaitMs?: number;
|
|
1916
|
+
};
|
|
1917
|
+
/**
|
|
1918
|
+
* Custom transport implementation.
|
|
1919
|
+
*
|
|
1920
|
+
* By default `KafkaClient` uses `ConfluentTransport` which wraps
|
|
1921
|
+
* `@confluentinc/kafka-javascript` (librdkafka). Inject a different
|
|
1922
|
+
* `KafkaTransport` to target an alternative broker library, or to supply
|
|
1923
|
+
* a deterministic fake in unit tests without mocking the confluentinc module.
|
|
1924
|
+
*
|
|
1925
|
+
* @example
|
|
1926
|
+
* ```ts
|
|
1927
|
+
* // In tests — no jest.mock() needed
|
|
1928
|
+
* const kafka = new KafkaClient('svc', 'grp', [], {
|
|
1929
|
+
* transport: new FakeTransport(),
|
|
1930
|
+
* });
|
|
1931
|
+
* ```
|
|
1932
|
+
*/
|
|
1933
|
+
transport?: KafkaTransport;
|
|
933
1934
|
}
|
|
934
|
-
/**
|
|
1935
|
+
/**
|
|
1936
|
+
* Options for consumer subscribe retry when topic doesn't exist yet.
|
|
1937
|
+
*
|
|
1938
|
+
* @example
|
|
1939
|
+
* ```ts
|
|
1940
|
+
* await kafka.startConsumer(['orders.created'], handler, {
|
|
1941
|
+
* subscribeRetry: { retries: 10, backoffMs: 2_000 },
|
|
1942
|
+
* });
|
|
1943
|
+
* ```
|
|
1944
|
+
*/
|
|
935
1945
|
interface SubscribeRetryOptions {
|
|
936
1946
|
/** Maximum number of subscribe attempts. Default: `5`. */
|
|
937
1947
|
retries?: number;
|
|
@@ -939,4 +1949,4 @@ interface SubscribeRetryOptions {
|
|
|
939
1949
|
backoffMs?: number;
|
|
940
1950
|
}
|
|
941
1951
|
|
|
942
|
-
export { type
|
|
1952
|
+
export { type KafkaMetrics as $, type CheckpointRestoreResult as A, type BatchMessageItem as B, type ClientId as C, type CheckpointResult as D, type CircuitBreakerOptions as E, type CompressionType as F, type GroupId as G, type ConsumerGroupSummary as H, type IKafkaClient as I, type ConsumerHandle as J, type KafkaInstrumentation as K, type ConsumerInterceptor as L, type DeduplicationOptions as M, type DlqReason as N, type DlqReplayOptions as O, type EnvelopeHeaderOptions as P, type EventEnvelope as Q, HEADER_CORRELATION_ID as R, type SchemaLike as S, type TopicMapConstraint as T, HEADER_EVENT_ID as U, HEADER_LAMPORT_CLOCK as V, HEADER_SCHEMA_VERSION as W, HEADER_TIMESTAMP as X, HEADER_TRACEPARENT as Y, type InferSchema as Z, type KafkaLogger as _, type IAdmin as a, type MessageHeaders as a0, type MessageLostContext as a1, type ReadSnapshotOptions as a2, type RestoreCheckpointOptions as a3, type RetryOptions as a4, type RoutingOptions as a5, type SchemaParseContext as a6, type SendOptions as a7, type SubscribeRetryOptions as a8, type TTopicMessageMap as a9, type TopicDescription as aa, type TopicPartitionInfo as ab, type TopicsFrom as ac, type TransactionContext as ad, type TransactionalHandlerContext as ae, type TtlExpiredContext as af, type WindowConsumerOptions as ag, type WindowMeta as ah, buildEnvelopeHeaders as ai, decodeHeaders as aj, extractEnvelope as ak, getEnvelopeContext as al, runWithEnvelopeContext as am, topic as an, type IPartitionWatermarks as b, type IGroupTopicOffsets as c, type IPartitionOffset as d, type IGroupDescription as e, type ITopicMetadata as f, type IConsumer as g, type IConsumerCreationOptions as h, type IConsumerRunConfig as i, type ITopicPartitions as j, type ITopicPartitionOffset as k, type ITopicPartition as l, type IMessage as m, type IProducer as n, type IProducerRecord as o, type ITransaction as p, type IProducerCreationOptions as q, type KafkaTransport as r, type KafkaClientOptions as s, type ConsumerOptions as t, type TopicDescriptor as u, type KafkaHealthResult as v, type BatchMeta as w, type BatchSendOptions as x, type BeforeConsumeResult as y, type CheckpointEntry as z };
|