@drarzter/kafka-client 0.6.9 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +132 -1
- package/dist/{chunk-4526Y4PV.mjs → chunk-MJ342P4R.mjs} +249 -13
- package/dist/chunk-MJ342P4R.mjs.map +1 -0
- package/dist/core.d.mts +28 -3
- package/dist/core.d.ts +28 -3
- package/dist/core.js +248 -12
- package/dist/core.js.map +1 -1
- package/dist/core.mjs +1 -1
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +248 -12
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1 -1
- package/dist/otel.d.mts +1 -1
- package/dist/otel.d.ts +1 -1
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +5 -0
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +5 -0
- package/dist/testing.mjs.map +1 -1
- package/dist/{types-736Gj0J3.d.mts → types-DqQ7IXZr.d.mts} +83 -2
- package/dist/{types-736Gj0J3.d.ts → types-DqQ7IXZr.d.ts} +83 -2
- package/package.json +1 -1
- package/dist/chunk-4526Y4PV.mjs.map +0 -1
package/README.md
CHANGED
|
@@ -20,6 +20,7 @@ Type-safe Kafka client for Node.js. Framework-agnostic core with a first-class N
|
|
|
20
20
|
- [Consuming messages](#consuming-messages)
|
|
21
21
|
- [Declarative: @SubscribeTo()](#declarative-subscribeto)
|
|
22
22
|
- [Imperative: startConsumer()](#imperative-startconsumer)
|
|
23
|
+
- [Iterator: consume()](#iterator-consume)
|
|
23
24
|
- [Multiple consumer groups](#multiple-consumer-groups)
|
|
24
25
|
- [Partition key](#partition-key)
|
|
25
26
|
- [Message headers](#message-headers)
|
|
@@ -34,7 +35,10 @@ Type-safe Kafka client for Node.js. Framework-agnostic core with a first-class N
|
|
|
34
35
|
- [Retry topic chain](#retry-topic-chain)
|
|
35
36
|
- [stopConsumer](#stopconsumer)
|
|
36
37
|
- [Pause and resume](#pause-and-resume)
|
|
38
|
+
- [Circuit breaker](#circuit-breaker)
|
|
37
39
|
- [Reset consumer offsets](#reset-consumer-offsets)
|
|
40
|
+
- [Seek to offset](#seek-to-offset)
|
|
41
|
+
- [Message TTL](#message-ttl)
|
|
38
42
|
- [DLQ replay](#dlq-replay)
|
|
39
43
|
- [Graceful shutdown](#graceful-shutdown)
|
|
40
44
|
- [Consumer handles](#consumer-handles)
|
|
@@ -86,6 +90,10 @@ Safe by default. Configurable when you need it. Escape hatches for when you know
|
|
|
86
90
|
- **Health check** — built-in health indicator for monitoring
|
|
87
91
|
- **Multiple consumer groups** — named clients for different bounded contexts
|
|
88
92
|
- **Declarative & imperative** — use `@SubscribeTo()` decorator or `startConsumer()` directly
|
|
93
|
+
- **Async iterator** — `consume<K>()` returns an `AsyncIterableIterator<EventEnvelope<T[K]>>` for `for await` consumption; breaking out of the loop stops the consumer automatically
|
|
94
|
+
- **Message TTL** — `messageTtlMs` drops or DLQs messages older than a configurable threshold, preventing stale events from poisoning downstream systems after a lag spike
|
|
95
|
+
- **Circuit breaker** — `circuitBreaker` option applies a sliding-window breaker per topic-partition; pauses delivery on repeated DLQ failures and resumes after a configurable recovery window
|
|
96
|
+
- **Seek to offset** — `seekToOffset(groupId, assignments)` seeks individual partitions to explicit offsets for fine-grained replay
|
|
89
97
|
|
|
90
98
|
See the [Roadmap](./ROADMAP.md) for upcoming features and version history.
|
|
91
99
|
|
|
@@ -379,7 +387,7 @@ export class OrdersService {
|
|
|
379
387
|
|
|
380
388
|
## Consuming messages
|
|
381
389
|
|
|
382
|
-
|
|
390
|
+
Three ways — choose what fits your style.
|
|
383
391
|
|
|
384
392
|
### Declarative: @SubscribeTo()
|
|
385
393
|
|
|
@@ -428,6 +436,35 @@ export class OrdersService implements OnModuleInit {
|
|
|
428
436
|
}
|
|
429
437
|
```
|
|
430
438
|
|
|
439
|
+
### Iterator: consume()
|
|
440
|
+
|
|
441
|
+
Stream messages from a single topic as an `AsyncIterableIterator` — useful for scripts, one-off tasks, or any context where you prefer `for await` over a callback:
|
|
442
|
+
|
|
443
|
+
```typescript
|
|
444
|
+
for await (const envelope of kafka.consume('order.created')) {
|
|
445
|
+
console.log('Order:', envelope.payload.orderId);
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
// Breaking out of the loop stops the consumer automatically
|
|
449
|
+
for await (const envelope of kafka.consume('order.created')) {
|
|
450
|
+
if (envelope.payload.orderId === targetId) break;
|
|
451
|
+
}
|
|
452
|
+
```
|
|
453
|
+
|
|
454
|
+
`consume()` accepts the same `ConsumerOptions` as `startConsumer()`:
|
|
455
|
+
|
|
456
|
+
```typescript
|
|
457
|
+
for await (const envelope of kafka.consume('orders', {
|
|
458
|
+
retry: { maxRetries: 3 },
|
|
459
|
+
dlq: true,
|
|
460
|
+
messageTtlMs: 60_000,
|
|
461
|
+
})) {
|
|
462
|
+
await processOrder(envelope.payload);
|
|
463
|
+
}
|
|
464
|
+
```
|
|
465
|
+
|
|
466
|
+
`break`, `return`, or any early exit from the loop calls the iterator's `return()` method, which closes the internal queue and calls `handle.stop()` on the background consumer.
|
|
467
|
+
|
|
431
468
|
## Multiple consumer groups
|
|
432
469
|
|
|
433
470
|
### Per-consumer groupId
|
|
@@ -828,6 +865,12 @@ Options for `sendMessage()` — the third argument:
|
|
|
828
865
|
| `handlerTimeoutMs` | — | Log a warning if the handler hasn't resolved within this window (ms) — does not cancel the handler |
|
|
829
866
|
| `deduplication.strategy` | `'drop'` | What to do with duplicate messages: `'drop'` silently discards, `'dlq'` forwards to `{topic}.dlq` (requires `dlq: true`), `'topic'` forwards to `{topic}.duplicates` |
|
|
830
867
|
| `deduplication.duplicatesTopic` | `{topic}.duplicates` | Custom destination for `strategy: 'topic'` |
|
|
868
|
+
| `messageTtlMs` | — | Drop (or DLQ) messages older than this many milliseconds at consumption time; evaluated against the `x-timestamp` header; see [Message TTL](#message-ttl) |
|
|
869
|
+
| `circuitBreaker` | — | Enable circuit breaker with `{}` for zero-config defaults; requires `dlq: true`; see [Circuit breaker](#circuit-breaker) |
|
|
870
|
+
| `circuitBreaker.threshold` | `5` | DLQ failures within `windowSize` that opens the circuit |
|
|
871
|
+
| `circuitBreaker.recoveryMs` | `30_000` | Milliseconds to wait in OPEN state before entering HALF_OPEN |
|
|
872
|
+
| `circuitBreaker.windowSize` | `threshold × 2, min 10` | Sliding window size in messages |
|
|
873
|
+
| `circuitBreaker.halfOpenSuccesses` | `1` | Consecutive successes in HALF_OPEN required to close the circuit |
|
|
831
874
|
| `batch` | `false` | (decorator only) Use `startBatchConsumer` instead of `startConsumer` |
|
|
832
875
|
| `subscribeRetry.retries` | `5` | Max attempts for `consumer.subscribe()` when topic doesn't exist yet |
|
|
833
876
|
| `subscribeRetry.backoffMs` | `5000` | Delay between subscribe retry attempts (ms) |
|
|
@@ -1109,6 +1152,54 @@ The first argument is the consumer group ID — pass `undefined` to target the d
|
|
|
1109
1152
|
|
|
1110
1153
|
Pausing is non-destructive: the consumer stays connected and Kafka preserves the partition assignment for as long as the group session is alive. Messages accumulate in the topic and are delivered once the consumer resumes. Typical use: apply backpressure when a downstream dependency (e.g. a database) is temporarily overloaded.
|
|
1111
1154
|
|
|
1155
|
+
## Circuit breaker
|
|
1156
|
+
|
|
1157
|
+
Automatically pause delivery from a topic-partition when its DLQ error rate exceeds a threshold. After a recovery window the partition is resumed automatically.
|
|
1158
|
+
|
|
1159
|
+
**`dlq: true` is required** — the breaker counts DLQ events as failures. Without it no failures are recorded and the circuit never opens.
|
|
1160
|
+
|
|
1161
|
+
Zero-config start — all options have sensible defaults:
|
|
1162
|
+
|
|
1163
|
+
```typescript
|
|
1164
|
+
await kafka.startConsumer(['orders'], handler, {
|
|
1165
|
+
dlq: true,
|
|
1166
|
+
circuitBreaker: {},
|
|
1167
|
+
});
|
|
1168
|
+
```
|
|
1169
|
+
|
|
1170
|
+
Full config for fine-tuning:
|
|
1171
|
+
|
|
1172
|
+
```typescript
|
|
1173
|
+
await kafka.startConsumer(['orders'], handler, {
|
|
1174
|
+
dlq: true,
|
|
1175
|
+
circuitBreaker: {
|
|
1176
|
+
threshold: 10, // open after 10 failures (default: 5)
|
|
1177
|
+
recoveryMs: 60_000, // wait 60 s before probing (default: 30 s)
|
|
1178
|
+
windowSize: 50, // track last 50 messages (default: threshold × 2, min 10)
|
|
1179
|
+
halfOpenSuccesses: 3, // 3 successes to close (default: 1)
|
|
1180
|
+
},
|
|
1181
|
+
});
|
|
1182
|
+
```
|
|
1183
|
+
|
|
1184
|
+
State machine per `${groupId}:${topic}:${partition}`:
|
|
1185
|
+
|
|
1186
|
+
| State | Behaviour |
|
|
1187
|
+
| ----- | --------- |
|
|
1188
|
+
| **CLOSED** (normal) | Messages delivered. Failures recorded in sliding window. Opens when `failures ≥ threshold`. |
|
|
1189
|
+
| **OPEN** | Partition paused via `pauseConsumer`. After `recoveryMs` ms transitions to HALF_OPEN. |
|
|
1190
|
+
| **HALF_OPEN** | Partition resumed. After `halfOpenSuccesses` consecutive successes the circuit closes. Any single failure immediately re-opens it. |
|
|
1191
|
+
|
|
1192
|
+
Successful `onMessage` completions count as successes. The retry topic path is not subject to the breaker — it has its own backoff and EOS guarantees.
|
|
1193
|
+
|
|
1194
|
+
Options:
|
|
1195
|
+
|
|
1196
|
+
| Option | Default | Description |
|
|
1197
|
+
| ------ | ------- | ----------- |
|
|
1198
|
+
| `threshold` | `5` | DLQ failures within `windowSize` that opens the circuit |
|
|
1199
|
+
| `recoveryMs` | `30_000` | Milliseconds to wait in OPEN state before entering HALF_OPEN |
|
|
1200
|
+
| `windowSize` | `threshold × 2, min 10` | Sliding window size in messages |
|
|
1201
|
+
| `halfOpenSuccesses` | `1` | Consecutive successes in HALF_OPEN required to close the circuit |
|
|
1202
|
+
|
|
1112
1203
|
## Reset consumer offsets
|
|
1113
1204
|
|
|
1114
1205
|
Seek a consumer group's committed offsets to the beginning or end of a topic:
|
|
@@ -1126,6 +1217,46 @@ await kafka.resetOffsets('payments-group', 'orders', 'earliest');
|
|
|
1126
1217
|
|
|
1127
1218
|
**Important:** the consumer for the specified group must be stopped before calling `resetOffsets`. An error is thrown if the group is currently running — this prevents the reset from racing with an active offset commit.
|
|
1128
1219
|
|
|
1220
|
+
## Seek to offset
|
|
1221
|
+
|
|
1222
|
+
Seek individual topic-partitions to explicit offsets — useful when `resetOffsets` is too coarse and you need per-partition control:
|
|
1223
|
+
|
|
1224
|
+
```typescript
|
|
1225
|
+
// Seek partition 0 of 'orders' to offset 100, partition 1 to offset 200
|
|
1226
|
+
await kafka.seekToOffset(undefined, [
|
|
1227
|
+
{ topic: 'orders', partition: 0, offset: '100' },
|
|
1228
|
+
{ topic: 'orders', partition: 1, offset: '200' },
|
|
1229
|
+
]);
|
|
1230
|
+
|
|
1231
|
+
// Multiple topics in one call
|
|
1232
|
+
await kafka.seekToOffset('payments-group', [
|
|
1233
|
+
{ topic: 'payments', partition: 0, offset: '0' },
|
|
1234
|
+
{ topic: 'refunds', partition: 0, offset: '500' },
|
|
1235
|
+
]);
|
|
1236
|
+
```
|
|
1237
|
+
|
|
1238
|
+
The first argument is the consumer group ID — pass `undefined` to target the default group. Assignments are grouped by topic internally so each `admin.setOffsets` call covers all partitions of one topic.
|
|
1239
|
+
|
|
1240
|
+
**Important:** the consumer for the specified group must be stopped before calling `seekToOffset`. An error is thrown if the group is currently running.
|
|
1241
|
+
|
|
1242
|
+
## Message TTL
|
|
1243
|
+
|
|
1244
|
+
Drop or route expired messages using `messageTtlMs` in `ConsumerOptions`:
|
|
1245
|
+
|
|
1246
|
+
```typescript
|
|
1247
|
+
await kafka.startConsumer(['orders'], handler, {
|
|
1248
|
+
messageTtlMs: 60_000, // drop messages older than 60 s
|
|
1249
|
+
dlq: true, // route expired messages to DLQ instead of dropping
|
|
1250
|
+
});
|
|
1251
|
+
```
|
|
1252
|
+
|
|
1253
|
+
The TTL is evaluated against the `x-timestamp` header stamped on every outgoing message by the producer. Messages whose age at consumption time exceeds `messageTtlMs` are:
|
|
1254
|
+
|
|
1255
|
+
- **Routed to DLQ** with `x-dlq-reason: ttl-expired` when `dlq: true`
|
|
1256
|
+
- **Dropped** (calling `onMessageLost`) otherwise
|
|
1257
|
+
|
|
1258
|
+
Typical use: prevent stale events from poisoning downstream systems after a consumer lag spike — e.g. discard order events or push notifications that are no longer actionable.
|
|
1259
|
+
|
|
1129
1260
|
## DLQ replay
|
|
1130
1261
|
|
|
1131
1262
|
Re-publish messages from a dead letter queue back to the original topic:
|
|
@@ -336,9 +336,16 @@ var RETRY_HEADER_MAX_RETRIES = "x-retry-max-retries";
|
|
|
336
336
|
var RETRY_HEADER_ORIGINAL_TOPIC = "x-retry-original-topic";
|
|
337
337
|
function buildRetryTopicPayload(originalTopic, rawMessages, attempt, maxRetries, delayMs, originalHeaders) {
|
|
338
338
|
const retryTopic = `${originalTopic}.retry.${attempt}`;
|
|
339
|
-
const STRIP = /* @__PURE__ */ new Set([
|
|
339
|
+
const STRIP = /* @__PURE__ */ new Set([
|
|
340
|
+
RETRY_HEADER_ATTEMPT,
|
|
341
|
+
RETRY_HEADER_AFTER,
|
|
342
|
+
RETRY_HEADER_MAX_RETRIES,
|
|
343
|
+
RETRY_HEADER_ORIGINAL_TOPIC
|
|
344
|
+
]);
|
|
340
345
|
function buildHeaders(hdr) {
|
|
341
|
-
const userHeaders = Object.fromEntries(
|
|
346
|
+
const userHeaders = Object.fromEntries(
|
|
347
|
+
Object.entries(hdr).filter(([k]) => !STRIP.has(k))
|
|
348
|
+
);
|
|
342
349
|
return {
|
|
343
350
|
...userHeaders,
|
|
344
351
|
[RETRY_HEADER_ATTEMPT]: String(attempt),
|
|
@@ -711,6 +718,31 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
711
718
|
return;
|
|
712
719
|
}
|
|
713
720
|
}
|
|
721
|
+
if (opts.messageTtlMs !== void 0) {
|
|
722
|
+
const ageMs = Date.now() - new Date(envelope.timestamp).getTime();
|
|
723
|
+
if (ageMs > opts.messageTtlMs) {
|
|
724
|
+
deps.logger.warn(
|
|
725
|
+
`[KafkaClient] TTL expired on ${topic2}: age ${ageMs}ms > ${opts.messageTtlMs}ms`
|
|
726
|
+
);
|
|
727
|
+
if (dlq) {
|
|
728
|
+
await sendToDlq(topic2, message.value.toString(), deps, {
|
|
729
|
+
error: new Error(`Message TTL expired: age ${ageMs}ms`),
|
|
730
|
+
attempt: 0,
|
|
731
|
+
originalHeaders: envelope.headers
|
|
732
|
+
});
|
|
733
|
+
deps.onDlq?.(envelope, "ttl-expired");
|
|
734
|
+
} else {
|
|
735
|
+
await deps.onMessageLost?.({
|
|
736
|
+
topic: topic2,
|
|
737
|
+
error: new Error(`TTL expired: ${ageMs}ms`),
|
|
738
|
+
attempt: 0,
|
|
739
|
+
headers: envelope.headers
|
|
740
|
+
});
|
|
741
|
+
}
|
|
742
|
+
await commitOffset?.();
|
|
743
|
+
return;
|
|
744
|
+
}
|
|
745
|
+
}
|
|
714
746
|
await executeWithRetry(
|
|
715
747
|
() => {
|
|
716
748
|
const fn = () => runWithEnvelopeContext(
|
|
@@ -813,6 +845,30 @@ async function handleEachBatch(payload, opts, deps) {
|
|
|
813
845
|
);
|
|
814
846
|
if (isDuplicate) continue;
|
|
815
847
|
}
|
|
848
|
+
if (opts.messageTtlMs !== void 0) {
|
|
849
|
+
const ageMs = Date.now() - new Date(envelope.timestamp).getTime();
|
|
850
|
+
if (ageMs > opts.messageTtlMs) {
|
|
851
|
+
deps.logger.warn(
|
|
852
|
+
`[KafkaClient] TTL expired on ${batch.topic}: age ${ageMs}ms > ${opts.messageTtlMs}ms`
|
|
853
|
+
);
|
|
854
|
+
if (dlq) {
|
|
855
|
+
await sendToDlq(batch.topic, message.value.toString(), deps, {
|
|
856
|
+
error: new Error(`Message TTL expired: age ${ageMs}ms`),
|
|
857
|
+
attempt: 0,
|
|
858
|
+
originalHeaders: envelope.headers
|
|
859
|
+
});
|
|
860
|
+
deps.onDlq?.(envelope, "ttl-expired");
|
|
861
|
+
} else {
|
|
862
|
+
await deps.onMessageLost?.({
|
|
863
|
+
topic: batch.topic,
|
|
864
|
+
error: new Error(`TTL expired: ${ageMs}ms`),
|
|
865
|
+
attempt: 0,
|
|
866
|
+
headers: envelope.headers
|
|
867
|
+
});
|
|
868
|
+
}
|
|
869
|
+
continue;
|
|
870
|
+
}
|
|
871
|
+
}
|
|
816
872
|
envelopes.push(envelope);
|
|
817
873
|
rawMessages.push(message.value.toString());
|
|
818
874
|
}
|
|
@@ -1122,6 +1178,31 @@ async function startRetryTopicConsumers(originalTopics, originalGroupId, handleM
|
|
|
1122
1178
|
// src/client/kafka.client/index.ts
|
|
1123
1179
|
var { Kafka: KafkaClass, logLevel: KafkaLogLevel } = KafkaJS;
|
|
1124
1180
|
var _activeTransactionalIds = /* @__PURE__ */ new Set();
|
|
1181
|
+
var AsyncQueue = class {
|
|
1182
|
+
items = [];
|
|
1183
|
+
waiting = [];
|
|
1184
|
+
closed = false;
|
|
1185
|
+
push(item) {
|
|
1186
|
+
if (this.waiting.length > 0) {
|
|
1187
|
+
this.waiting.shift()({ value: item, done: false });
|
|
1188
|
+
} else {
|
|
1189
|
+
this.items.push(item);
|
|
1190
|
+
}
|
|
1191
|
+
}
|
|
1192
|
+
close() {
|
|
1193
|
+
this.closed = true;
|
|
1194
|
+
for (const r of this.waiting.splice(0)) {
|
|
1195
|
+
r({ value: void 0, done: true });
|
|
1196
|
+
}
|
|
1197
|
+
}
|
|
1198
|
+
next() {
|
|
1199
|
+
if (this.items.length > 0)
|
|
1200
|
+
return Promise.resolve({ value: this.items.shift(), done: false });
|
|
1201
|
+
if (this.closed)
|
|
1202
|
+
return Promise.resolve({ value: void 0, done: true });
|
|
1203
|
+
return new Promise((r) => this.waiting.push(r));
|
|
1204
|
+
}
|
|
1205
|
+
};
|
|
1125
1206
|
var KafkaClient = class _KafkaClient {
|
|
1126
1207
|
kafka;
|
|
1127
1208
|
producer;
|
|
@@ -1155,6 +1236,10 @@ var KafkaClient = class _KafkaClient {
|
|
|
1155
1236
|
_lamportClock = 0;
|
|
1156
1237
|
/** Per-groupId deduplication state: `"topic:partition"` → last processed clock. */
|
|
1157
1238
|
dedupStates = /* @__PURE__ */ new Map();
|
|
1239
|
+
/** Circuit breaker state per `"${gid}:${topic}:${partition}"` key. */
|
|
1240
|
+
circuitStates = /* @__PURE__ */ new Map();
|
|
1241
|
+
/** Circuit breaker config per groupId, set at startConsumer/startBatchConsumer time. */
|
|
1242
|
+
circuitConfigs = /* @__PURE__ */ new Map();
|
|
1158
1243
|
isAdminConnected = false;
|
|
1159
1244
|
inFlightTotal = 0;
|
|
1160
1245
|
drainResolvers = [];
|
|
@@ -1296,7 +1381,9 @@ var KafkaClient = class _KafkaClient {
|
|
|
1296
1381
|
}
|
|
1297
1382
|
const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
|
|
1298
1383
|
const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachMessage", setupOptions);
|
|
1299
|
-
|
|
1384
|
+
if (options.circuitBreaker)
|
|
1385
|
+
this.circuitConfigs.set(gid, options.circuitBreaker);
|
|
1386
|
+
const deps = this.messageDepsFor(gid);
|
|
1300
1387
|
const timeoutMs = options.handlerTimeoutMs;
|
|
1301
1388
|
const deduplication = this.resolveDeduplicationContext(
|
|
1302
1389
|
gid,
|
|
@@ -1322,6 +1409,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1322
1409
|
timeoutMs,
|
|
1323
1410
|
wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this),
|
|
1324
1411
|
deduplication,
|
|
1412
|
+
messageTtlMs: options.messageTtlMs,
|
|
1325
1413
|
eosMainContext
|
|
1326
1414
|
},
|
|
1327
1415
|
deps
|
|
@@ -1362,7 +1450,9 @@ var KafkaClient = class _KafkaClient {
|
|
|
1362
1450
|
}
|
|
1363
1451
|
const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
|
|
1364
1452
|
const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry } = await this.setupConsumer(topics, "eachBatch", setupOptions);
|
|
1365
|
-
|
|
1453
|
+
if (options.circuitBreaker)
|
|
1454
|
+
this.circuitConfigs.set(gid, options.circuitBreaker);
|
|
1455
|
+
const deps = this.messageDepsFor(gid);
|
|
1366
1456
|
const timeoutMs = options.handlerTimeoutMs;
|
|
1367
1457
|
const deduplication = this.resolveDeduplicationContext(
|
|
1368
1458
|
gid,
|
|
@@ -1388,6 +1478,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1388
1478
|
timeoutMs,
|
|
1389
1479
|
wrapWithTimeout: this.wrapWithTimeoutWarning.bind(this),
|
|
1390
1480
|
deduplication,
|
|
1481
|
+
messageTtlMs: options.messageTtlMs,
|
|
1391
1482
|
eosMainContext
|
|
1392
1483
|
},
|
|
1393
1484
|
deps
|
|
@@ -1424,6 +1515,37 @@ var KafkaClient = class _KafkaClient {
|
|
|
1424
1515
|
}
|
|
1425
1516
|
return { groupId: gid, stop: () => this.stopConsumer(gid) };
|
|
1426
1517
|
}
|
|
1518
|
+
/**
|
|
1519
|
+
* Consume messages from a topic as an AsyncIterableIterator.
|
|
1520
|
+
* Use with `for await` — breaking out of the loop automatically stops the consumer.
|
|
1521
|
+
*
|
|
1522
|
+
* @example
|
|
1523
|
+
* for await (const envelope of kafka.consume('my.topic')) {
|
|
1524
|
+
* console.log(envelope.data);
|
|
1525
|
+
* }
|
|
1526
|
+
*/
|
|
1527
|
+
consume(topic2, options) {
|
|
1528
|
+
const queue = new AsyncQueue();
|
|
1529
|
+
const handlePromise = this.startConsumer(
|
|
1530
|
+
[topic2],
|
|
1531
|
+
async (envelope) => {
|
|
1532
|
+
queue.push(envelope);
|
|
1533
|
+
},
|
|
1534
|
+
options
|
|
1535
|
+
);
|
|
1536
|
+
return {
|
|
1537
|
+
[Symbol.asyncIterator]() {
|
|
1538
|
+
return this;
|
|
1539
|
+
},
|
|
1540
|
+
next: () => queue.next(),
|
|
1541
|
+
return: async () => {
|
|
1542
|
+
queue.close();
|
|
1543
|
+
const handle = await handlePromise;
|
|
1544
|
+
await handle.stop();
|
|
1545
|
+
return { value: void 0, done: true };
|
|
1546
|
+
}
|
|
1547
|
+
};
|
|
1548
|
+
}
|
|
1427
1549
|
// ── Consumer lifecycle ───────────────────────────────────────────
|
|
1428
1550
|
async stopConsumer(groupId) {
|
|
1429
1551
|
if (groupId !== void 0) {
|
|
@@ -1444,6 +1566,13 @@ var KafkaClient = class _KafkaClient {
|
|
|
1444
1566
|
this.runningConsumers.delete(groupId);
|
|
1445
1567
|
this.consumerCreationOptions.delete(groupId);
|
|
1446
1568
|
this.dedupStates.delete(groupId);
|
|
1569
|
+
for (const key of [...this.circuitStates.keys()]) {
|
|
1570
|
+
if (key.startsWith(`${groupId}:`)) {
|
|
1571
|
+
clearTimeout(this.circuitStates.get(key).timer);
|
|
1572
|
+
this.circuitStates.delete(key);
|
|
1573
|
+
}
|
|
1574
|
+
}
|
|
1575
|
+
this.circuitConfigs.delete(groupId);
|
|
1447
1576
|
this.logger.log(`Consumer disconnected: group "${groupId}"`);
|
|
1448
1577
|
const mainTxId = `${groupId}-main-tx`;
|
|
1449
1578
|
const mainTxProducer = this.retryTxProducers.get(mainTxId);
|
|
@@ -1504,6 +1633,10 @@ var KafkaClient = class _KafkaClient {
|
|
|
1504
1633
|
this.companionGroupIds.clear();
|
|
1505
1634
|
this.retryTxProducers.clear();
|
|
1506
1635
|
this.dedupStates.clear();
|
|
1636
|
+
for (const state of this.circuitStates.values())
|
|
1637
|
+
clearTimeout(state.timer);
|
|
1638
|
+
this.circuitStates.clear();
|
|
1639
|
+
this.circuitConfigs.clear();
|
|
1507
1640
|
this.logger.log("All consumers disconnected");
|
|
1508
1641
|
}
|
|
1509
1642
|
}
|
|
@@ -1577,9 +1710,7 @@ var KafkaClient = class _KafkaClient {
|
|
|
1577
1710
|
this.consumerCreationOptions.delete(tempGroupId);
|
|
1578
1711
|
});
|
|
1579
1712
|
};
|
|
1580
|
-
consumer.connect().then(
|
|
1581
|
-
() => subscribeWithRetry(consumer, [dlqTopic], this.logger)
|
|
1582
|
-
).then(
|
|
1713
|
+
consumer.connect().then(() => subscribeWithRetry(consumer, [dlqTopic], this.logger)).then(
|
|
1583
1714
|
() => consumer.run({
|
|
1584
1715
|
eachMessage: async ({ partition, message }) => {
|
|
1585
1716
|
if (!message.value) return;
|
|
@@ -1645,6 +1776,32 @@ var KafkaClient = class _KafkaClient {
|
|
|
1645
1776
|
`Offsets reset to ${position} for group "${gid}" on topic "${topic2}"`
|
|
1646
1777
|
);
|
|
1647
1778
|
}
|
|
1779
|
+
/**
|
|
1780
|
+
* Seek specific topic-partition pairs to explicit offsets for a stopped consumer group.
|
|
1781
|
+
* Throws if the group is still running — call `stopConsumer(groupId)` first.
|
|
1782
|
+
* Assignments are grouped by topic and committed via `admin.setOffsets`.
|
|
1783
|
+
*/
|
|
1784
|
+
async seekToOffset(groupId, assignments) {
|
|
1785
|
+
const gid = groupId ?? this.defaultGroupId;
|
|
1786
|
+
if (this.runningConsumers.has(gid)) {
|
|
1787
|
+
throw new Error(
|
|
1788
|
+
`seekToOffset: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before seeking offsets.`
|
|
1789
|
+
);
|
|
1790
|
+
}
|
|
1791
|
+
await this.ensureAdminConnected();
|
|
1792
|
+
const byTopic = /* @__PURE__ */ new Map();
|
|
1793
|
+
for (const { topic: topic2, partition, offset } of assignments) {
|
|
1794
|
+
const list = byTopic.get(topic2) ?? [];
|
|
1795
|
+
list.push({ partition, offset });
|
|
1796
|
+
byTopic.set(topic2, list);
|
|
1797
|
+
}
|
|
1798
|
+
for (const [topic2, partitions] of byTopic) {
|
|
1799
|
+
await this.admin.setOffsets({ groupId: gid, topic: topic2, partitions });
|
|
1800
|
+
this.logger.log(
|
|
1801
|
+
`Offsets set for group "${gid}" on "${topic2}": ${JSON.stringify(partitions)}`
|
|
1802
|
+
);
|
|
1803
|
+
}
|
|
1804
|
+
}
|
|
1648
1805
|
/**
|
|
1649
1806
|
* Query consumer group lag per partition.
|
|
1650
1807
|
* Lag = broker high-watermark − last committed offset.
|
|
@@ -1750,6 +1907,9 @@ var KafkaClient = class _KafkaClient {
|
|
|
1750
1907
|
this.runningConsumers.clear();
|
|
1751
1908
|
this.consumerCreationOptions.clear();
|
|
1752
1909
|
this.companionGroupIds.clear();
|
|
1910
|
+
for (const state of this.circuitStates.values()) clearTimeout(state.timer);
|
|
1911
|
+
this.circuitStates.clear();
|
|
1912
|
+
this.circuitConfigs.clear();
|
|
1753
1913
|
this.logger.log("All connections closed");
|
|
1754
1914
|
}
|
|
1755
1915
|
// ── Graceful shutdown ────────────────────────────────────────────
|
|
@@ -1843,11 +2003,59 @@ var KafkaClient = class _KafkaClient {
|
|
|
1843
2003
|
inst.onRetry?.(envelope, attempt, maxRetries);
|
|
1844
2004
|
}
|
|
1845
2005
|
}
|
|
1846
|
-
notifyDlq(envelope, reason) {
|
|
2006
|
+
notifyDlq(envelope, reason, gid) {
|
|
1847
2007
|
this.metricsFor(envelope.topic).dlqCount++;
|
|
1848
2008
|
for (const inst of this.instrumentation) {
|
|
1849
2009
|
inst.onDlq?.(envelope, reason);
|
|
1850
2010
|
}
|
|
2011
|
+
if (!gid) return;
|
|
2012
|
+
const cfg = this.circuitConfigs.get(gid);
|
|
2013
|
+
if (!cfg) return;
|
|
2014
|
+
const threshold = cfg.threshold ?? 5;
|
|
2015
|
+
const recoveryMs = cfg.recoveryMs ?? 3e4;
|
|
2016
|
+
const stateKey = `${gid}:${envelope.topic}:${envelope.partition}`;
|
|
2017
|
+
let state = this.circuitStates.get(stateKey);
|
|
2018
|
+
if (!state) {
|
|
2019
|
+
state = { status: "closed", window: [], successes: 0 };
|
|
2020
|
+
this.circuitStates.set(stateKey, state);
|
|
2021
|
+
}
|
|
2022
|
+
if (state.status === "open") return;
|
|
2023
|
+
const openCircuit = () => {
|
|
2024
|
+
state.status = "open";
|
|
2025
|
+
this.pauseConsumer(gid, [
|
|
2026
|
+
{ topic: envelope.topic, partitions: [envelope.partition] }
|
|
2027
|
+
]);
|
|
2028
|
+
state.timer = setTimeout(() => {
|
|
2029
|
+
state.status = "half-open";
|
|
2030
|
+
state.successes = 0;
|
|
2031
|
+
this.logger.log(
|
|
2032
|
+
`[CircuitBreaker] HALF-OPEN \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
2033
|
+
);
|
|
2034
|
+
this.resumeConsumer(gid, [
|
|
2035
|
+
{ topic: envelope.topic, partitions: [envelope.partition] }
|
|
2036
|
+
]);
|
|
2037
|
+
}, recoveryMs);
|
|
2038
|
+
};
|
|
2039
|
+
if (state.status === "half-open") {
|
|
2040
|
+
clearTimeout(state.timer);
|
|
2041
|
+
this.logger.warn(
|
|
2042
|
+
`[CircuitBreaker] OPEN (half-open failure) \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
2043
|
+
);
|
|
2044
|
+
openCircuit();
|
|
2045
|
+
return;
|
|
2046
|
+
}
|
|
2047
|
+
const windowSize = cfg.windowSize ?? Math.max(threshold * 2, 10);
|
|
2048
|
+
state.window = [...state.window, false];
|
|
2049
|
+
if (state.window.length > windowSize) {
|
|
2050
|
+
state.window = state.window.slice(state.window.length - windowSize);
|
|
2051
|
+
}
|
|
2052
|
+
const failures = state.window.filter((v) => !v).length;
|
|
2053
|
+
if (failures >= threshold) {
|
|
2054
|
+
this.logger.warn(
|
|
2055
|
+
`[CircuitBreaker] OPEN \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition} (${failures}/${state.window.length} failures, threshold=${threshold})`
|
|
2056
|
+
);
|
|
2057
|
+
openCircuit();
|
|
2058
|
+
}
|
|
1851
2059
|
}
|
|
1852
2060
|
notifyDuplicate(envelope, strategy) {
|
|
1853
2061
|
this.metricsFor(envelope.topic).dedupCount++;
|
|
@@ -1855,11 +2063,38 @@ var KafkaClient = class _KafkaClient {
|
|
|
1855
2063
|
inst.onDuplicate?.(envelope, strategy);
|
|
1856
2064
|
}
|
|
1857
2065
|
}
|
|
1858
|
-
notifyMessage(envelope) {
|
|
2066
|
+
notifyMessage(envelope, gid) {
|
|
1859
2067
|
this.metricsFor(envelope.topic).processedCount++;
|
|
1860
2068
|
for (const inst of this.instrumentation) {
|
|
1861
2069
|
inst.onMessage?.(envelope);
|
|
1862
2070
|
}
|
|
2071
|
+
if (!gid) return;
|
|
2072
|
+
const cfg = this.circuitConfigs.get(gid);
|
|
2073
|
+
if (!cfg) return;
|
|
2074
|
+
const stateKey = `${gid}:${envelope.topic}:${envelope.partition}`;
|
|
2075
|
+
const state = this.circuitStates.get(stateKey);
|
|
2076
|
+
if (!state) return;
|
|
2077
|
+
const halfOpenSuccesses = cfg.halfOpenSuccesses ?? 1;
|
|
2078
|
+
if (state.status === "half-open") {
|
|
2079
|
+
state.successes++;
|
|
2080
|
+
if (state.successes >= halfOpenSuccesses) {
|
|
2081
|
+
clearTimeout(state.timer);
|
|
2082
|
+
state.timer = void 0;
|
|
2083
|
+
state.status = "closed";
|
|
2084
|
+
state.window = [];
|
|
2085
|
+
state.successes = 0;
|
|
2086
|
+
this.logger.log(
|
|
2087
|
+
`[CircuitBreaker] CLOSED \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
2088
|
+
);
|
|
2089
|
+
}
|
|
2090
|
+
} else if (state.status === "closed") {
|
|
2091
|
+
const threshold = cfg.threshold ?? 5;
|
|
2092
|
+
const windowSize = cfg.windowSize ?? Math.max(threshold * 2, 10);
|
|
2093
|
+
state.window = [...state.window, true];
|
|
2094
|
+
if (state.window.length > windowSize) {
|
|
2095
|
+
state.window = state.window.slice(state.window.length - windowSize);
|
|
2096
|
+
}
|
|
2097
|
+
}
|
|
1863
2098
|
}
|
|
1864
2099
|
/**
|
|
1865
2100
|
* Start a timer that logs a warning if `fn` hasn't resolved within `timeoutMs`.
|
|
@@ -2080,16 +2315,17 @@ var KafkaClient = class _KafkaClient {
|
|
|
2080
2315
|
logger: this.logger
|
|
2081
2316
|
};
|
|
2082
2317
|
}
|
|
2083
|
-
|
|
2318
|
+
/** Build MessageHandlerDeps with circuit breaker callbacks bound to the given groupId. */
|
|
2319
|
+
messageDepsFor(gid) {
|
|
2084
2320
|
return {
|
|
2085
2321
|
logger: this.logger,
|
|
2086
2322
|
producer: this.producer,
|
|
2087
2323
|
instrumentation: this.instrumentation,
|
|
2088
2324
|
onMessageLost: this.onMessageLost,
|
|
2089
2325
|
onRetry: this.notifyRetry.bind(this),
|
|
2090
|
-
onDlq: this.notifyDlq
|
|
2326
|
+
onDlq: (envelope, reason) => this.notifyDlq(envelope, reason, gid),
|
|
2091
2327
|
onDuplicate: this.notifyDuplicate.bind(this),
|
|
2092
|
-
onMessage: this.notifyMessage
|
|
2328
|
+
onMessage: (envelope) => this.notifyMessage(envelope, gid)
|
|
2093
2329
|
};
|
|
2094
2330
|
}
|
|
2095
2331
|
get retryTopicDeps() {
|
|
@@ -2143,4 +2379,4 @@ export {
|
|
|
2143
2379
|
KafkaClient,
|
|
2144
2380
|
topic
|
|
2145
2381
|
};
|
|
2146
|
-
//# sourceMappingURL=chunk-
|
|
2382
|
+
//# sourceMappingURL=chunk-MJ342P4R.mjs.map
|