@drarzter/kafka-client 0.9.3 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +625 -8
- package/dist/chunk-CMO7SMVK.mjs +4814 -0
- package/dist/chunk-CMO7SMVK.mjs.map +1 -0
- package/dist/cli/dlq.d.ts +119 -0
- package/dist/cli/dlq.d.ts.map +1 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/{chunk-TPIP5VV7.mjs → cli/index.js} +965 -265
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/index.mjs +355 -0
- package/dist/cli/index.mjs.map +1 -0
- package/dist/client/config/from-env.d.ts +188 -0
- package/dist/client/config/from-env.d.ts.map +1 -0
- package/dist/client/config/index.d.ts +2 -0
- package/dist/client/config/index.d.ts.map +1 -0
- package/dist/client/errors.d.ts +67 -0
- package/dist/client/errors.d.ts.map +1 -0
- package/dist/client/kafka.client/admin/ops.d.ts +114 -0
- package/dist/client/kafka.client/admin/ops.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/delayed.d.ts +24 -0
- package/dist/client/kafka.client/consumer/features/delayed.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/dlq-replay.d.ts +52 -0
- package/dist/client/kafka.client/consumer/features/dlq-replay.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/routed.d.ts +4 -0
- package/dist/client/kafka.client/consumer/features/routed.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/snapshot.d.ts +10 -0
- package/dist/client/kafka.client/consumer/features/snapshot.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/window.d.ts +5 -0
- package/dist/client/kafka.client/consumer/features/window.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/handler.d.ts +149 -0
- package/dist/client/kafka.client/consumer/handler.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/ops.d.ts +51 -0
- package/dist/client/kafka.client/consumer/ops.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/pipeline.d.ts +167 -0
- package/dist/client/kafka.client/consumer/pipeline.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/queue.d.ts +37 -0
- package/dist/client/kafka.client/consumer/queue.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/retry-topic.d.ts +65 -0
- package/dist/client/kafka.client/consumer/retry-topic.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/setup.d.ts +63 -0
- package/dist/client/kafka.client/consumer/setup.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/start.d.ts +7 -0
- package/dist/client/kafka.client/consumer/start.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/stop.d.ts +19 -0
- package/dist/client/kafka.client/consumer/stop.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/subscribe-retry.d.ts +4 -0
- package/dist/client/kafka.client/consumer/subscribe-retry.d.ts.map +1 -0
- package/dist/client/kafka.client/context.d.ts +72 -0
- package/dist/client/kafka.client/context.d.ts.map +1 -0
- package/dist/client/kafka.client/index.d.ts +155 -0
- package/dist/client/kafka.client/index.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/circuit-breaker.manager.d.ts +61 -0
- package/dist/client/kafka.client/infra/circuit-breaker.manager.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/dedup.store.d.ts +28 -0
- package/dist/client/kafka.client/infra/dedup.store.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/inflight.tracker.d.ts +22 -0
- package/dist/client/kafka.client/infra/inflight.tracker.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/metrics.manager.d.ts +67 -0
- package/dist/client/kafka.client/infra/metrics.manager.d.ts.map +1 -0
- package/dist/client/kafka.client/producer/lifecycle.d.ts +41 -0
- package/dist/client/kafka.client/producer/lifecycle.d.ts.map +1 -0
- package/dist/client/kafka.client/producer/ops.d.ts +70 -0
- package/dist/client/kafka.client/producer/ops.d.ts.map +1 -0
- package/dist/client/kafka.client/producer/send.d.ts +21 -0
- package/dist/client/kafka.client/producer/send.d.ts.map +1 -0
- package/dist/client/kafka.client/validate-options.d.ts +11 -0
- package/dist/client/kafka.client/validate-options.d.ts.map +1 -0
- package/dist/client/message/envelope.d.ts +105 -0
- package/dist/client/message/envelope.d.ts.map +1 -0
- package/dist/client/message/schema-registry.d.ts +105 -0
- package/dist/client/message/schema-registry.d.ts.map +1 -0
- package/dist/client/message/topic.d.ts +138 -0
- package/dist/client/message/topic.d.ts.map +1 -0
- package/dist/client/message/versioned-schema.d.ts +53 -0
- package/dist/client/message/versioned-schema.d.ts.map +1 -0
- package/dist/client/outbox/index.d.ts +4 -0
- package/dist/client/outbox/index.d.ts.map +1 -0
- package/dist/client/outbox/outbox.relay.d.ts +90 -0
- package/dist/client/outbox/outbox.relay.d.ts.map +1 -0
- package/dist/client/outbox/outbox.store.d.ts +42 -0
- package/dist/client/outbox/outbox.store.d.ts.map +1 -0
- package/dist/client/outbox/outbox.types.d.ts +144 -0
- package/dist/client/outbox/outbox.types.d.ts.map +1 -0
- package/dist/client/security/acl.d.ts +108 -0
- package/dist/client/security/acl.d.ts.map +1 -0
- package/dist/client/security/index.d.ts +5 -0
- package/dist/client/security/index.d.ts.map +1 -0
- package/dist/client/security/providers.d.ts +88 -0
- package/dist/client/security/providers.d.ts.map +1 -0
- package/dist/client/security/resolve-security.d.ts +19 -0
- package/dist/client/security/resolve-security.d.ts.map +1 -0
- package/dist/client/security/security.types.d.ts +76 -0
- package/dist/client/security/security.types.d.ts.map +1 -0
- package/dist/client/transport/confluent.transport.d.ts +32 -0
- package/dist/client/transport/confluent.transport.d.ts.map +1 -0
- package/dist/client/transport/transport.interface.d.ts +216 -0
- package/dist/client/transport/transport.interface.d.ts.map +1 -0
- package/dist/client/types/admin.interface.d.ts +174 -0
- package/dist/client/types/admin.interface.d.ts.map +1 -0
- package/dist/client/types/admin.types.d.ts +140 -0
- package/dist/client/types/admin.types.d.ts.map +1 -0
- package/dist/client/types/client.d.ts +21 -0
- package/dist/client/types/client.d.ts.map +1 -0
- package/dist/client/types/common.d.ts +84 -0
- package/dist/client/types/common.d.ts.map +1 -0
- package/dist/client/types/config.types.d.ts +150 -0
- package/dist/client/types/config.types.d.ts.map +1 -0
- package/dist/client/types/consumer.interface.d.ts +115 -0
- package/dist/client/types/consumer.interface.d.ts.map +1 -0
- package/dist/{consumer.types-fFCag3VJ.d.mts → client/types/consumer.types.d.ts} +62 -383
- package/dist/client/types/consumer.types.d.ts.map +1 -0
- package/dist/client/types/dedup.types.d.ts +50 -0
- package/dist/client/types/dedup.types.d.ts.map +1 -0
- package/dist/client/types/lifecycle.interface.d.ts +72 -0
- package/dist/client/types/lifecycle.interface.d.ts.map +1 -0
- package/dist/client/types/producer.interface.d.ts +52 -0
- package/dist/client/types/producer.interface.d.ts.map +1 -0
- package/dist/client/types/producer.types.d.ts +90 -0
- package/dist/client/types/producer.types.d.ts.map +1 -0
- package/dist/client/types.d.ts +8 -0
- package/dist/client/types.d.ts.map +1 -0
- package/dist/core.d.ts +10 -314
- package/dist/core.d.ts.map +1 -0
- package/dist/core.js +1326 -74
- package/dist/core.js.map +1 -1
- package/dist/core.mjs +39 -3
- package/dist/index.d.ts +7 -128
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1343 -74
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +56 -3
- package/dist/index.mjs.map +1 -1
- package/dist/nest/kafka.constants.d.ts +5 -0
- package/dist/nest/kafka.constants.d.ts.map +1 -0
- package/dist/nest/kafka.decorator.d.ts +49 -0
- package/dist/nest/kafka.decorator.d.ts.map +1 -0
- package/dist/nest/kafka.explorer.d.ts +17 -0
- package/dist/nest/kafka.explorer.d.ts.map +1 -0
- package/dist/nest/kafka.health.d.ts +7 -0
- package/dist/nest/kafka.health.d.ts.map +1 -0
- package/dist/nest/kafka.module.d.ts +61 -0
- package/dist/nest/kafka.module.d.ts.map +1 -0
- package/dist/otel.d.ts +83 -5
- package/dist/otel.d.ts.map +1 -0
- package/dist/otel.js +100 -6
- package/dist/otel.js.map +1 -1
- package/dist/otel.mjs +98 -5
- package/dist/otel.mjs.map +1 -1
- package/dist/testing/client.mock.d.ts +47 -0
- package/dist/testing/client.mock.d.ts.map +1 -0
- package/dist/testing/index.d.ts +4 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/test.container.d.ts +63 -0
- package/dist/testing/test.container.d.ts.map +1 -0
- package/dist/{testing.d.mts → testing/transport.fake.d.ts} +7 -111
- package/dist/testing/transport.fake.d.ts.map +1 -0
- package/dist/testing.d.ts +2 -318
- package/dist/testing.d.ts.map +1 -0
- package/dist/testing.js +28 -2
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +28 -2
- package/dist/testing.mjs.map +1 -1
- package/package.json +22 -9
- package/dist/chunk-TPIP5VV7.mjs.map +0 -1
- package/dist/client-CBBUDDtu.d.ts +0 -751
- package/dist/client-D-SxYV2b.d.mts +0 -751
- package/dist/consumer.types-fFCag3VJ.d.ts +0 -958
- package/dist/core.d.mts +0 -314
- package/dist/index.d.mts +0 -128
- package/dist/otel.d.mts +0 -27
|
@@ -1,10 +1,14 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
|
|
4
|
+
// src/client/transport/confluent.transport.ts
|
|
5
|
+
var import_kafka_javascript = require("@confluentinc/kafka-javascript");
|
|
6
|
+
var { Kafka: KafkaClass, logLevel: KafkaLogLevel, PartitionAssigners } = import_kafka_javascript.KafkaJS;
|
|
4
7
|
var ConfluentTransaction = class {
|
|
5
8
|
constructor(tx) {
|
|
6
9
|
this.tx = tx;
|
|
7
10
|
}
|
|
11
|
+
tx;
|
|
8
12
|
async send(record) {
|
|
9
13
|
await this.tx.send(record);
|
|
10
14
|
}
|
|
@@ -26,10 +30,17 @@ var ConfluentProducer = class {
|
|
|
26
30
|
constructor(producer) {
|
|
27
31
|
this.producer = producer;
|
|
28
32
|
}
|
|
33
|
+
producer;
|
|
34
|
+
connectPromise;
|
|
29
35
|
async connect() {
|
|
30
|
-
|
|
36
|
+
this.connectPromise ??= this.producer.connect().catch((err) => {
|
|
37
|
+
this.connectPromise = void 0;
|
|
38
|
+
throw err;
|
|
39
|
+
});
|
|
40
|
+
return this.connectPromise;
|
|
31
41
|
}
|
|
32
42
|
async disconnect() {
|
|
43
|
+
this.connectPromise = void 0;
|
|
33
44
|
await this.producer.disconnect();
|
|
34
45
|
}
|
|
35
46
|
async send(record) {
|
|
@@ -44,6 +55,7 @@ var ConfluentConsumer = class {
|
|
|
44
55
|
constructor(consumer) {
|
|
45
56
|
this.consumer = consumer;
|
|
46
57
|
}
|
|
58
|
+
consumer;
|
|
47
59
|
/** Returns the underlying KafkaJS.Consumer — used by ConfluentTransaction.sendOffsets. */
|
|
48
60
|
getNative() {
|
|
49
61
|
return this.consumer;
|
|
@@ -83,6 +95,7 @@ var ConfluentAdmin = class {
|
|
|
83
95
|
constructor(admin) {
|
|
84
96
|
this.admin = admin;
|
|
85
97
|
}
|
|
98
|
+
admin;
|
|
86
99
|
async connect() {
|
|
87
100
|
await this.admin.connect();
|
|
88
101
|
}
|
|
@@ -92,11 +105,11 @@ var ConfluentAdmin = class {
|
|
|
92
105
|
async createTopics(options) {
|
|
93
106
|
await this.admin.createTopics(options);
|
|
94
107
|
}
|
|
95
|
-
async fetchTopicOffsets(
|
|
96
|
-
return this.admin.fetchTopicOffsets(
|
|
108
|
+
async fetchTopicOffsets(topic) {
|
|
109
|
+
return this.admin.fetchTopicOffsets(topic);
|
|
97
110
|
}
|
|
98
|
-
async fetchTopicOffsetsByTimestamp(
|
|
99
|
-
return this.admin.
|
|
111
|
+
async fetchTopicOffsetsByTimestamp(topic, timestamp) {
|
|
112
|
+
return this.admin.fetchTopicOffsetsByTimestamp(topic, timestamp);
|
|
100
113
|
}
|
|
101
114
|
async fetchOffsets(options) {
|
|
102
115
|
return this.admin.fetchOffsets(options);
|
|
@@ -122,10 +135,29 @@ var ConfluentAdmin = class {
|
|
|
122
135
|
};
|
|
123
136
|
var ConfluentTransport = class {
|
|
124
137
|
kafka;
|
|
125
|
-
constructor(clientId, brokers) {
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
138
|
+
constructor(clientId, brokers, security) {
|
|
139
|
+
const kafkaJS = { clientId, brokers, logLevel: KafkaLogLevel.ERROR };
|
|
140
|
+
if (security?.ssl !== void 0) kafkaJS.ssl = security.ssl;
|
|
141
|
+
if (security?.sasl) {
|
|
142
|
+
if (security.sasl.mechanism === "oauthbearer") {
|
|
143
|
+
const provider = security.sasl.oauthBearerProvider;
|
|
144
|
+
kafkaJS.sasl = {
|
|
145
|
+
mechanism: "oauthbearer",
|
|
146
|
+
oauthBearerProvider: async () => {
|
|
147
|
+
const token = await provider();
|
|
148
|
+
return {
|
|
149
|
+
value: token.value,
|
|
150
|
+
principal: token.principal ?? "kafka-client",
|
|
151
|
+
lifetime: token.lifetimeMs ?? Date.now() + 15 * 6e4,
|
|
152
|
+
...token.extensions && { extensions: token.extensions }
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
} else {
|
|
157
|
+
kafkaJS.sasl = security.sasl;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
this.kafka = new KafkaClass({ kafkaJS });
|
|
129
161
|
}
|
|
130
162
|
producer(options) {
|
|
131
163
|
const native = this.kafka.producer({
|
|
@@ -152,6 +184,9 @@ var ConfluentTransport = class {
|
|
|
152
184
|
partitionAssigners: [assigner]
|
|
153
185
|
}
|
|
154
186
|
};
|
|
187
|
+
if (options.groupInstanceId) {
|
|
188
|
+
config["group.instance.id"] = options.groupInstanceId;
|
|
189
|
+
}
|
|
155
190
|
if (options.onRebalance) {
|
|
156
191
|
const cb = options.onRebalance;
|
|
157
192
|
config.rebalance_cb = (err, assignment) => {
|
|
@@ -169,16 +204,37 @@ var ConfluentTransport = class {
|
|
|
169
204
|
}
|
|
170
205
|
};
|
|
171
206
|
|
|
207
|
+
// src/client/kafka.client/infra/dedup.store.ts
|
|
208
|
+
var InMemoryDedupStore = class {
|
|
209
|
+
constructor(states) {
|
|
210
|
+
this.states = states;
|
|
211
|
+
}
|
|
212
|
+
states;
|
|
213
|
+
getLastClock(groupId, topicPartition) {
|
|
214
|
+
return this.states.get(groupId)?.get(topicPartition);
|
|
215
|
+
}
|
|
216
|
+
setLastClock(groupId, topicPartition, clock) {
|
|
217
|
+
let group = this.states.get(groupId);
|
|
218
|
+
if (!group) {
|
|
219
|
+
group = /* @__PURE__ */ new Map();
|
|
220
|
+
this.states.set(groupId, group);
|
|
221
|
+
}
|
|
222
|
+
group.set(topicPartition, clock);
|
|
223
|
+
}
|
|
224
|
+
};
|
|
225
|
+
|
|
172
226
|
// src/client/message/envelope.ts
|
|
173
|
-
|
|
174
|
-
|
|
227
|
+
var import_node_async_hooks = require("async_hooks");
|
|
228
|
+
var import_node_crypto = require("crypto");
|
|
175
229
|
var HEADER_EVENT_ID = "x-event-id";
|
|
176
230
|
var HEADER_CORRELATION_ID = "x-correlation-id";
|
|
177
231
|
var HEADER_TIMESTAMP = "x-timestamp";
|
|
178
232
|
var HEADER_SCHEMA_VERSION = "x-schema-version";
|
|
179
233
|
var HEADER_TRACEPARENT = "traceparent";
|
|
180
234
|
var HEADER_LAMPORT_CLOCK = "x-lamport-clock";
|
|
181
|
-
var
|
|
235
|
+
var HEADER_DELAYED_UNTIL = "x-delayed-until";
|
|
236
|
+
var HEADER_DELAYED_TARGET = "x-delayed-target";
|
|
237
|
+
var envelopeStorage = new import_node_async_hooks.AsyncLocalStorage();
|
|
182
238
|
function getEnvelopeContext() {
|
|
183
239
|
return envelopeStorage.getStore();
|
|
184
240
|
}
|
|
@@ -187,8 +243,8 @@ function runWithEnvelopeContext(ctx, fn) {
|
|
|
187
243
|
}
|
|
188
244
|
function buildEnvelopeHeaders(options = {}) {
|
|
189
245
|
const ctx = getEnvelopeContext();
|
|
190
|
-
const correlationId = options.correlationId ?? ctx?.correlationId ?? randomUUID();
|
|
191
|
-
const eventId = options.eventId ?? randomUUID();
|
|
246
|
+
const correlationId = options.correlationId ?? ctx?.correlationId ?? (0, import_node_crypto.randomUUID)();
|
|
247
|
+
const eventId = options.eventId ?? (0, import_node_crypto.randomUUID)();
|
|
192
248
|
const timestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
193
249
|
const schemaVersion = String(options.schemaVersion ?? 1);
|
|
194
250
|
const envelope = {
|
|
@@ -216,14 +272,14 @@ function decodeHeaders(raw) {
|
|
|
216
272
|
}
|
|
217
273
|
return result;
|
|
218
274
|
}
|
|
219
|
-
function extractEnvelope(payload, headers,
|
|
275
|
+
function extractEnvelope(payload, headers, topic, partition, offset) {
|
|
220
276
|
return {
|
|
221
277
|
payload,
|
|
222
|
-
topic
|
|
278
|
+
topic,
|
|
223
279
|
partition,
|
|
224
280
|
offset,
|
|
225
|
-
eventId: headers[HEADER_EVENT_ID] ?? randomUUID(),
|
|
226
|
-
correlationId: headers[HEADER_CORRELATION_ID] ?? randomUUID(),
|
|
281
|
+
eventId: headers[HEADER_EVENT_ID] ?? (0, import_node_crypto.randomUUID)(),
|
|
282
|
+
correlationId: headers[HEADER_CORRELATION_ID] ?? (0, import_node_crypto.randomUUID)(),
|
|
227
283
|
timestamp: headers[HEADER_TIMESTAMP] ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
228
284
|
schemaVersion: Number(headers[HEADER_SCHEMA_VERSION] ?? 1),
|
|
229
285
|
traceparent: headers[HEADER_TRACEPARENT],
|
|
@@ -232,35 +288,43 @@ function extractEnvelope(payload, headers, topic2, partition, offset) {
|
|
|
232
288
|
}
|
|
233
289
|
|
|
234
290
|
// src/client/errors.ts
|
|
291
|
+
function toError(error) {
|
|
292
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
293
|
+
}
|
|
235
294
|
var KafkaProcessingError = class extends Error {
|
|
236
|
-
constructor(message,
|
|
295
|
+
constructor(message, topic, originalMessage, options) {
|
|
237
296
|
super(message, options);
|
|
238
|
-
this.topic =
|
|
297
|
+
this.topic = topic;
|
|
239
298
|
this.originalMessage = originalMessage;
|
|
240
299
|
this.name = "KafkaProcessingError";
|
|
241
300
|
if (options?.cause) this.cause = options.cause;
|
|
242
301
|
}
|
|
302
|
+
topic;
|
|
303
|
+
originalMessage;
|
|
243
304
|
};
|
|
244
305
|
var KafkaValidationError = class extends Error {
|
|
245
|
-
constructor(
|
|
246
|
-
super(`Schema validation failed for topic "${
|
|
247
|
-
this.topic =
|
|
306
|
+
constructor(topic, originalMessage, options) {
|
|
307
|
+
super(`Schema validation failed for topic "${topic}"`, options);
|
|
308
|
+
this.topic = topic;
|
|
248
309
|
this.originalMessage = originalMessage;
|
|
249
310
|
this.name = "KafkaValidationError";
|
|
250
311
|
if (options?.cause) this.cause = options.cause;
|
|
251
312
|
}
|
|
313
|
+
topic;
|
|
314
|
+
originalMessage;
|
|
252
315
|
};
|
|
253
316
|
var KafkaRetryExhaustedError = class extends KafkaProcessingError {
|
|
254
|
-
constructor(
|
|
317
|
+
constructor(topic, originalMessage, attempts, options) {
|
|
255
318
|
super(
|
|
256
|
-
`Message processing failed after ${attempts} attempts on topic "${
|
|
257
|
-
|
|
319
|
+
`Message processing failed after ${attempts} attempts on topic "${topic}"`,
|
|
320
|
+
topic,
|
|
258
321
|
originalMessage,
|
|
259
322
|
options
|
|
260
323
|
);
|
|
261
324
|
this.attempts = attempts;
|
|
262
325
|
this.name = "KafkaRetryExhaustedError";
|
|
263
326
|
}
|
|
327
|
+
attempts;
|
|
264
328
|
};
|
|
265
329
|
|
|
266
330
|
// src/client/kafka.client/producer/ops.ts
|
|
@@ -273,14 +337,14 @@ function resolveTopicName(topicOrDescriptor) {
|
|
|
273
337
|
}
|
|
274
338
|
function registerSchema(topicOrDesc, schemaRegistry, logger) {
|
|
275
339
|
if (topicOrDesc?.__schema) {
|
|
276
|
-
const
|
|
277
|
-
const existing = schemaRegistry.get(
|
|
340
|
+
const topic = resolveTopicName(topicOrDesc);
|
|
341
|
+
const existing = schemaRegistry.get(topic);
|
|
278
342
|
if (existing && existing !== topicOrDesc.__schema) {
|
|
279
343
|
logger?.warn(
|
|
280
|
-
`Schema conflict for topic "${
|
|
344
|
+
`Schema conflict for topic "${topic}": a different schema is already registered. Using the new schema \u2014 ensure consistent schemas to avoid silent validation mismatches.`
|
|
281
345
|
);
|
|
282
346
|
}
|
|
283
|
-
schemaRegistry.set(
|
|
347
|
+
schemaRegistry.set(topic, topicOrDesc.__schema);
|
|
284
348
|
}
|
|
285
349
|
}
|
|
286
350
|
async function validateMessage(topicOrDesc, message, deps, ctx) {
|
|
@@ -309,7 +373,7 @@ async function validateMessage(topicOrDesc, message, deps, ctx) {
|
|
|
309
373
|
return message;
|
|
310
374
|
}
|
|
311
375
|
async function buildSendPayload(topicOrDesc, messages, deps, compression) {
|
|
312
|
-
const
|
|
376
|
+
const topic = resolveTopicName(topicOrDesc);
|
|
313
377
|
const builtMessages = await Promise.all(
|
|
314
378
|
messages.map(async (m) => {
|
|
315
379
|
const envelopeHeaders = buildEnvelopeHeaders({
|
|
@@ -322,10 +386,10 @@ async function buildSendPayload(topicOrDesc, messages, deps, compression) {
|
|
|
322
386
|
envelopeHeaders[HEADER_LAMPORT_CLOCK] = String(deps.nextLamportClock());
|
|
323
387
|
}
|
|
324
388
|
for (const inst of deps.instrumentation) {
|
|
325
|
-
inst.beforeSend?.(
|
|
389
|
+
inst.beforeSend?.(topic, envelopeHeaders);
|
|
326
390
|
}
|
|
327
391
|
const sendCtx = {
|
|
328
|
-
topic
|
|
392
|
+
topic,
|
|
329
393
|
headers: envelopeHeaders,
|
|
330
394
|
version: m.schemaVersion ?? 1
|
|
331
395
|
};
|
|
@@ -333,16 +397,18 @@ async function buildSendPayload(topicOrDesc, messages, deps, compression) {
|
|
|
333
397
|
value: JSON.stringify(
|
|
334
398
|
await validateMessage(topicOrDesc, m.value, deps, sendCtx)
|
|
335
399
|
),
|
|
336
|
-
key
|
|
400
|
+
// Explicit key wins; otherwise fall back to the descriptor's .key()
|
|
401
|
+
// extractor (runs on the original, pre-validation payload).
|
|
402
|
+
key: m.key ?? topicOrDesc?.__key?.(m.value) ?? null,
|
|
337
403
|
headers: envelopeHeaders
|
|
338
404
|
};
|
|
339
405
|
})
|
|
340
406
|
);
|
|
341
|
-
return { topic
|
|
407
|
+
return { topic, messages: builtMessages, ...compression && { compression } };
|
|
342
408
|
}
|
|
343
409
|
|
|
344
410
|
// src/client/kafka.client/consumer/ops.ts
|
|
345
|
-
function getOrCreateConsumer(groupId, fromBeginning, autoCommit, deps, partitionAssigner, onFirstAssignment) {
|
|
411
|
+
function getOrCreateConsumer(groupId, fromBeginning, autoCommit, deps, partitionAssigner, onFirstAssignment, groupInstanceId) {
|
|
346
412
|
const { consumers, consumerCreationOptions, transport, onRebalance, logger } = deps;
|
|
347
413
|
if (consumers.has(groupId)) {
|
|
348
414
|
const prev = consumerCreationOptions.get(groupId);
|
|
@@ -375,6 +441,7 @@ function getOrCreateConsumer(groupId, fromBeginning, autoCommit, deps, partition
|
|
|
375
441
|
fromBeginning,
|
|
376
442
|
autoCommit,
|
|
377
443
|
partitionAssigner: partitionAssigner ?? "cooperative-sticky",
|
|
444
|
+
groupInstanceId,
|
|
378
445
|
onRebalance: (type, assignments) => {
|
|
379
446
|
if (type === "assign") fireOnAssignment();
|
|
380
447
|
else if (type === "revoke") scheduleSettle();
|
|
@@ -420,6 +487,7 @@ var AdminOps = class {
|
|
|
420
487
|
constructor(deps) {
|
|
421
488
|
this.deps = deps;
|
|
422
489
|
}
|
|
490
|
+
deps;
|
|
423
491
|
isConnected = false;
|
|
424
492
|
/** Underlying admin client — used by index.ts for topic validation. */
|
|
425
493
|
get admin() {
|
|
@@ -450,7 +518,7 @@ var AdminOps = class {
|
|
|
450
518
|
await this.deps.admin.disconnect();
|
|
451
519
|
this.isConnected = false;
|
|
452
520
|
}
|
|
453
|
-
async resetOffsets(groupId,
|
|
521
|
+
async resetOffsets(groupId, topic, position) {
|
|
454
522
|
const gid = groupId ?? this.deps.defaultGroupId;
|
|
455
523
|
if (this.deps.runningConsumers.has(gid)) {
|
|
456
524
|
throw new Error(
|
|
@@ -458,14 +526,14 @@ var AdminOps = class {
|
|
|
458
526
|
);
|
|
459
527
|
}
|
|
460
528
|
await this.ensureConnected();
|
|
461
|
-
const partitionOffsets = await this.deps.admin.fetchTopicOffsets(
|
|
529
|
+
const partitionOffsets = await this.deps.admin.fetchTopicOffsets(topic);
|
|
462
530
|
const partitions = partitionOffsets.map(({ partition, low, high }) => ({
|
|
463
531
|
partition,
|
|
464
532
|
offset: position === "earliest" ? low : high
|
|
465
533
|
}));
|
|
466
|
-
await this.deps.admin.setOffsets({ groupId: gid, topic
|
|
534
|
+
await this.deps.admin.setOffsets({ groupId: gid, topic, partitions });
|
|
467
535
|
this.deps.logger.log(
|
|
468
|
-
`Offsets reset to ${position} for group "${gid}" on topic "${
|
|
536
|
+
`Offsets reset to ${position} for group "${gid}" on topic "${topic}"`
|
|
469
537
|
);
|
|
470
538
|
}
|
|
471
539
|
/**
|
|
@@ -482,15 +550,15 @@ var AdminOps = class {
|
|
|
482
550
|
}
|
|
483
551
|
await this.ensureConnected();
|
|
484
552
|
const byTopic = /* @__PURE__ */ new Map();
|
|
485
|
-
for (const { topic
|
|
486
|
-
const list = byTopic.get(
|
|
553
|
+
for (const { topic, partition, offset } of assignments) {
|
|
554
|
+
const list = byTopic.get(topic) ?? [];
|
|
487
555
|
list.push({ partition, offset });
|
|
488
|
-
byTopic.set(
|
|
556
|
+
byTopic.set(topic, list);
|
|
489
557
|
}
|
|
490
|
-
for (const [
|
|
491
|
-
await this.deps.admin.setOffsets({ groupId: gid, topic
|
|
558
|
+
for (const [topic, partitions] of byTopic) {
|
|
559
|
+
await this.deps.admin.setOffsets({ groupId: gid, topic, partitions });
|
|
492
560
|
this.deps.logger.log(
|
|
493
|
-
`Offsets set for group "${gid}" on "${
|
|
561
|
+
`Offsets set for group "${gid}" on "${topic}": ${JSON.stringify(partitions)}`
|
|
494
562
|
);
|
|
495
563
|
}
|
|
496
564
|
}
|
|
@@ -511,27 +579,30 @@ var AdminOps = class {
|
|
|
511
579
|
}
|
|
512
580
|
await this.ensureConnected();
|
|
513
581
|
const byTopic = /* @__PURE__ */ new Map();
|
|
514
|
-
for (const { topic
|
|
515
|
-
const list = byTopic.get(
|
|
582
|
+
for (const { topic, partition, timestamp } of assignments) {
|
|
583
|
+
const list = byTopic.get(topic) ?? [];
|
|
516
584
|
list.push({ partition, timestamp });
|
|
517
|
-
byTopic.set(
|
|
585
|
+
byTopic.set(topic, list);
|
|
518
586
|
}
|
|
519
|
-
for (const [
|
|
587
|
+
for (const [topic, parts] of byTopic) {
|
|
520
588
|
const offsets = await Promise.all(
|
|
521
589
|
parts.map(async ({ partition, timestamp }) => {
|
|
522
590
|
const results = await this.deps.admin.fetchTopicOffsetsByTimestamp(
|
|
523
|
-
|
|
591
|
+
topic,
|
|
524
592
|
timestamp
|
|
525
593
|
);
|
|
526
594
|
const found = results.find(
|
|
527
595
|
(r) => r.partition === partition
|
|
528
596
|
);
|
|
529
|
-
return { partition, offset: found
|
|
597
|
+
if (found) return { partition, offset: found.offset };
|
|
598
|
+
const topicOffsets = await this.deps.admin.fetchTopicOffsets(topic);
|
|
599
|
+
const po = topicOffsets.find((o) => o.partition === partition);
|
|
600
|
+
return { partition, offset: po?.high ?? "0" };
|
|
530
601
|
})
|
|
531
602
|
);
|
|
532
|
-
await this.deps.admin.setOffsets({ groupId: gid, topic
|
|
603
|
+
await this.deps.admin.setOffsets({ groupId: gid, topic, partitions: offsets });
|
|
533
604
|
this.deps.logger.log(
|
|
534
|
-
`Offsets set by timestamp for group "${gid}" on "${
|
|
605
|
+
`Offsets set by timestamp for group "${gid}" on "${topic}": ${JSON.stringify(offsets)}`
|
|
535
606
|
);
|
|
536
607
|
}
|
|
537
608
|
}
|
|
@@ -551,11 +622,11 @@ var AdminOps = class {
|
|
|
551
622
|
await this.ensureConnected();
|
|
552
623
|
const committedByTopic = await this.deps.admin.fetchOffsets({ groupId: gid });
|
|
553
624
|
const brokerOffsetsAll = await Promise.all(
|
|
554
|
-
committedByTopic.map(({ topic
|
|
625
|
+
committedByTopic.map(({ topic }) => this.deps.admin.fetchTopicOffsets(topic))
|
|
555
626
|
);
|
|
556
627
|
const result = [];
|
|
557
628
|
for (let i = 0; i < committedByTopic.length; i++) {
|
|
558
|
-
const { topic
|
|
629
|
+
const { topic, partitions } = committedByTopic[i];
|
|
559
630
|
const brokerOffsets = brokerOffsetsAll[i];
|
|
560
631
|
for (const { partition, offset } of partitions) {
|
|
561
632
|
const broker = brokerOffsets.find((o) => o.partition === partition);
|
|
@@ -563,7 +634,7 @@ var AdminOps = class {
|
|
|
563
634
|
const committed = parseInt(offset, 10);
|
|
564
635
|
const high = parseInt(broker.high, 10);
|
|
565
636
|
const lag = committed === -1 ? high : Math.max(0, high - committed);
|
|
566
|
-
result.push({ topic
|
|
637
|
+
result.push({ topic, partition, lag });
|
|
567
638
|
}
|
|
568
639
|
}
|
|
569
640
|
return result;
|
|
@@ -606,8 +677,9 @@ var AdminOps = class {
|
|
|
606
677
|
return result.topics.map((t) => ({
|
|
607
678
|
name: t.name,
|
|
608
679
|
partitions: t.partitions.map((p) => ({
|
|
609
|
-
partition: p.partitionId ?? p.partition,
|
|
610
|
-
leader
|
|
680
|
+
partition: p.partitionId ?? p.partition ?? 0,
|
|
681
|
+
// -1 is Kafka's own "no leader" sentinel; 0 is a valid broker id
|
|
682
|
+
leader: p.leader ?? -1,
|
|
611
683
|
replicas: (p.replicas ?? []).map(
|
|
612
684
|
(r) => typeof r === "number" ? r : r.nodeId
|
|
613
685
|
),
|
|
@@ -632,9 +704,9 @@ var AdminOps = class {
|
|
|
632
704
|
* Delete records from a topic up to (but not including) the given offsets.
|
|
633
705
|
* All messages with offsets **before** the given offset are deleted.
|
|
634
706
|
*/
|
|
635
|
-
async deleteRecords(
|
|
707
|
+
async deleteRecords(topic, partitions) {
|
|
636
708
|
await this.ensureConnected();
|
|
637
|
-
await this.deps.admin.deleteTopicRecords({ topic
|
|
709
|
+
await this.deps.admin.deleteTopicRecords({ topic, partitions });
|
|
638
710
|
}
|
|
639
711
|
/**
|
|
640
712
|
* When `retryTopics: true` and `autoCreateTopics: false`, verify that every
|
|
@@ -691,28 +763,25 @@ var AdminOps = class {
|
|
|
691
763
|
};
|
|
692
764
|
|
|
693
765
|
// src/client/kafka.client/consumer/pipeline.ts
|
|
694
|
-
function toError(error) {
|
|
695
|
-
return error instanceof Error ? error : new Error(String(error));
|
|
696
|
-
}
|
|
697
766
|
function sleep(ms) {
|
|
698
767
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
699
768
|
}
|
|
700
|
-
function parseJsonMessage(raw,
|
|
769
|
+
function parseJsonMessage(raw, topic, logger) {
|
|
701
770
|
try {
|
|
702
771
|
return JSON.parse(raw);
|
|
703
772
|
} catch (error) {
|
|
704
773
|
logger.error(
|
|
705
|
-
`Failed to parse message from topic ${
|
|
774
|
+
`Failed to parse message from topic ${topic}:`,
|
|
706
775
|
toError(error).stack
|
|
707
776
|
);
|
|
708
777
|
return null;
|
|
709
778
|
}
|
|
710
779
|
}
|
|
711
|
-
async function validateWithSchema(message, raw,
|
|
712
|
-
const schema = schemaMap.get(
|
|
780
|
+
async function validateWithSchema(message, raw, topic, schemaMap, interceptors, dlq, deps) {
|
|
781
|
+
const schema = schemaMap.get(topic);
|
|
713
782
|
if (!schema) return message;
|
|
714
783
|
const ctx = {
|
|
715
|
-
topic
|
|
784
|
+
topic,
|
|
716
785
|
headers: deps.originalHeaders ?? {},
|
|
717
786
|
version: Number(deps.originalHeaders?.["x-schema-version"] ?? 1)
|
|
718
787
|
};
|
|
@@ -720,22 +789,22 @@ async function validateWithSchema(message, raw, topic2, schemaMap, interceptors,
|
|
|
720
789
|
return await schema.parse(message, ctx);
|
|
721
790
|
} catch (error) {
|
|
722
791
|
const err = toError(error);
|
|
723
|
-
const validationError = new KafkaValidationError(
|
|
792
|
+
const validationError = new KafkaValidationError(topic, message, {
|
|
724
793
|
cause: err
|
|
725
794
|
});
|
|
726
795
|
deps.logger.error(
|
|
727
|
-
`Schema validation failed for topic ${
|
|
796
|
+
`Schema validation failed for topic ${topic}:`,
|
|
728
797
|
err.message
|
|
729
798
|
);
|
|
730
799
|
if (dlq) {
|
|
731
|
-
await sendToDlq(
|
|
800
|
+
await sendToDlq(topic, raw, deps, {
|
|
732
801
|
error: validationError,
|
|
733
802
|
attempt: 0,
|
|
734
803
|
originalHeaders: deps.originalHeaders
|
|
735
804
|
});
|
|
736
805
|
} else {
|
|
737
806
|
await deps.onMessageLost?.({
|
|
738
|
-
topic
|
|
807
|
+
topic,
|
|
739
808
|
error: validationError,
|
|
740
809
|
attempt: 0,
|
|
741
810
|
headers: deps.originalHeaders ?? {}
|
|
@@ -744,7 +813,7 @@ async function validateWithSchema(message, raw, topic2, schemaMap, interceptors,
|
|
|
744
813
|
const errorEnvelope = extractEnvelope(
|
|
745
814
|
message,
|
|
746
815
|
deps.originalHeaders ?? {},
|
|
747
|
-
|
|
816
|
+
topic,
|
|
748
817
|
-1,
|
|
749
818
|
""
|
|
750
819
|
);
|
|
@@ -757,11 +826,11 @@ async function validateWithSchema(message, raw, topic2, schemaMap, interceptors,
|
|
|
757
826
|
return null;
|
|
758
827
|
}
|
|
759
828
|
}
|
|
760
|
-
function buildDlqPayload(
|
|
761
|
-
const dlqTopic = `${
|
|
829
|
+
function buildDlqPayload(topic, rawMessage, meta) {
|
|
830
|
+
const dlqTopic = `${topic}.dlq`;
|
|
762
831
|
const headers = {
|
|
763
832
|
...meta?.originalHeaders ?? {},
|
|
764
|
-
"x-dlq-original-topic":
|
|
833
|
+
"x-dlq-original-topic": topic,
|
|
765
834
|
"x-dlq-failed-at": (/* @__PURE__ */ new Date()).toISOString(),
|
|
766
835
|
"x-dlq-error-message": meta?.error.message ?? "unknown",
|
|
767
836
|
"x-dlq-error-stack": meta?.error.stack?.slice(0, 2e3) ?? "",
|
|
@@ -769,8 +838,8 @@ function buildDlqPayload(topic2, rawMessage, meta) {
|
|
|
769
838
|
};
|
|
770
839
|
return { topic: dlqTopic, messages: [{ value: rawMessage, headers }] };
|
|
771
840
|
}
|
|
772
|
-
async function sendToDlq(
|
|
773
|
-
const payload = buildDlqPayload(
|
|
841
|
+
async function sendToDlq(topic, rawMessage, deps, meta) {
|
|
842
|
+
const payload = buildDlqPayload(topic, rawMessage, meta);
|
|
774
843
|
try {
|
|
775
844
|
await deps.producer.send(payload);
|
|
776
845
|
deps.logger.warn(`Message sent to DLQ: ${payload.topic}`);
|
|
@@ -781,7 +850,7 @@ async function sendToDlq(topic2, rawMessage, deps, meta) {
|
|
|
781
850
|
err.stack
|
|
782
851
|
);
|
|
783
852
|
await deps.onMessageLost?.({
|
|
784
|
-
topic
|
|
853
|
+
topic,
|
|
785
854
|
error: err,
|
|
786
855
|
attempt: meta?.attempt ?? 0,
|
|
787
856
|
headers: meta?.originalHeaders ?? {}
|
|
@@ -954,7 +1023,7 @@ async function executeWithRetry(fn, ctx, deps) {
|
|
|
954
1023
|
const backoffMs = retry?.backoffMs ?? 1e3;
|
|
955
1024
|
const maxBackoffMs = retry?.maxBackoffMs ?? 3e4;
|
|
956
1025
|
const envelopes = Array.isArray(envelope) ? envelope : [envelope];
|
|
957
|
-
const
|
|
1026
|
+
const topic = envelopes[0]?.topic ?? "unknown";
|
|
958
1027
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
959
1028
|
const error = await runHandlerWithPipeline(
|
|
960
1029
|
fn,
|
|
@@ -977,16 +1046,17 @@ async function executeWithRetry(fn, ctx, deps) {
|
|
|
977
1046
|
for (const env of envelopes) deps.onMessage?.(env);
|
|
978
1047
|
return;
|
|
979
1048
|
}
|
|
1049
|
+
deps.onFailure?.(envelopes[0]);
|
|
980
1050
|
const isLastAttempt = attempt === maxAttempts;
|
|
981
1051
|
const reportedError = isLastAttempt && maxAttempts > 1 ? new KafkaRetryExhaustedError(
|
|
982
|
-
|
|
1052
|
+
topic,
|
|
983
1053
|
envelopes.map((e) => e.payload),
|
|
984
1054
|
maxAttempts,
|
|
985
1055
|
{ cause: error }
|
|
986
1056
|
) : error;
|
|
987
1057
|
await notifyInterceptorsOnError(envelopes, interceptors, reportedError);
|
|
988
1058
|
deps.logger.error(
|
|
989
|
-
`Error processing ${isBatch ? "batch" : "message"} from topic ${
|
|
1059
|
+
`Error processing ${isBatch ? "batch" : "message"} from topic ${topic} (attempt ${attempt}/${maxAttempts}):`,
|
|
990
1060
|
error.stack
|
|
991
1061
|
);
|
|
992
1062
|
if (retryTopics && retry) {
|
|
@@ -1004,7 +1074,7 @@ async function executeWithRetry(fn, ctx, deps) {
|
|
|
1004
1074
|
}
|
|
1005
1075
|
} else {
|
|
1006
1076
|
await sendToRetryTopic(
|
|
1007
|
-
|
|
1077
|
+
topic,
|
|
1008
1078
|
rawMessages,
|
|
1009
1079
|
1,
|
|
1010
1080
|
retry.maxRetries,
|
|
@@ -1017,7 +1087,7 @@ async function executeWithRetry(fn, ctx, deps) {
|
|
|
1017
1087
|
} else if (isLastAttempt) {
|
|
1018
1088
|
if (dlq) {
|
|
1019
1089
|
for (let i = 0; i < rawMessages.length; i++) {
|
|
1020
|
-
await sendToDlq(
|
|
1090
|
+
await sendToDlq(topic, rawMessages[i], deps, {
|
|
1021
1091
|
error,
|
|
1022
1092
|
attempt,
|
|
1023
1093
|
originalHeaders: envelopes[i]?.headers
|
|
@@ -1026,7 +1096,7 @@ async function executeWithRetry(fn, ctx, deps) {
|
|
|
1026
1096
|
}
|
|
1027
1097
|
} else {
|
|
1028
1098
|
await deps.onMessageLost?.({
|
|
1029
|
-
topic
|
|
1099
|
+
topic,
|
|
1030
1100
|
error,
|
|
1031
1101
|
attempt,
|
|
1032
1102
|
headers: envelopes[0]?.headers ?? {}
|
|
@@ -1061,9 +1131,14 @@ async function subscribeWithRetry(consumer, topics, logger, retryOpts) {
|
|
|
1061
1131
|
}
|
|
1062
1132
|
}
|
|
1063
1133
|
|
|
1064
|
-
// src/client/kafka.client/consumer/dlq-replay.ts
|
|
1065
|
-
async function replayDlqTopic(
|
|
1066
|
-
|
|
1134
|
+
// src/client/kafka.client/consumer/features/dlq-replay.ts
|
|
1135
|
+
async function replayDlqTopic(topic, deps, options = {}) {
|
|
1136
|
+
if (topic.endsWith(".dlq")) {
|
|
1137
|
+
throw new Error(
|
|
1138
|
+
`replayDlq: pass the ORIGINAL topic name \u2014 "${topic}" already ends in ".dlq" (the ".dlq" suffix is appended internally, so this would read "${topic}.dlq")`
|
|
1139
|
+
);
|
|
1140
|
+
}
|
|
1141
|
+
const dlqTopic = `${topic}.dlq`;
|
|
1067
1142
|
const partitionOffsets = await deps.fetchTopicOffsets(dlqTopic);
|
|
1068
1143
|
const activePartitions = partitionOffsets.filter(
|
|
1069
1144
|
(p) => Number.parseInt(p.high, 10) > Number.parseInt(p.low, 10)
|
|
@@ -1128,19 +1203,20 @@ var MetricsManager = class {
|
|
|
1128
1203
|
constructor(deps) {
|
|
1129
1204
|
this.deps = deps;
|
|
1130
1205
|
}
|
|
1206
|
+
deps;
|
|
1131
1207
|
topicMetrics = /* @__PURE__ */ new Map();
|
|
1132
|
-
metricsFor(
|
|
1133
|
-
let m = this.topicMetrics.get(
|
|
1208
|
+
metricsFor(topic) {
|
|
1209
|
+
let m = this.topicMetrics.get(topic);
|
|
1134
1210
|
if (!m) {
|
|
1135
1211
|
m = { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
1136
|
-
this.topicMetrics.set(
|
|
1212
|
+
this.topicMetrics.set(topic, m);
|
|
1137
1213
|
}
|
|
1138
1214
|
return m;
|
|
1139
1215
|
}
|
|
1140
1216
|
/** Fire `afterSend` instrumentation hooks for each message in a batch. */
|
|
1141
|
-
notifyAfterSend(
|
|
1217
|
+
notifyAfterSend(topic, count) {
|
|
1142
1218
|
for (let i = 0; i < count; i++)
|
|
1143
|
-
for (const inst of this.deps.instrumentation) inst.afterSend?.(
|
|
1219
|
+
for (const inst of this.deps.instrumentation) inst.afterSend?.(topic);
|
|
1144
1220
|
}
|
|
1145
1221
|
/**
|
|
1146
1222
|
* Increment the retry counter for the envelope's topic and fire all `onRetry` instrumentation hooks.
|
|
@@ -1153,16 +1229,25 @@ var MetricsManager = class {
|
|
|
1153
1229
|
for (const inst of this.deps.instrumentation) inst.onRetry?.(envelope, attempt, maxRetries);
|
|
1154
1230
|
}
|
|
1155
1231
|
/**
|
|
1156
|
-
* Increment the DLQ counter for the envelope's topic
|
|
1157
|
-
*
|
|
1232
|
+
* Increment the DLQ counter for the envelope's topic and fire all `onDlq` instrumentation hooks.
|
|
1233
|
+
* Circuit breaker failures are recorded separately via `notifyFailure` at the
|
|
1234
|
+
* handler-error boundary — dead-lettering itself is not a circuit event.
|
|
1158
1235
|
* @param envelope The message envelope being sent to the DLQ.
|
|
1159
1236
|
* @param reason The reason the message is being dead-lettered.
|
|
1160
|
-
* @param gid Consumer group ID — used to drive circuit breaker state.
|
|
1161
1237
|
*/
|
|
1162
|
-
notifyDlq(envelope, reason
|
|
1238
|
+
notifyDlq(envelope, reason) {
|
|
1163
1239
|
this.metricsFor(envelope.topic).dlqCount++;
|
|
1164
1240
|
for (const inst of this.deps.instrumentation) inst.onDlq?.(envelope, reason);
|
|
1165
|
-
|
|
1241
|
+
}
|
|
1242
|
+
/**
|
|
1243
|
+
* Notify the circuit breaker of a handler failure. Fired on every failed
|
|
1244
|
+
* handler attempt (in-process retries and retry-topic levels included),
|
|
1245
|
+
* independent of whether the message is ultimately dead-lettered.
|
|
1246
|
+
* @param envelope The message envelope whose handler failed.
|
|
1247
|
+
* @param gid Consumer group ID — used to drive circuit breaker state.
|
|
1248
|
+
*/
|
|
1249
|
+
notifyFailure(envelope, gid) {
|
|
1250
|
+
this.deps.onCircuitFailure(envelope, gid);
|
|
1166
1251
|
}
|
|
1167
1252
|
/**
|
|
1168
1253
|
* Increment the deduplication counter for the envelope's topic and fire all `onDuplicate` hooks.
|
|
@@ -1189,9 +1274,9 @@ var MetricsManager = class {
|
|
|
1189
1274
|
* @param topic When provided, returns counters for that topic only; otherwise aggregates all topics.
|
|
1190
1275
|
* @returns Read-only `KafkaMetrics` snapshot. Returns zero-valued counters if the topic has no events.
|
|
1191
1276
|
*/
|
|
1192
|
-
getMetrics(
|
|
1193
|
-
if (
|
|
1194
|
-
const m = this.topicMetrics.get(
|
|
1277
|
+
getMetrics(topic) {
|
|
1278
|
+
if (topic !== void 0) {
|
|
1279
|
+
const m = this.topicMetrics.get(topic);
|
|
1195
1280
|
return m ? { ...m } : { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
1196
1281
|
}
|
|
1197
1282
|
const agg = { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
@@ -1207,9 +1292,9 @@ var MetricsManager = class {
|
|
|
1207
1292
|
* Reset event counters to zero.
|
|
1208
1293
|
* @param topic When provided, clears counters for that topic only; otherwise clears all topics.
|
|
1209
1294
|
*/
|
|
1210
|
-
resetMetrics(
|
|
1211
|
-
if (
|
|
1212
|
-
this.topicMetrics.delete(
|
|
1295
|
+
resetMetrics(topic) {
|
|
1296
|
+
if (topic !== void 0) {
|
|
1297
|
+
this.topicMetrics.delete(topic);
|
|
1213
1298
|
return;
|
|
1214
1299
|
}
|
|
1215
1300
|
this.topicMetrics.clear();
|
|
@@ -1221,6 +1306,7 @@ var InFlightTracker = class {
|
|
|
1221
1306
|
constructor(warn) {
|
|
1222
1307
|
this.warn = warn;
|
|
1223
1308
|
}
|
|
1309
|
+
warn;
|
|
1224
1310
|
inFlightTotal = 0;
|
|
1225
1311
|
drainResolvers = [];
|
|
1226
1312
|
/**
|
|
@@ -1231,10 +1317,16 @@ var InFlightTracker = class {
|
|
|
1231
1317
|
*/
|
|
1232
1318
|
track(fn) {
|
|
1233
1319
|
this.inFlightTotal++;
|
|
1234
|
-
|
|
1320
|
+
const done = () => {
|
|
1235
1321
|
this.inFlightTotal--;
|
|
1236
1322
|
if (this.inFlightTotal === 0) this.drainResolvers.splice(0).forEach((r) => r());
|
|
1237
|
-
}
|
|
1323
|
+
};
|
|
1324
|
+
try {
|
|
1325
|
+
return fn().finally(done);
|
|
1326
|
+
} catch (err) {
|
|
1327
|
+
done();
|
|
1328
|
+
throw err;
|
|
1329
|
+
}
|
|
1238
1330
|
}
|
|
1239
1331
|
/**
|
|
1240
1332
|
* Resolve when all tracked handlers have completed, or after `timeoutMs` elapses.
|
|
@@ -1268,6 +1360,7 @@ var CircuitBreakerManager = class {
|
|
|
1268
1360
|
constructor(deps) {
|
|
1269
1361
|
this.deps = deps;
|
|
1270
1362
|
}
|
|
1363
|
+
deps;
|
|
1271
1364
|
states = /* @__PURE__ */ new Map();
|
|
1272
1365
|
configs = /* @__PURE__ */ new Map();
|
|
1273
1366
|
/**
|
|
@@ -1283,8 +1376,8 @@ var CircuitBreakerManager = class {
|
|
|
1283
1376
|
* Returns a snapshot of the circuit breaker state for a given topic-partition.
|
|
1284
1377
|
* Returns `undefined` when no state exists for the key.
|
|
1285
1378
|
*/
|
|
1286
|
-
getState(
|
|
1287
|
-
const state = this.states.get(`${gid}:${
|
|
1379
|
+
getState(topic, partition, gid) {
|
|
1380
|
+
const state = this.states.get(`${gid}:${topic}:${partition}`);
|
|
1288
1381
|
if (!state) return void 0;
|
|
1289
1382
|
return {
|
|
1290
1383
|
status: state.status,
|
|
@@ -1412,6 +1505,9 @@ var AsyncQueue = class {
|
|
|
1412
1505
|
this.onFull = onFull;
|
|
1413
1506
|
this.onDrained = onDrained;
|
|
1414
1507
|
}
|
|
1508
|
+
highWaterMark;
|
|
1509
|
+
onFull;
|
|
1510
|
+
onDrained;
|
|
1415
1511
|
items = [];
|
|
1416
1512
|
waiting = [];
|
|
1417
1513
|
closed = false;
|
|
@@ -1423,6 +1519,7 @@ var AsyncQueue = class {
|
|
|
1423
1519
|
* @param item The value to enqueue.
|
|
1424
1520
|
*/
|
|
1425
1521
|
push(item) {
|
|
1522
|
+
if (this.closed) return;
|
|
1426
1523
|
if (this.waiting.length > 0) {
|
|
1427
1524
|
this.waiting.shift().resolve({ value: item, done: false });
|
|
1428
1525
|
} else {
|
|
@@ -1473,20 +1570,115 @@ var AsyncQueue = class {
|
|
|
1473
1570
|
}
|
|
1474
1571
|
};
|
|
1475
1572
|
|
|
1573
|
+
// src/client/kafka.client/validate-options.ts
|
|
1574
|
+
function validateClientOptions(clientId, groupId, brokers, options) {
|
|
1575
|
+
const problems = [];
|
|
1576
|
+
if (typeof clientId !== "string" || clientId.trim() === "") {
|
|
1577
|
+
problems.push("clientId must be a non-empty string");
|
|
1578
|
+
}
|
|
1579
|
+
if (typeof groupId !== "string" || groupId.trim() === "") {
|
|
1580
|
+
problems.push("groupId must be a non-empty string");
|
|
1581
|
+
}
|
|
1582
|
+
if (!Array.isArray(brokers) || brokers.length === 0 && !options?.transport) {
|
|
1583
|
+
problems.push("brokers must be a non-empty array of broker addresses");
|
|
1584
|
+
} else if (brokers.some((b) => typeof b !== "string" || b.trim() === "")) {
|
|
1585
|
+
problems.push("brokers must not contain empty entries");
|
|
1586
|
+
}
|
|
1587
|
+
if (options) {
|
|
1588
|
+
const {
|
|
1589
|
+
numPartitions,
|
|
1590
|
+
transactionalId,
|
|
1591
|
+
clockRecovery,
|
|
1592
|
+
lagThrottle
|
|
1593
|
+
} = options;
|
|
1594
|
+
if (numPartitions !== void 0 && (!Number.isInteger(numPartitions) || numPartitions < 1)) {
|
|
1595
|
+
problems.push(
|
|
1596
|
+
`numPartitions must be a positive integer (got ${numPartitions})`
|
|
1597
|
+
);
|
|
1598
|
+
}
|
|
1599
|
+
if (transactionalId !== void 0 && transactionalId.trim() === "") {
|
|
1600
|
+
problems.push("transactionalId must be a non-empty string when set");
|
|
1601
|
+
}
|
|
1602
|
+
if (clockRecovery) {
|
|
1603
|
+
if (!Array.isArray(clockRecovery.topics)) {
|
|
1604
|
+
problems.push("clockRecovery.topics must be an array of topic names");
|
|
1605
|
+
}
|
|
1606
|
+
if (clockRecovery.timeoutMs !== void 0 && !(clockRecovery.timeoutMs > 0)) {
|
|
1607
|
+
problems.push(
|
|
1608
|
+
`clockRecovery.timeoutMs must be > 0 (got ${clockRecovery.timeoutMs})`
|
|
1609
|
+
);
|
|
1610
|
+
}
|
|
1611
|
+
}
|
|
1612
|
+
if (lagThrottle) {
|
|
1613
|
+
if (!(lagThrottle.maxLag >= 0)) {
|
|
1614
|
+
problems.push(`lagThrottle.maxLag must be >= 0 (got ${lagThrottle.maxLag})`);
|
|
1615
|
+
}
|
|
1616
|
+
if (lagThrottle.pollIntervalMs !== void 0 && !(lagThrottle.pollIntervalMs > 0)) {
|
|
1617
|
+
problems.push(
|
|
1618
|
+
`lagThrottle.pollIntervalMs must be > 0 (got ${lagThrottle.pollIntervalMs})`
|
|
1619
|
+
);
|
|
1620
|
+
}
|
|
1621
|
+
if (lagThrottle.maxWaitMs !== void 0 && !(lagThrottle.maxWaitMs >= 0)) {
|
|
1622
|
+
problems.push(
|
|
1623
|
+
`lagThrottle.maxWaitMs must be >= 0 (got ${lagThrottle.maxWaitMs})`
|
|
1624
|
+
);
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
if (problems.length > 0) {
|
|
1629
|
+
throw new Error(
|
|
1630
|
+
`KafkaClient: invalid configuration:
|
|
1631
|
+
- ${problems.join("\n- ")}`
|
|
1632
|
+
);
|
|
1633
|
+
}
|
|
1634
|
+
}
|
|
1635
|
+
|
|
1636
|
+
// src/client/security/resolve-security.ts
|
|
1637
|
+
var LOCAL_HOST_PATTERNS = [
|
|
1638
|
+
/^localhost(:\d+)?$/i,
|
|
1639
|
+
/^127\.\d+\.\d+\.\d+(:\d+)?$/,
|
|
1640
|
+
/^\[?::1\]?(:\d+)?$/,
|
|
1641
|
+
/^0\.0\.0\.0(:\d+)?$/,
|
|
1642
|
+
/^host\.docker\.internal(:\d+)?$/i
|
|
1643
|
+
];
|
|
1644
|
+
function isLocalBroker(broker) {
|
|
1645
|
+
return LOCAL_HOST_PATTERNS.some((re) => re.test(broker.trim()));
|
|
1646
|
+
}
|
|
1647
|
+
function resolveSecurityOptions(security, brokers, logger) {
|
|
1648
|
+
const hasRemoteBroker = brokers.some((b) => !isLocalBroker(b));
|
|
1649
|
+
if (!security?.sasl && security?.ssl !== true) {
|
|
1650
|
+
if (hasRemoteBroker && !security?.allowInsecure) {
|
|
1651
|
+
logger.warn(
|
|
1652
|
+
"Connecting to non-local brokers without TLS or SASL \u2014 traffic and payloads travel in plaintext. Configure `security: { ssl, sasl }` for production clusters, or set `security: { allowInsecure: true }` to acknowledge an intentionally insecure setup."
|
|
1653
|
+
);
|
|
1654
|
+
}
|
|
1655
|
+
return security;
|
|
1656
|
+
}
|
|
1657
|
+
if (security.sasl && security.ssl === void 0) {
|
|
1658
|
+
return { ...security, ssl: true };
|
|
1659
|
+
}
|
|
1660
|
+
if (security.sasl && security.ssl === false) {
|
|
1661
|
+
logger.warn(
|
|
1662
|
+
"SASL credentials are configured with `ssl: false` \u2014 credentials will be sent over plaintext. This is only safe on fully trusted networks."
|
|
1663
|
+
);
|
|
1664
|
+
}
|
|
1665
|
+
return security;
|
|
1666
|
+
}
|
|
1667
|
+
|
|
1476
1668
|
// src/client/kafka.client/producer/lifecycle.ts
|
|
1477
1669
|
var _activeTransactionalIds = /* @__PURE__ */ new Set();
|
|
1478
|
-
async function ensureTopic(ctx,
|
|
1479
|
-
if (!ctx.autoCreateTopicsEnabled || ctx.ensuredTopics.has(
|
|
1480
|
-
let p = ctx.ensureTopicPromises.get(
|
|
1670
|
+
async function ensureTopic(ctx, topic) {
|
|
1671
|
+
if (!ctx.autoCreateTopicsEnabled || ctx.ensuredTopics.has(topic)) return;
|
|
1672
|
+
let p = ctx.ensureTopicPromises.get(topic);
|
|
1481
1673
|
if (!p) {
|
|
1482
1674
|
p = (async () => {
|
|
1483
1675
|
await ctx.adminOps.ensureConnected();
|
|
1484
1676
|
await ctx.adminOps.admin.createTopics({
|
|
1485
|
-
topics: [{ topic
|
|
1677
|
+
topics: [{ topic, numPartitions: ctx.numPartitions }]
|
|
1486
1678
|
});
|
|
1487
|
-
ctx.ensuredTopics.add(
|
|
1488
|
-
})().finally(() => ctx.ensureTopicPromises.delete(
|
|
1489
|
-
ctx.ensureTopicPromises.set(
|
|
1679
|
+
ctx.ensuredTopics.add(topic);
|
|
1680
|
+
})().finally(() => ctx.ensureTopicPromises.delete(topic));
|
|
1681
|
+
ctx.ensureTopicPromises.set(topic, p);
|
|
1490
1682
|
}
|
|
1491
1683
|
await p;
|
|
1492
1684
|
}
|
|
@@ -1606,6 +1798,7 @@ async function recoverLamportClockImpl(ctx, topics) {
|
|
|
1606
1798
|
const remaining = new Set(
|
|
1607
1799
|
partitionsToRead.map((p) => `${p.topic}:${p.partition}`)
|
|
1608
1800
|
);
|
|
1801
|
+
let settled = false;
|
|
1609
1802
|
const cleanup = () => {
|
|
1610
1803
|
consumer.disconnect().catch(() => {
|
|
1611
1804
|
}).finally(() => {
|
|
@@ -1613,6 +1806,16 @@ async function recoverLamportClockImpl(ctx, topics) {
|
|
|
1613
1806
|
});
|
|
1614
1807
|
});
|
|
1615
1808
|
};
|
|
1809
|
+
const timeoutTimer = setTimeout(() => {
|
|
1810
|
+
if (settled) return;
|
|
1811
|
+
settled = true;
|
|
1812
|
+
ctx.logger.warn(
|
|
1813
|
+
`Clock recovery: timed out after ${ctx.clockRecoveryTimeoutMs} ms with ${remaining.size} partition(s) unread \u2014 proceeding with partial result`
|
|
1814
|
+
);
|
|
1815
|
+
cleanup();
|
|
1816
|
+
resolve();
|
|
1817
|
+
}, ctx.clockRecoveryTimeoutMs);
|
|
1818
|
+
timeoutTimer.unref?.();
|
|
1616
1819
|
consumer.connect().then(async () => {
|
|
1617
1820
|
const uniqueTopics = [
|
|
1618
1821
|
...new Set(partitionsToRead.map((p) => p.topic))
|
|
@@ -1633,13 +1836,18 @@ async function recoverLamportClockImpl(ctx, topics) {
|
|
|
1633
1836
|
const clock = Number(raw);
|
|
1634
1837
|
if (!Number.isNaN(clock) && clock > maxClock) maxClock = clock;
|
|
1635
1838
|
}
|
|
1636
|
-
if (remaining.size === 0) {
|
|
1839
|
+
if (remaining.size === 0 && !settled) {
|
|
1840
|
+
settled = true;
|
|
1841
|
+
clearTimeout(timeoutTimer);
|
|
1637
1842
|
cleanup();
|
|
1638
1843
|
resolve();
|
|
1639
1844
|
}
|
|
1640
1845
|
}
|
|
1641
1846
|
})
|
|
1642
1847
|
).catch((err) => {
|
|
1848
|
+
if (settled) return;
|
|
1849
|
+
settled = true;
|
|
1850
|
+
clearTimeout(timeoutTimer);
|
|
1643
1851
|
cleanup();
|
|
1644
1852
|
reject(err);
|
|
1645
1853
|
});
|
|
@@ -1655,14 +1863,14 @@ async function recoverLamportClockImpl(ctx, topics) {
|
|
|
1655
1863
|
);
|
|
1656
1864
|
}
|
|
1657
1865
|
}
|
|
1658
|
-
function wrapWithTimeoutWarning(logger, fn, timeoutMs,
|
|
1866
|
+
function wrapWithTimeoutWarning(logger, fn, timeoutMs, topic) {
|
|
1659
1867
|
let timer;
|
|
1660
1868
|
const promise = fn().finally(() => {
|
|
1661
1869
|
if (timer !== void 0) clearTimeout(timer);
|
|
1662
1870
|
});
|
|
1663
1871
|
timer = setTimeout(() => {
|
|
1664
1872
|
logger.warn(
|
|
1665
|
-
`Handler for topic "${
|
|
1873
|
+
`Handler for topic "${topic}" has not resolved after ${timeoutMs}ms \u2014 possible stuck handler`
|
|
1666
1874
|
);
|
|
1667
1875
|
}, timeoutMs);
|
|
1668
1876
|
return promise;
|
|
@@ -1680,6 +1888,15 @@ async function preparePayload(ctx, topicOrDesc, messages, compression) {
|
|
|
1680
1888
|
await ensureTopic(ctx, payload.topic);
|
|
1681
1889
|
return payload;
|
|
1682
1890
|
}
|
|
1891
|
+
async function redirectToDelayed(ctx, payload, deliverAfterMs) {
|
|
1892
|
+
const until = String(Date.now() + deliverAfterMs);
|
|
1893
|
+
for (const m of payload.messages) {
|
|
1894
|
+
m.headers[HEADER_DELAYED_UNTIL] = until;
|
|
1895
|
+
m.headers[HEADER_DELAYED_TARGET] = payload.topic;
|
|
1896
|
+
}
|
|
1897
|
+
payload.topic = `${payload.topic}.delayed`;
|
|
1898
|
+
await ensureTopic(ctx, payload.topic);
|
|
1899
|
+
}
|
|
1683
1900
|
async function sendMessageImpl(ctx, topicOrDesc, message, options = {}) {
|
|
1684
1901
|
await waitIfThrottled(ctx);
|
|
1685
1902
|
const payload = await preparePayload(
|
|
@@ -1697,6 +1914,9 @@ async function sendMessageImpl(ctx, topicOrDesc, message, options = {}) {
|
|
|
1697
1914
|
],
|
|
1698
1915
|
options.compression
|
|
1699
1916
|
);
|
|
1917
|
+
if (options.deliverAfterMs && options.deliverAfterMs > 0) {
|
|
1918
|
+
await redirectToDelayed(ctx, payload, options.deliverAfterMs);
|
|
1919
|
+
}
|
|
1700
1920
|
await ctx.producer.send(payload);
|
|
1701
1921
|
ctx.metrics.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1702
1922
|
}
|
|
@@ -1708,19 +1928,22 @@ async function sendBatchImpl(ctx, topicOrDesc, messages, options) {
|
|
|
1708
1928
|
messages,
|
|
1709
1929
|
options?.compression
|
|
1710
1930
|
);
|
|
1931
|
+
if (options?.deliverAfterMs && options.deliverAfterMs > 0) {
|
|
1932
|
+
await redirectToDelayed(ctx, payload, options.deliverAfterMs);
|
|
1933
|
+
}
|
|
1711
1934
|
await ctx.producer.send(payload);
|
|
1712
1935
|
ctx.metrics.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1713
1936
|
}
|
|
1714
|
-
async function sendTombstoneImpl(ctx,
|
|
1937
|
+
async function sendTombstoneImpl(ctx, topic, key, headers) {
|
|
1715
1938
|
await waitIfThrottled(ctx);
|
|
1716
1939
|
const hdrs = { ...headers };
|
|
1717
|
-
for (const inst of ctx.instrumentation) inst.beforeSend?.(
|
|
1718
|
-
await ensureTopic(ctx,
|
|
1940
|
+
for (const inst of ctx.instrumentation) inst.beforeSend?.(topic, hdrs);
|
|
1941
|
+
await ensureTopic(ctx, topic);
|
|
1719
1942
|
await ctx.producer.send({
|
|
1720
|
-
topic
|
|
1943
|
+
topic,
|
|
1721
1944
|
messages: [{ value: null, key, headers: hdrs }]
|
|
1722
1945
|
});
|
|
1723
|
-
for (const inst of ctx.instrumentation) inst.afterSend?.(
|
|
1946
|
+
for (const inst of ctx.instrumentation) inst.afterSend?.(topic);
|
|
1724
1947
|
}
|
|
1725
1948
|
async function transactionImpl(ctx, fn) {
|
|
1726
1949
|
if (!ctx.txProducerInitPromise) {
|
|
@@ -1744,6 +1967,17 @@ async function transactionImpl(ctx, fn) {
|
|
|
1744
1967
|
});
|
|
1745
1968
|
}
|
|
1746
1969
|
ctx.txProducer = await ctx.txProducerInitPromise;
|
|
1970
|
+
const prev = ctx._txChain;
|
|
1971
|
+
let release;
|
|
1972
|
+
ctx._txChain = new Promise((r) => release = r);
|
|
1973
|
+
await prev;
|
|
1974
|
+
try {
|
|
1975
|
+
await runTransaction(ctx, fn);
|
|
1976
|
+
} finally {
|
|
1977
|
+
release();
|
|
1978
|
+
}
|
|
1979
|
+
}
|
|
1980
|
+
async function runTransaction(ctx, fn) {
|
|
1747
1981
|
const tx = await ctx.txProducer.transaction();
|
|
1748
1982
|
try {
|
|
1749
1983
|
const txCtx = {
|
|
@@ -1911,6 +2145,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
|
|
|
1911
2145
|
await consumer.commitOffsets([nextOffset]);
|
|
1912
2146
|
return;
|
|
1913
2147
|
}
|
|
2148
|
+
deps.onFailure?.(envelope);
|
|
1914
2149
|
const exhausted = level >= currentMaxRetries;
|
|
1915
2150
|
const reportedError = exhausted && currentMaxRetries > 1 ? new KafkaRetryExhaustedError(
|
|
1916
2151
|
originalTopic,
|
|
@@ -2126,7 +2361,8 @@ async function setupConsumer(ctx, topics, mode, options) {
|
|
|
2126
2361
|
options.autoCommit ?? true,
|
|
2127
2362
|
ctx.consumerOpsDeps,
|
|
2128
2363
|
options.partitionAssigner,
|
|
2129
|
-
resolveReady
|
|
2364
|
+
resolveReady,
|
|
2365
|
+
options.groupInstanceId
|
|
2130
2366
|
);
|
|
2131
2367
|
const schemaMap = buildSchemaMap(
|
|
2132
2368
|
stringTopics,
|
|
@@ -2138,6 +2374,9 @@ async function setupConsumer(ctx, topics, mode, options) {
|
|
|
2138
2374
|
const subscribeTopics = [...topicNames, ...regexTopics];
|
|
2139
2375
|
await ensureConsumerTopics(ctx, topicNames, dlq, options.deduplication);
|
|
2140
2376
|
await consumer.connect();
|
|
2377
|
+
if (dlq || options.retryTopics || options.deduplication) {
|
|
2378
|
+
await ctx.producer.connect();
|
|
2379
|
+
}
|
|
2141
2380
|
await subscribeWithRetry(
|
|
2142
2381
|
consumer,
|
|
2143
2382
|
subscribeTopics,
|
|
@@ -2152,9 +2391,8 @@ async function setupConsumer(ctx, topics, mode, options) {
|
|
|
2152
2391
|
}
|
|
2153
2392
|
function resolveDeduplicationContext(ctx, groupId, options) {
|
|
2154
2393
|
if (!options) return void 0;
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
return { options, state: ctx.dedupStates.get(groupId) };
|
|
2394
|
+
const store = options.store ?? new InMemoryDedupStore(ctx.dedupStates);
|
|
2395
|
+
return { options, store, groupId };
|
|
2158
2396
|
}
|
|
2159
2397
|
function messageDepsFor(ctx, gid, options) {
|
|
2160
2398
|
const notifyRetry = ctx.metrics.notifyRetry.bind(ctx.metrics);
|
|
@@ -2168,9 +2406,10 @@ function messageDepsFor(ctx, gid, options) {
|
|
|
2168
2406
|
notifyRetry(envelope, attempt, max);
|
|
2169
2407
|
return options.onRetry(envelope, attempt, max);
|
|
2170
2408
|
} : notifyRetry,
|
|
2171
|
-
onDlq: (envelope, reason) => ctx.metrics.notifyDlq(envelope, reason
|
|
2409
|
+
onDlq: (envelope, reason) => ctx.metrics.notifyDlq(envelope, reason),
|
|
2172
2410
|
onDuplicate: ctx.metrics.notifyDuplicate.bind(ctx.metrics),
|
|
2173
|
-
onMessage: (envelope) => ctx.metrics.notifyMessage(envelope, gid)
|
|
2411
|
+
onMessage: (envelope) => ctx.metrics.notifyMessage(envelope, gid),
|
|
2412
|
+
onFailure: (envelope) => ctx.metrics.notifyFailure(envelope, gid)
|
|
2174
2413
|
};
|
|
2175
2414
|
}
|
|
2176
2415
|
function buildRetryTopicDeps(ctx) {
|
|
@@ -2209,6 +2448,11 @@ async function launchRetryChain(ctx, gid, topicNames, handleMessage, opts) {
|
|
|
2209
2448
|
schemaMap,
|
|
2210
2449
|
{
|
|
2211
2450
|
...ctx.retryTopicDeps,
|
|
2451
|
+
// Bind circuit breaker events to the MAIN consumer group so failures and
|
|
2452
|
+
// successes inside the retry chain drive the same breaker as the main
|
|
2453
|
+
// consumer (the retry chain has no breaker config of its own).
|
|
2454
|
+
onFailure: (envelope) => ctx.metrics.notifyFailure(envelope, gid),
|
|
2455
|
+
onMessage: (envelope) => ctx.metrics.notifyMessage(envelope, gid),
|
|
2212
2456
|
onLevelStarted: (levelGroupId) => {
|
|
2213
2457
|
ctx.companionGroupIds.get(gid).push(levelGroupId);
|
|
2214
2458
|
}
|
|
@@ -2224,7 +2468,15 @@ async function applyDeduplication(envelope, raw, dedup, dlq, deps) {
|
|
|
2224
2468
|
const incomingClock = Number(clockRaw);
|
|
2225
2469
|
if (Number.isNaN(incomingClock)) return false;
|
|
2226
2470
|
const stateKey = `${envelope.topic}:${envelope.partition}`;
|
|
2227
|
-
|
|
2471
|
+
let lastProcessedClock;
|
|
2472
|
+
try {
|
|
2473
|
+
lastProcessedClock = await dedup.store.getLastClock(dedup.groupId, stateKey) ?? -1;
|
|
2474
|
+
} catch (err) {
|
|
2475
|
+
deps.logger.error(
|
|
2476
|
+
`Dedup store getLastClock failed on ${envelope.topic}[${envelope.partition}] \u2014 treating message as not a duplicate (fail-open): ${err.message}`
|
|
2477
|
+
);
|
|
2478
|
+
return false;
|
|
2479
|
+
}
|
|
2228
2480
|
if (incomingClock <= lastProcessedClock) {
|
|
2229
2481
|
const meta = {
|
|
2230
2482
|
incomingClock,
|
|
@@ -2254,32 +2506,38 @@ async function applyDeduplication(envelope, raw, dedup, dlq, deps) {
|
|
|
2254
2506
|
}
|
|
2255
2507
|
return true;
|
|
2256
2508
|
}
|
|
2257
|
-
|
|
2509
|
+
try {
|
|
2510
|
+
await dedup.store.setLastClock(dedup.groupId, stateKey, incomingClock);
|
|
2511
|
+
} catch (err) {
|
|
2512
|
+
deps.logger.error(
|
|
2513
|
+
`Dedup store setLastClock failed on ${envelope.topic}[${envelope.partition}] \u2014 processing message anyway (fail-open): ${err.message}`
|
|
2514
|
+
);
|
|
2515
|
+
}
|
|
2258
2516
|
return false;
|
|
2259
2517
|
}
|
|
2260
|
-
async function parseSingleMessage(message,
|
|
2518
|
+
async function parseSingleMessage(message, topic, partition, schemaMap, interceptors, dlq, deps) {
|
|
2261
2519
|
if (!message.value) {
|
|
2262
|
-
deps.logger.warn(`Received empty message from topic ${
|
|
2520
|
+
deps.logger.warn(`Received empty message from topic ${topic}`);
|
|
2263
2521
|
return null;
|
|
2264
2522
|
}
|
|
2265
2523
|
const raw = message.value.toString();
|
|
2266
|
-
const parsed = parseJsonMessage(raw,
|
|
2524
|
+
const parsed = parseJsonMessage(raw, topic, deps.logger);
|
|
2267
2525
|
if (parsed === null) return null;
|
|
2268
2526
|
const headers = decodeHeaders(message.headers);
|
|
2269
2527
|
const validated = await validateWithSchema(
|
|
2270
2528
|
parsed,
|
|
2271
2529
|
raw,
|
|
2272
|
-
|
|
2530
|
+
topic,
|
|
2273
2531
|
schemaMap,
|
|
2274
2532
|
interceptors,
|
|
2275
2533
|
dlq,
|
|
2276
2534
|
{ ...deps, originalHeaders: headers }
|
|
2277
2535
|
);
|
|
2278
2536
|
if (validated === null) return null;
|
|
2279
|
-
return extractEnvelope(validated, headers,
|
|
2537
|
+
return extractEnvelope(validated, headers, topic, partition, message.offset);
|
|
2280
2538
|
}
|
|
2281
2539
|
async function handleEachMessage(payload, opts, deps) {
|
|
2282
|
-
const { topic
|
|
2540
|
+
const { topic, partition, message } = payload;
|
|
2283
2541
|
const {
|
|
2284
2542
|
schemaMap,
|
|
2285
2543
|
handleMessage,
|
|
@@ -2294,12 +2552,12 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2294
2552
|
const nextOffsetStr = (parseInt(message.offset, 10) + 1).toString();
|
|
2295
2553
|
const commitOffset = eos ? async () => {
|
|
2296
2554
|
await eos.consumer.commitOffsets([
|
|
2297
|
-
{ topic
|
|
2555
|
+
{ topic, partition, offset: nextOffsetStr }
|
|
2298
2556
|
]);
|
|
2299
2557
|
} : void 0;
|
|
2300
2558
|
const eosRouteToRetry = eos && retry ? async (rawMsgs, envelopes, delay) => {
|
|
2301
2559
|
const { topic: rtTopic, messages: rtMsgs } = buildRetryTopicPayload(
|
|
2302
|
-
|
|
2560
|
+
topic,
|
|
2303
2561
|
rawMsgs,
|
|
2304
2562
|
1,
|
|
2305
2563
|
retry.maxRetries,
|
|
@@ -2313,7 +2571,7 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2313
2571
|
consumer: eos.consumer,
|
|
2314
2572
|
topics: [
|
|
2315
2573
|
{
|
|
2316
|
-
topic
|
|
2574
|
+
topic,
|
|
2317
2575
|
partitions: [{ partition, offset: nextOffsetStr }]
|
|
2318
2576
|
}
|
|
2319
2577
|
]
|
|
@@ -2329,7 +2587,7 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2329
2587
|
} : void 0;
|
|
2330
2588
|
const envelope = await parseSingleMessage(
|
|
2331
2589
|
message,
|
|
2332
|
-
|
|
2590
|
+
topic,
|
|
2333
2591
|
partition,
|
|
2334
2592
|
schemaMap,
|
|
2335
2593
|
interceptors,
|
|
@@ -2357,10 +2615,10 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2357
2615
|
const ageMs = Date.now() - new Date(envelope.timestamp).getTime();
|
|
2358
2616
|
if (ageMs > opts.messageTtlMs) {
|
|
2359
2617
|
deps.logger.warn(
|
|
2360
|
-
`[KafkaClient] TTL expired on ${
|
|
2618
|
+
`[KafkaClient] TTL expired on ${topic}: age ${ageMs}ms > ${opts.messageTtlMs}ms`
|
|
2361
2619
|
);
|
|
2362
2620
|
if (dlq) {
|
|
2363
|
-
await sendToDlq(
|
|
2621
|
+
await sendToDlq(topic, message.value.toString(), deps, {
|
|
2364
2622
|
error: new Error(`Message TTL expired: age ${ageMs}ms`),
|
|
2365
2623
|
attempt: 0,
|
|
2366
2624
|
originalHeaders: envelope.headers
|
|
@@ -2369,7 +2627,7 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2369
2627
|
} else {
|
|
2370
2628
|
const ttlHandler = opts.onTtlExpired ?? deps.onTtlExpired;
|
|
2371
2629
|
await ttlHandler?.({
|
|
2372
|
-
topic
|
|
2630
|
+
topic,
|
|
2373
2631
|
ageMs,
|
|
2374
2632
|
messageTtlMs: opts.messageTtlMs,
|
|
2375
2633
|
headers: envelope.headers
|
|
@@ -2388,7 +2646,7 @@ async function handleEachMessage(payload, opts, deps) {
|
|
|
2388
2646
|
},
|
|
2389
2647
|
() => handleMessage(envelope)
|
|
2390
2648
|
);
|
|
2391
|
-
return timeoutMs ? wrapWithTimeout(fn, timeoutMs,
|
|
2649
|
+
return timeoutMs ? wrapWithTimeout(fn, timeoutMs, topic) : fn();
|
|
2392
2650
|
},
|
|
2393
2651
|
{
|
|
2394
2652
|
envelope,
|
|
@@ -2632,7 +2890,7 @@ function pauseConsumerImpl(ctx, groupId, assignments) {
|
|
|
2632
2890
|
}
|
|
2633
2891
|
consumer.pause(
|
|
2634
2892
|
assignments.flatMap(
|
|
2635
|
-
({ topic
|
|
2893
|
+
({ topic, partitions }) => partitions.map((p) => ({ topic, partitions: [p] }))
|
|
2636
2894
|
)
|
|
2637
2895
|
);
|
|
2638
2896
|
}
|
|
@@ -2645,25 +2903,25 @@ function resumeConsumerImpl(ctx, groupId, assignments) {
|
|
|
2645
2903
|
}
|
|
2646
2904
|
consumer.resume(
|
|
2647
2905
|
assignments.flatMap(
|
|
2648
|
-
({ topic
|
|
2906
|
+
({ topic, partitions }) => partitions.map((p) => ({ topic, partitions: [p] }))
|
|
2649
2907
|
)
|
|
2650
2908
|
);
|
|
2651
2909
|
}
|
|
2652
|
-
function pauseTopicAllPartitions(ctx, gid,
|
|
2910
|
+
function pauseTopicAllPartitions(ctx, gid, topic) {
|
|
2653
2911
|
const consumer = ctx.consumers.get(gid);
|
|
2654
2912
|
if (!consumer) return;
|
|
2655
2913
|
const assignment = consumer.assignment();
|
|
2656
|
-
const partitions = assignment.filter((a) => a.topic ===
|
|
2914
|
+
const partitions = assignment.filter((a) => a.topic === topic).map((a) => a.partition);
|
|
2657
2915
|
if (partitions.length > 0)
|
|
2658
|
-
consumer.pause(partitions.map((p) => ({ topic
|
|
2916
|
+
consumer.pause(partitions.map((p) => ({ topic, partitions: [p] })));
|
|
2659
2917
|
}
|
|
2660
|
-
function resumeTopicAllPartitions(ctx, gid,
|
|
2918
|
+
function resumeTopicAllPartitions(ctx, gid, topic) {
|
|
2661
2919
|
const consumer = ctx.consumers.get(gid);
|
|
2662
2920
|
if (!consumer) return;
|
|
2663
2921
|
const assignment = consumer.assignment();
|
|
2664
|
-
const partitions = assignment.filter((a) => a.topic ===
|
|
2922
|
+
const partitions = assignment.filter((a) => a.topic === topic).map((a) => a.partition);
|
|
2665
2923
|
if (partitions.length > 0)
|
|
2666
|
-
consumer.resume(partitions.map((p) => ({ topic
|
|
2924
|
+
consumer.resume(partitions.map((p) => ({ topic, partitions: [p] })));
|
|
2667
2925
|
}
|
|
2668
2926
|
|
|
2669
2927
|
// src/client/kafka.client/consumer/start.ts
|
|
@@ -2687,7 +2945,7 @@ async function startConsumerImpl(ctx, topics, handleMessage, options = {}) {
|
|
|
2687
2945
|
retry,
|
|
2688
2946
|
retryTopics: options.retryTopics,
|
|
2689
2947
|
timeoutMs: options.handlerTimeoutMs,
|
|
2690
|
-
wrapWithTimeout: (fn, ms,
|
|
2948
|
+
wrapWithTimeout: (fn, ms, topic) => wrapWithTimeoutWarning(ctx.logger, fn, ms, topic),
|
|
2691
2949
|
deduplication: resolveDeduplicationContext(
|
|
2692
2950
|
ctx,
|
|
2693
2951
|
gid,
|
|
@@ -2738,7 +2996,7 @@ async function startBatchConsumerImpl(ctx, topics, handleBatch, options = {}) {
|
|
|
2738
2996
|
retry,
|
|
2739
2997
|
retryTopics: options.retryTopics,
|
|
2740
2998
|
timeoutMs: options.handlerTimeoutMs,
|
|
2741
|
-
wrapWithTimeout: (fn, ms,
|
|
2999
|
+
wrapWithTimeout: (fn, ms, topic) => wrapWithTimeoutWarning(ctx.logger, fn, ms, topic),
|
|
2742
3000
|
deduplication: resolveDeduplicationContext(
|
|
2743
3001
|
ctx,
|
|
2744
3002
|
gid,
|
|
@@ -2790,10 +3048,10 @@ async function startTransactionalConsumerImpl(ctx, topics, handler, options = {}
|
|
|
2790
3048
|
const txProducer = await createRetryTxProducer(ctx, `${gid}-txc`);
|
|
2791
3049
|
const deps = messageDepsFor(ctx, gid);
|
|
2792
3050
|
await consumer.run({
|
|
2793
|
-
eachMessage: ({ topic
|
|
3051
|
+
eachMessage: ({ topic, partition, message }) => ctx.inFlight.track(async () => {
|
|
2794
3052
|
const envelope = await parseSingleMessage(
|
|
2795
3053
|
message,
|
|
2796
|
-
|
|
3054
|
+
topic,
|
|
2797
3055
|
partition,
|
|
2798
3056
|
schemaMap,
|
|
2799
3057
|
options.interceptors ?? [],
|
|
@@ -2803,7 +3061,7 @@ async function startTransactionalConsumerImpl(ctx, topics, handler, options = {}
|
|
|
2803
3061
|
const nextOffset = String(Number.parseInt(message.offset, 10) + 1);
|
|
2804
3062
|
if (envelope === null) {
|
|
2805
3063
|
await consumer.commitOffsets([
|
|
2806
|
-
{ topic
|
|
3064
|
+
{ topic, partition, offset: nextOffset }
|
|
2807
3065
|
]);
|
|
2808
3066
|
return;
|
|
2809
3067
|
}
|
|
@@ -2843,7 +3101,7 @@ async function startTransactionalConsumerImpl(ctx, topics, handler, options = {}
|
|
|
2843
3101
|
await tx.sendOffsets({
|
|
2844
3102
|
consumer,
|
|
2845
3103
|
topics: [
|
|
2846
|
-
{ topic
|
|
3104
|
+
{ topic, partitions: [{ partition, offset: nextOffset }] }
|
|
2847
3105
|
]
|
|
2848
3106
|
});
|
|
2849
3107
|
await tx.commit();
|
|
@@ -2854,7 +3112,7 @@ async function startTransactionalConsumerImpl(ctx, topics, handler, options = {}
|
|
|
2854
3112
|
} catch {
|
|
2855
3113
|
}
|
|
2856
3114
|
ctx.logger.warn(
|
|
2857
|
-
`startTransactionalConsumer: handler failed on ${
|
|
3115
|
+
`startTransactionalConsumer: handler failed on ${topic}[${partition}]@${message.offset} \u2014 tx aborted, message will be redelivered (${toError(err).message})`
|
|
2858
3116
|
);
|
|
2859
3117
|
throw err;
|
|
2860
3118
|
}
|
|
@@ -2867,8 +3125,8 @@ function stopConsumerByGid(ctx, gid) {
|
|
|
2867
3125
|
return stopConsumerImpl(ctx, gid);
|
|
2868
3126
|
}
|
|
2869
3127
|
|
|
2870
|
-
// src/client/kafka.client/consumer/window.ts
|
|
2871
|
-
async function startWindowConsumerImpl(ctx,
|
|
3128
|
+
// src/client/kafka.client/consumer/features/window.ts
|
|
3129
|
+
async function startWindowConsumerImpl(ctx, topic, handler, options) {
|
|
2872
3130
|
const { maxMessages, maxMs, ...consumerOptions } = options;
|
|
2873
3131
|
if (maxMessages <= 0)
|
|
2874
3132
|
throw new Error("startWindowConsumer: maxMessages must be > 0");
|
|
@@ -2881,6 +3139,7 @@ async function startWindowConsumerImpl(ctx, topic2, handler, options) {
|
|
|
2881
3139
|
const buffer = [];
|
|
2882
3140
|
let flushTimer = null;
|
|
2883
3141
|
let windowStart = 0;
|
|
3142
|
+
const onLost = consumerOptions.onMessageLost ?? ctx.onMessageLost;
|
|
2884
3143
|
const flush = async (trigger) => {
|
|
2885
3144
|
if (flushTimer !== null) {
|
|
2886
3145
|
clearTimeout(flushTimer);
|
|
@@ -2888,22 +3147,37 @@ async function startWindowConsumerImpl(ctx, topic2, handler, options) {
|
|
|
2888
3147
|
}
|
|
2889
3148
|
if (buffer.length === 0) return;
|
|
2890
3149
|
const envelopes = buffer.splice(0);
|
|
2891
|
-
|
|
3150
|
+
try {
|
|
3151
|
+
await handler(envelopes, { trigger, windowStart, windowEnd: Date.now() });
|
|
3152
|
+
} catch (err) {
|
|
3153
|
+
const error = toError(err);
|
|
3154
|
+
ctx.logger.error(
|
|
3155
|
+
`startWindowConsumer: ${trigger}-triggered flush failed \u2014 window of ${envelopes.length} message(s) lost:`,
|
|
3156
|
+
error.stack
|
|
3157
|
+
);
|
|
3158
|
+
for (const envelope of envelopes) {
|
|
3159
|
+
await Promise.resolve(
|
|
3160
|
+
onLost?.({
|
|
3161
|
+
topic: envelope.topic,
|
|
3162
|
+
error,
|
|
3163
|
+
attempt: 0,
|
|
3164
|
+
headers: envelope.headers
|
|
3165
|
+
})
|
|
3166
|
+
).catch(() => {
|
|
3167
|
+
});
|
|
3168
|
+
}
|
|
3169
|
+
}
|
|
2892
3170
|
};
|
|
2893
3171
|
const scheduleFlush = () => {
|
|
2894
3172
|
if (flushTimer !== null) return;
|
|
2895
3173
|
flushTimer = setTimeout(() => {
|
|
2896
3174
|
flushTimer = null;
|
|
2897
|
-
flush("time")
|
|
2898
|
-
ctx.logger.warn(
|
|
2899
|
-
`startWindowConsumer: time-triggered flush error \u2014 ${toError(err).message}`
|
|
2900
|
-
);
|
|
2901
|
-
});
|
|
3175
|
+
void flush("time");
|
|
2902
3176
|
}, maxMs);
|
|
2903
3177
|
};
|
|
2904
3178
|
const handle = await startConsumerImpl(
|
|
2905
3179
|
ctx,
|
|
2906
|
-
[
|
|
3180
|
+
[topic],
|
|
2907
3181
|
async (envelope) => {
|
|
2908
3182
|
if (buffer.length === 0) windowStart = Date.now();
|
|
2909
3183
|
buffer.push(envelope);
|
|
@@ -2914,40 +3188,13 @@ async function startWindowConsumerImpl(ctx, topic2, handler, options) {
|
|
|
2914
3188
|
);
|
|
2915
3189
|
const originalStop = handle.stop.bind(handle);
|
|
2916
3190
|
handle.stop = async () => {
|
|
2917
|
-
|
|
2918
|
-
clearTimeout(flushTimer);
|
|
2919
|
-
flushTimer = null;
|
|
2920
|
-
}
|
|
2921
|
-
if (buffer.length > 0) {
|
|
2922
|
-
const envelopes = buffer.splice(0);
|
|
2923
|
-
await handler(envelopes, {
|
|
2924
|
-
trigger: "time",
|
|
2925
|
-
windowStart,
|
|
2926
|
-
windowEnd: Date.now()
|
|
2927
|
-
}).catch(async (err) => {
|
|
2928
|
-
const error = toError(err);
|
|
2929
|
-
ctx.logger.warn(
|
|
2930
|
-
`startWindowConsumer: shutdown flush error \u2014 ${error.message}`
|
|
2931
|
-
);
|
|
2932
|
-
for (const envelope of envelopes) {
|
|
2933
|
-
await Promise.resolve(
|
|
2934
|
-
ctx.onMessageLost?.({
|
|
2935
|
-
topic: envelope.topic,
|
|
2936
|
-
error,
|
|
2937
|
-
attempt: 0,
|
|
2938
|
-
headers: envelope.headers
|
|
2939
|
-
})
|
|
2940
|
-
).catch(() => {
|
|
2941
|
-
});
|
|
2942
|
-
}
|
|
2943
|
-
});
|
|
2944
|
-
}
|
|
3191
|
+
await flush("time");
|
|
2945
3192
|
return originalStop();
|
|
2946
3193
|
};
|
|
2947
3194
|
return handle;
|
|
2948
3195
|
}
|
|
2949
3196
|
|
|
2950
|
-
// src/client/kafka.client/consumer/routed.ts
|
|
3197
|
+
// src/client/kafka.client/consumer/features/routed.ts
|
|
2951
3198
|
async function startRoutedConsumerImpl(ctx, topics, routing, options) {
|
|
2952
3199
|
const { header, routes, fallback } = routing;
|
|
2953
3200
|
const handleMessage = async (envelope) => {
|
|
@@ -2962,15 +3209,126 @@ async function startRoutedConsumerImpl(ctx, topics, routing, options) {
|
|
|
2962
3209
|
return startConsumerImpl(ctx, topics, handleMessage, options);
|
|
2963
3210
|
}
|
|
2964
3211
|
|
|
2965
|
-
// src/client/kafka.client/consumer/
|
|
2966
|
-
|
|
3212
|
+
// src/client/kafka.client/consumer/features/delayed.ts
|
|
3213
|
+
function delayedTopicName(topic) {
|
|
3214
|
+
return `${topic}.delayed`;
|
|
3215
|
+
}
|
|
3216
|
+
async function startDelayedRelayImpl(ctx, topics, options) {
|
|
3217
|
+
if (topics.length === 0) {
|
|
3218
|
+
throw new Error("startDelayedRelay: at least one topic is required");
|
|
3219
|
+
}
|
|
3220
|
+
const gid = options?.groupId ?? `${ctx.defaultGroupId}-delayed-relay`;
|
|
3221
|
+
if (ctx.runningConsumers.has(gid)) {
|
|
3222
|
+
throw new Error(
|
|
3223
|
+
`startDelayedRelay("${gid}") called twice \u2014 this group is already consuming. Call stopConsumer("${gid}") first or pass a different groupId.`
|
|
3224
|
+
);
|
|
3225
|
+
}
|
|
3226
|
+
const delayedTopics = topics.map(delayedTopicName);
|
|
3227
|
+
for (const t of delayedTopics) await ensureTopic(ctx, t);
|
|
3228
|
+
const txProducer = await createRetryTxProducer(ctx, `${gid}-tx`);
|
|
3229
|
+
let resolveReady;
|
|
3230
|
+
const readyPromise = new Promise((resolve) => {
|
|
3231
|
+
resolveReady = resolve;
|
|
3232
|
+
});
|
|
3233
|
+
const consumer = getOrCreateConsumer(
|
|
3234
|
+
gid,
|
|
3235
|
+
false,
|
|
3236
|
+
false,
|
|
3237
|
+
ctx.consumerOpsDeps,
|
|
3238
|
+
void 0,
|
|
3239
|
+
resolveReady
|
|
3240
|
+
);
|
|
3241
|
+
await consumer.connect();
|
|
3242
|
+
await subscribeWithRetry(consumer, delayedTopics, ctx.logger);
|
|
3243
|
+
await consumer.run({
|
|
3244
|
+
eachMessage: async ({ topic: stagingTopic, partition, message }) => {
|
|
3245
|
+
const nextOffset = {
|
|
3246
|
+
topic: stagingTopic,
|
|
3247
|
+
partition,
|
|
3248
|
+
offset: (parseInt(message.offset, 10) + 1).toString()
|
|
3249
|
+
};
|
|
3250
|
+
if (!message.value) {
|
|
3251
|
+
await consumer.commitOffsets([nextOffset]);
|
|
3252
|
+
return;
|
|
3253
|
+
}
|
|
3254
|
+
const headers = decodeHeaders(message.headers);
|
|
3255
|
+
const target = headers[HEADER_DELAYED_TARGET] ?? stagingTopic.replace(/\.delayed$/, "");
|
|
3256
|
+
const until = parseInt(
|
|
3257
|
+
headers[HEADER_DELAYED_UNTIL] ?? "0",
|
|
3258
|
+
10
|
|
3259
|
+
);
|
|
3260
|
+
const remaining = until - Date.now();
|
|
3261
|
+
if (remaining > 0) {
|
|
3262
|
+
consumer.pause([{ topic: stagingTopic, partitions: [partition] }]);
|
|
3263
|
+
await sleep(remaining);
|
|
3264
|
+
consumer.resume([{ topic: stagingTopic, partitions: [partition] }]);
|
|
3265
|
+
}
|
|
3266
|
+
const forwardHeaders = Object.fromEntries(
|
|
3267
|
+
Object.entries(headers).filter(
|
|
3268
|
+
([k]) => k !== HEADER_DELAYED_UNTIL && k !== HEADER_DELAYED_TARGET
|
|
3269
|
+
)
|
|
3270
|
+
);
|
|
3271
|
+
const tx = await txProducer.transaction();
|
|
3272
|
+
try {
|
|
3273
|
+
await tx.send({
|
|
3274
|
+
topic: target,
|
|
3275
|
+
messages: [
|
|
3276
|
+
{
|
|
3277
|
+
value: message.value.toString(),
|
|
3278
|
+
key: message.key ? message.key.toString() : null,
|
|
3279
|
+
headers: forwardHeaders
|
|
3280
|
+
}
|
|
3281
|
+
]
|
|
3282
|
+
});
|
|
3283
|
+
await tx.sendOffsets({
|
|
3284
|
+
consumer,
|
|
3285
|
+
topics: [
|
|
3286
|
+
{
|
|
3287
|
+
topic: nextOffset.topic,
|
|
3288
|
+
partitions: [
|
|
3289
|
+
{ partition: nextOffset.partition, offset: nextOffset.offset }
|
|
3290
|
+
]
|
|
3291
|
+
}
|
|
3292
|
+
]
|
|
3293
|
+
});
|
|
3294
|
+
await tx.commit();
|
|
3295
|
+
ctx.logger.debug?.(
|
|
3296
|
+
`Delayed message relayed to "${target}" (deadline ${new Date(until).toISOString()})`
|
|
3297
|
+
);
|
|
3298
|
+
} catch (txErr) {
|
|
3299
|
+
try {
|
|
3300
|
+
await tx.abort();
|
|
3301
|
+
} catch {
|
|
3302
|
+
}
|
|
3303
|
+
ctx.logger.error(
|
|
3304
|
+
`Delayed relay to "${target}" failed \u2014 message will be redelivered:`,
|
|
3305
|
+
toError(txErr).stack
|
|
3306
|
+
);
|
|
3307
|
+
}
|
|
3308
|
+
}
|
|
3309
|
+
});
|
|
3310
|
+
ctx.runningConsumers.set(gid, "eachMessage");
|
|
3311
|
+
ctx.logger.log(
|
|
3312
|
+
`Delayed relay started for: ${delayedTopics.join(", ")} (group: ${gid})`
|
|
3313
|
+
);
|
|
3314
|
+
return {
|
|
3315
|
+
groupId: gid,
|
|
3316
|
+
ready: () => readyPromise,
|
|
3317
|
+
stop: async () => {
|
|
3318
|
+
await stopConsumerImpl(ctx, gid);
|
|
3319
|
+
}
|
|
3320
|
+
};
|
|
3321
|
+
}
|
|
3322
|
+
|
|
3323
|
+
// src/client/kafka.client/consumer/features/snapshot.ts
|
|
3324
|
+
async function readSnapshotImpl(ctx, topic, options = {}) {
|
|
2967
3325
|
await ctx.adminOps.ensureConnected();
|
|
2968
3326
|
let offsets;
|
|
2969
3327
|
try {
|
|
2970
|
-
offsets = await ctx.adminOps.admin.fetchTopicOffsets(
|
|
3328
|
+
offsets = await ctx.adminOps.admin.fetchTopicOffsets(topic);
|
|
2971
3329
|
} catch {
|
|
2972
3330
|
ctx.logger.warn(
|
|
2973
|
-
`readSnapshot: could not fetch offsets for "${String(
|
|
3331
|
+
`readSnapshot: could not fetch offsets for "${String(topic)}", returning empty snapshot`
|
|
2974
3332
|
);
|
|
2975
3333
|
return /* @__PURE__ */ new Map();
|
|
2976
3334
|
}
|
|
@@ -2982,7 +3340,7 @@ async function readSnapshotImpl(ctx, topic2, options = {}) {
|
|
|
2982
3340
|
}
|
|
2983
3341
|
if (targets.size === 0) {
|
|
2984
3342
|
ctx.logger.debug?.(
|
|
2985
|
-
`readSnapshot: topic "${String(
|
|
3343
|
+
`readSnapshot: topic "${String(topic)}" is empty \u2014 returning empty snapshot`
|
|
2986
3344
|
);
|
|
2987
3345
|
return /* @__PURE__ */ new Map();
|
|
2988
3346
|
}
|
|
@@ -3001,7 +3359,7 @@ async function readSnapshotImpl(ctx, topic2, options = {}) {
|
|
|
3001
3359
|
});
|
|
3002
3360
|
});
|
|
3003
3361
|
};
|
|
3004
|
-
consumer.connect().then(() => consumer.subscribe({ topics: [
|
|
3362
|
+
consumer.connect().then(() => consumer.subscribe({ topics: [topic] })).then(
|
|
3005
3363
|
() => consumer.run({
|
|
3006
3364
|
eachMessage: async ({ topic: t, partition, message }) => {
|
|
3007
3365
|
if (!remaining.has(partition)) return;
|
|
@@ -3022,7 +3380,7 @@ async function readSnapshotImpl(ctx, topic2, options = {}) {
|
|
|
3022
3380
|
});
|
|
3023
3381
|
});
|
|
3024
3382
|
ctx.logger.log(
|
|
3025
|
-
`readSnapshot: ${snapshot.size} key(s) from "${String(
|
|
3383
|
+
`readSnapshot: ${snapshot.size} key(s) from "${String(topic)}"`
|
|
3026
3384
|
);
|
|
3027
3385
|
return snapshot;
|
|
3028
3386
|
}
|
|
@@ -3059,9 +3417,9 @@ async function checkpointOffsetsImpl(ctx, groupId, checkpointTopic) {
|
|
|
3059
3417
|
await ctx.adminOps.ensureConnected();
|
|
3060
3418
|
const committed = await ctx.adminOps.admin.fetchOffsets({ groupId: gid });
|
|
3061
3419
|
const offsets = [];
|
|
3062
|
-
for (const { topic
|
|
3420
|
+
for (const { topic, partitions } of committed) {
|
|
3063
3421
|
for (const { partition, offset } of partitions) {
|
|
3064
|
-
offsets.push({ topic
|
|
3422
|
+
offsets.push({ topic, partition, offset });
|
|
3065
3423
|
}
|
|
3066
3424
|
}
|
|
3067
3425
|
const savedAt = Date.now();
|
|
@@ -3228,6 +3586,7 @@ var KafkaClient = class {
|
|
|
3228
3586
|
* ```
|
|
3229
3587
|
*/
|
|
3230
3588
|
constructor(clientId, groupId, brokers, options) {
|
|
3589
|
+
validateClientOptions(clientId, groupId, brokers, options);
|
|
3231
3590
|
this.clientId = clientId;
|
|
3232
3591
|
const logger = options?.logger ?? {
|
|
3233
3592
|
log: (msg) => console.log(`[KafkaClient:${clientId}] ${msg}`),
|
|
@@ -3235,7 +3594,8 @@ var KafkaClient = class {
|
|
|
3235
3594
|
error: (msg, ...args) => console.error(`[KafkaClient:${clientId}] ${msg}`, ...args),
|
|
3236
3595
|
debug: (msg, ...args) => console.debug(`[KafkaClient:${clientId}] ${msg}`, ...args)
|
|
3237
3596
|
};
|
|
3238
|
-
const
|
|
3597
|
+
const security = resolveSecurityOptions(options?.security, brokers, logger);
|
|
3598
|
+
const transport = options?.transport ?? new ConfluentTransport(clientId, brokers, security);
|
|
3239
3599
|
const producer = transport.producer();
|
|
3240
3600
|
const runningConsumers = /* @__PURE__ */ new Map();
|
|
3241
3601
|
const consumers = /* @__PURE__ */ new Map();
|
|
@@ -3269,6 +3629,7 @@ var KafkaClient = class {
|
|
|
3269
3629
|
numPartitions: options?.numPartitions ?? 1,
|
|
3270
3630
|
txId: options?.transactionalId ?? `${clientId}-tx`,
|
|
3271
3631
|
clockRecoveryTopics: options?.clockRecovery?.topics ?? [],
|
|
3632
|
+
clockRecoveryTimeoutMs: options?.clockRecovery?.timeoutMs ?? 3e4,
|
|
3272
3633
|
lagThrottleOpts: options?.lagThrottle,
|
|
3273
3634
|
instrumentation: options?.instrumentation ?? [],
|
|
3274
3635
|
onMessageLost: options?.onMessageLost,
|
|
@@ -3278,6 +3639,7 @@ var KafkaClient = class {
|
|
|
3278
3639
|
producer,
|
|
3279
3640
|
txProducer: void 0,
|
|
3280
3641
|
txProducerInitPromise: void 0,
|
|
3642
|
+
_txChain: Promise.resolve(),
|
|
3281
3643
|
retryTxProducers: /* @__PURE__ */ new Map(),
|
|
3282
3644
|
consumers,
|
|
3283
3645
|
runningConsumers,
|
|
@@ -3326,8 +3688,8 @@ var KafkaClient = class {
|
|
|
3326
3688
|
return sendMessageImpl(this.ctx, topicOrDesc, message, options);
|
|
3327
3689
|
}
|
|
3328
3690
|
/** @inheritDoc */
|
|
3329
|
-
async sendTombstone(
|
|
3330
|
-
return sendTombstoneImpl(this.ctx,
|
|
3691
|
+
async sendTombstone(topic, key, headers) {
|
|
3692
|
+
return sendTombstoneImpl(this.ctx, topic, key, headers);
|
|
3331
3693
|
}
|
|
3332
3694
|
async sendBatch(topicOrDesc, messages, options) {
|
|
3333
3695
|
return sendBatchImpl(this.ctx, topicOrDesc, messages, options);
|
|
@@ -3358,7 +3720,7 @@ var KafkaClient = class {
|
|
|
3358
3720
|
}
|
|
3359
3721
|
// ── Consumer: AsyncIterableIterator ──────────────────────────────
|
|
3360
3722
|
/** @inheritDoc */
|
|
3361
|
-
consume(
|
|
3723
|
+
consume(topic, options) {
|
|
3362
3724
|
if (options?.retryTopics) {
|
|
3363
3725
|
throw new Error(
|
|
3364
3726
|
"consume() does not support retryTopics (EOS retry chains). Use startConsumer() with retryTopics: true for guaranteed retry delivery."
|
|
@@ -3367,11 +3729,11 @@ var KafkaClient = class {
|
|
|
3367
3729
|
const gid = options?.groupId ?? this.ctx.defaultGroupId;
|
|
3368
3730
|
const queue = new AsyncQueue(
|
|
3369
3731
|
options?.queueHighWaterMark,
|
|
3370
|
-
() => pauseTopicAllPartitions(this.ctx, gid,
|
|
3371
|
-
() => resumeTopicAllPartitions(this.ctx, gid,
|
|
3732
|
+
() => pauseTopicAllPartitions(this.ctx, gid, topic),
|
|
3733
|
+
() => resumeTopicAllPartitions(this.ctx, gid, topic)
|
|
3372
3734
|
);
|
|
3373
3735
|
const handlePromise = this.startConsumer(
|
|
3374
|
-
[
|
|
3736
|
+
[topic],
|
|
3375
3737
|
async (envelope) => {
|
|
3376
3738
|
queue.push(envelope);
|
|
3377
3739
|
},
|
|
@@ -3393,14 +3755,39 @@ var KafkaClient = class {
|
|
|
3393
3755
|
}
|
|
3394
3756
|
// ── Consumer: windowed ────────────────────────────────────────────
|
|
3395
3757
|
/** @inheritDoc */
|
|
3396
|
-
startWindowConsumer(
|
|
3397
|
-
return startWindowConsumerImpl(this.ctx,
|
|
3758
|
+
startWindowConsumer(topic, handler, options) {
|
|
3759
|
+
return startWindowConsumerImpl(this.ctx, topic, handler, options);
|
|
3398
3760
|
}
|
|
3399
3761
|
// ── Consumer: header routing ──────────────────────────────────────
|
|
3400
3762
|
/** @inheritDoc */
|
|
3401
3763
|
startRoutedConsumer(topics, routing, options) {
|
|
3402
3764
|
return startRoutedConsumerImpl(this.ctx, topics, routing, options);
|
|
3403
3765
|
}
|
|
3766
|
+
// ── Consumer: delayed delivery relay ──────────────────────────────
|
|
3767
|
+
/**
|
|
3768
|
+
* Start a relay that delivers messages produced with
|
|
3769
|
+
* `SendOptions.deliverAfterMs` from `<topic>.delayed` to their target topic
|
|
3770
|
+
* once their deadline passes.
|
|
3771
|
+
*
|
|
3772
|
+
* Forwarding is transactional (produce + source-offset commit are atomic),
|
|
3773
|
+
* so no duplicates are relayed even if the relay crashes mid-forward.
|
|
3774
|
+
* Delivery time is a lower bound — the relay must be running for delayed
|
|
3775
|
+
* messages to be delivered at all.
|
|
3776
|
+
*
|
|
3777
|
+
* @param topics Target topic name(s) whose `<topic>.delayed` staging topics to relay.
|
|
3778
|
+
* @param options Optional `groupId` override (default: `<defaultGroupId>-delayed-relay`).
|
|
3779
|
+
*
|
|
3780
|
+
* @example
|
|
3781
|
+
* ```ts
|
|
3782
|
+
* await kafka.startDelayedRelay(['orders.reminder']);
|
|
3783
|
+
* await kafka.sendMessage('orders.reminder', payload, { deliverAfterMs: 60_000 });
|
|
3784
|
+
* // → delivered to orders.reminder ~60 s later
|
|
3785
|
+
* ```
|
|
3786
|
+
*/
|
|
3787
|
+
async startDelayedRelay(topics, options) {
|
|
3788
|
+
const list = Array.isArray(topics) ? topics : [topics];
|
|
3789
|
+
return startDelayedRelayImpl(this.ctx, list, options);
|
|
3790
|
+
}
|
|
3404
3791
|
// ── Consumer: transactional EOS ───────────────────────────────────
|
|
3405
3792
|
/** @inheritDoc */
|
|
3406
3793
|
async startTransactionalConsumer(topics, handler, options = {}) {
|
|
@@ -3421,10 +3808,10 @@ var KafkaClient = class {
|
|
|
3421
3808
|
}
|
|
3422
3809
|
// ── DLQ replay ────────────────────────────────────────────────────
|
|
3423
3810
|
/** @inheritDoc */
|
|
3424
|
-
async replayDlq(
|
|
3811
|
+
async replayDlq(topic, options = {}) {
|
|
3425
3812
|
await this.ctx.adminOps.ensureConnected();
|
|
3426
3813
|
return replayDlqTopic(
|
|
3427
|
-
|
|
3814
|
+
topic,
|
|
3428
3815
|
{
|
|
3429
3816
|
logger: this.ctx.logger,
|
|
3430
3817
|
fetchTopicOffsets: (t) => this.ctx.adminOps.admin.fetchTopicOffsets(t),
|
|
@@ -3451,8 +3838,8 @@ var KafkaClient = class {
|
|
|
3451
3838
|
}
|
|
3452
3839
|
// ── Snapshot & checkpoint ─────────────────────────────────────────
|
|
3453
3840
|
/** @inheritDoc */
|
|
3454
|
-
async readSnapshot(
|
|
3455
|
-
return readSnapshotImpl(this.ctx,
|
|
3841
|
+
async readSnapshot(topic, options = {}) {
|
|
3842
|
+
return readSnapshotImpl(this.ctx, topic, options);
|
|
3456
3843
|
}
|
|
3457
3844
|
/** @inheritDoc */
|
|
3458
3845
|
async checkpointOffsets(groupId, checkpointTopic) {
|
|
@@ -3464,8 +3851,8 @@ var KafkaClient = class {
|
|
|
3464
3851
|
}
|
|
3465
3852
|
// ── Admin ─────────────────────────────────────────────────────────
|
|
3466
3853
|
/** @inheritDoc */
|
|
3467
|
-
async resetOffsets(groupId,
|
|
3468
|
-
return this.ctx.adminOps.resetOffsets(groupId,
|
|
3854
|
+
async resetOffsets(groupId, topic, position) {
|
|
3855
|
+
return this.ctx.adminOps.resetOffsets(groupId, topic, position);
|
|
3469
3856
|
}
|
|
3470
3857
|
/** @inheritDoc */
|
|
3471
3858
|
async seekToOffset(groupId, assignments) {
|
|
@@ -3492,26 +3879,26 @@ var KafkaClient = class {
|
|
|
3492
3879
|
return this.ctx.adminOps.describeTopics(topics);
|
|
3493
3880
|
}
|
|
3494
3881
|
/** @inheritDoc */
|
|
3495
|
-
async deleteRecords(
|
|
3496
|
-
return this.ctx.adminOps.deleteRecords(
|
|
3882
|
+
async deleteRecords(topic, partitions) {
|
|
3883
|
+
return this.ctx.adminOps.deleteRecords(topic, partitions);
|
|
3497
3884
|
}
|
|
3498
3885
|
// ── Circuit breaker ───────────────────────────────────────────────
|
|
3499
3886
|
/** @inheritDoc */
|
|
3500
|
-
getCircuitState(
|
|
3887
|
+
getCircuitState(topic, partition, groupId) {
|
|
3501
3888
|
return this.ctx.circuitBreaker.getState(
|
|
3502
|
-
|
|
3889
|
+
topic,
|
|
3503
3890
|
partition,
|
|
3504
3891
|
groupId ?? this.ctx.defaultGroupId
|
|
3505
3892
|
);
|
|
3506
3893
|
}
|
|
3507
3894
|
// ── Metrics ───────────────────────────────────────────────────────
|
|
3508
3895
|
/** @inheritDoc */
|
|
3509
|
-
getMetrics(
|
|
3510
|
-
return this.ctx.metrics.getMetrics(
|
|
3896
|
+
getMetrics(topic) {
|
|
3897
|
+
return this.ctx.metrics.getMetrics(topic);
|
|
3511
3898
|
}
|
|
3512
3899
|
/** @inheritDoc */
|
|
3513
|
-
resetMetrics(
|
|
3514
|
-
this.ctx.metrics.resetMetrics(
|
|
3900
|
+
resetMetrics(topic) {
|
|
3901
|
+
this.ctx.metrics.resetMetrics(topic);
|
|
3515
3902
|
}
|
|
3516
3903
|
getClientId() {
|
|
3517
3904
|
return this.clientId;
|
|
@@ -3542,38 +3929,351 @@ var KafkaClient = class {
|
|
|
3542
3929
|
}
|
|
3543
3930
|
};
|
|
3544
3931
|
|
|
3545
|
-
// src/
|
|
3546
|
-
|
|
3932
|
+
// src/cli/dlq.ts
|
|
3933
|
+
var DLQ_SUFFIX = ".dlq";
|
|
3934
|
+
var DlqUsageError = class extends Error {
|
|
3935
|
+
constructor(message) {
|
|
3936
|
+
super(message);
|
|
3937
|
+
this.name = "DlqUsageError";
|
|
3938
|
+
}
|
|
3939
|
+
};
|
|
3940
|
+
var USAGE = `kafka-client-dlq \u2014 dead-letter queue operations
|
|
3941
|
+
|
|
3942
|
+
Usage:
|
|
3943
|
+
kafka-client-dlq ls --brokers <b1,b2> [--prefix <name>]
|
|
3944
|
+
kafka-client-dlq peek --brokers <b1,b2> --topic <name> [--limit <n>]
|
|
3945
|
+
kafka-client-dlq replay --brokers <b1,b2> --topic <name> [--target <t>] [--dry-run] [--from-beginning | --incremental]
|
|
3946
|
+
|
|
3947
|
+
Commands:
|
|
3948
|
+
ls List DLQ topics (ending in .dlq) with per-topic message counts.
|
|
3949
|
+
peek Print up to N messages from <topic>.dlq (offset, x-dlq-* headers, value).
|
|
3950
|
+
replay Re-publish <topic>.dlq messages to their original topic (or --target).
|
|
3951
|
+
|
|
3952
|
+
Options:
|
|
3953
|
+
--brokers <list> Comma-separated broker addresses (required). e.g. localhost:9092
|
|
3954
|
+
--prefix <name> ls: only show DLQ topics whose base name starts with <name>.
|
|
3955
|
+
--topic <name> peek/replay: base topic name (the CLI uses <name>.dlq).
|
|
3956
|
+
--limit <n> peek: max messages to print (default 10).
|
|
3957
|
+
--target <t> replay: override destination topic.
|
|
3958
|
+
--dry-run replay: log without publishing.
|
|
3959
|
+
--from-beginning replay: full replay every call (default).
|
|
3960
|
+
--incremental replay: only messages added since the previous replay.
|
|
3961
|
+
-h, --help Show this help.
|
|
3962
|
+
|
|
3963
|
+
Examples:
|
|
3964
|
+
kafka-client-dlq ls --brokers localhost:9092
|
|
3965
|
+
kafka-client-dlq ls --brokers localhost:9092 --prefix orders
|
|
3966
|
+
kafka-client-dlq peek --brokers localhost:9092 --topic orders.created --limit 5
|
|
3967
|
+
kafka-client-dlq replay --brokers localhost:9092 --topic orders.created --dry-run
|
|
3968
|
+
kafka-client-dlq replay --brokers localhost:9092 --topic orders.created --target orders.manual --incremental
|
|
3969
|
+
`;
|
|
3970
|
+
var VALUE_FLAGS = /* @__PURE__ */ new Set([
|
|
3971
|
+
"--brokers",
|
|
3972
|
+
"--prefix",
|
|
3973
|
+
"--topic",
|
|
3974
|
+
"--limit",
|
|
3975
|
+
"--target"
|
|
3976
|
+
]);
|
|
3977
|
+
var BOOL_FLAGS = /* @__PURE__ */ new Set(["--dry-run", "--from-beginning", "--incremental"]);
|
|
3978
|
+
function parseFlags(args) {
|
|
3979
|
+
const values = {};
|
|
3980
|
+
const bools = /* @__PURE__ */ new Set();
|
|
3981
|
+
for (let i = 0; i < args.length; i++) {
|
|
3982
|
+
const arg = args[i];
|
|
3983
|
+
if (!arg.startsWith("--")) {
|
|
3984
|
+
throw new DlqUsageError(`Unexpected argument: "${arg}"`);
|
|
3985
|
+
}
|
|
3986
|
+
if (VALUE_FLAGS.has(arg)) {
|
|
3987
|
+
const value = args[i + 1];
|
|
3988
|
+
if (value === void 0 || value.startsWith("--")) {
|
|
3989
|
+
throw new DlqUsageError(`Flag "${arg}" requires a value.`);
|
|
3990
|
+
}
|
|
3991
|
+
values[arg] = value;
|
|
3992
|
+
i++;
|
|
3993
|
+
} else if (BOOL_FLAGS.has(arg)) {
|
|
3994
|
+
bools.add(arg);
|
|
3995
|
+
} else {
|
|
3996
|
+
throw new DlqUsageError(`Unknown flag: "${arg}"`);
|
|
3997
|
+
}
|
|
3998
|
+
}
|
|
3999
|
+
return { values, bools };
|
|
4000
|
+
}
|
|
4001
|
+
function requireBrokers(flags) {
|
|
4002
|
+
const raw = flags.values["--brokers"];
|
|
4003
|
+
if (raw === void 0) {
|
|
4004
|
+
throw new DlqUsageError("Missing required flag: --brokers");
|
|
4005
|
+
}
|
|
4006
|
+
const brokers = raw.split(",").map((b) => b.trim()).filter((b) => b.length > 0);
|
|
4007
|
+
if (brokers.length === 0) {
|
|
4008
|
+
throw new DlqUsageError("--brokers must list at least one broker address.");
|
|
4009
|
+
}
|
|
4010
|
+
return brokers;
|
|
4011
|
+
}
|
|
4012
|
+
function requireTopic(flags) {
|
|
4013
|
+
const topic = flags.values["--topic"];
|
|
4014
|
+
if (topic === void 0 || topic.length === 0) {
|
|
4015
|
+
throw new DlqUsageError("Missing required flag: --topic");
|
|
4016
|
+
}
|
|
4017
|
+
return topic;
|
|
4018
|
+
}
|
|
4019
|
+
function parseArgs(argv) {
|
|
4020
|
+
const [command, ...rest] = argv;
|
|
4021
|
+
if (command === void 0 || command === "-h" || command === "--help" || command === "help") {
|
|
4022
|
+
return { command: "help" };
|
|
4023
|
+
}
|
|
4024
|
+
switch (command) {
|
|
4025
|
+
case "ls": {
|
|
4026
|
+
const flags = parseFlags(rest);
|
|
4027
|
+
const brokers = requireBrokers(flags);
|
|
4028
|
+
const prefix = flags.values["--prefix"];
|
|
4029
|
+
return { command: "ls", brokers, prefix };
|
|
4030
|
+
}
|
|
4031
|
+
case "peek": {
|
|
4032
|
+
const flags = parseFlags(rest);
|
|
4033
|
+
const brokers = requireBrokers(flags);
|
|
4034
|
+
const topic = requireTopic(flags);
|
|
4035
|
+
const limit = parseLimit(flags.values["--limit"]);
|
|
4036
|
+
return { command: "peek", brokers, topic, limit };
|
|
4037
|
+
}
|
|
4038
|
+
case "replay": {
|
|
4039
|
+
const flags = parseFlags(rest);
|
|
4040
|
+
const brokers = requireBrokers(flags);
|
|
4041
|
+
const topic = requireTopic(flags);
|
|
4042
|
+
const target = flags.values["--target"];
|
|
4043
|
+
const dryRun = flags.bools.has("--dry-run");
|
|
4044
|
+
if (flags.bools.has("--from-beginning") && flags.bools.has("--incremental")) {
|
|
4045
|
+
throw new DlqUsageError(
|
|
4046
|
+
"--from-beginning and --incremental are mutually exclusive."
|
|
4047
|
+
);
|
|
4048
|
+
}
|
|
4049
|
+
const fromBeginning = !flags.bools.has("--incremental");
|
|
4050
|
+
return { command: "replay", brokers, topic, target, dryRun, fromBeginning };
|
|
4051
|
+
}
|
|
4052
|
+
default:
|
|
4053
|
+
throw new DlqUsageError(`Unknown command: "${command}"`);
|
|
4054
|
+
}
|
|
4055
|
+
}
|
|
4056
|
+
function parseLimit(raw) {
|
|
4057
|
+
if (raw === void 0) return 10;
|
|
4058
|
+
const n = Number(raw);
|
|
4059
|
+
if (!Number.isInteger(n) || n <= 0) {
|
|
4060
|
+
throw new DlqUsageError(`--limit must be a positive integer, got "${raw}".`);
|
|
4061
|
+
}
|
|
4062
|
+
return n;
|
|
4063
|
+
}
|
|
4064
|
+
function countFromWatermarks(watermarks) {
|
|
4065
|
+
let total = 0;
|
|
4066
|
+
for (const { low, high } of watermarks) {
|
|
4067
|
+
const width = Number(high) - Number(low);
|
|
4068
|
+
total += width > 0 ? width : 0;
|
|
4069
|
+
}
|
|
4070
|
+
return total;
|
|
4071
|
+
}
|
|
4072
|
+
function truncate(value, max = 200) {
|
|
4073
|
+
if (value.length <= max) return value;
|
|
4074
|
+
return `${value.slice(0, max)}\u2026 (${value.length} chars)`;
|
|
4075
|
+
}
|
|
4076
|
+
async function runDlqCommand(cmd, deps) {
|
|
4077
|
+
if (cmd.command === "help") {
|
|
4078
|
+
deps.out(USAGE);
|
|
4079
|
+
return { command: "help" };
|
|
4080
|
+
}
|
|
4081
|
+
const client = await deps.createClient(cmd.brokers);
|
|
4082
|
+
try {
|
|
4083
|
+
switch (cmd.command) {
|
|
4084
|
+
case "ls":
|
|
4085
|
+
return await runLs(cmd, client, deps);
|
|
4086
|
+
case "peek":
|
|
4087
|
+
return await runPeek(cmd, client, deps);
|
|
4088
|
+
case "replay":
|
|
4089
|
+
return await runReplay(cmd, client, deps);
|
|
4090
|
+
}
|
|
4091
|
+
} finally {
|
|
4092
|
+
await client.close();
|
|
4093
|
+
}
|
|
4094
|
+
}
|
|
4095
|
+
async function runLs(cmd, client, deps) {
|
|
4096
|
+
const allTopics = await client.listTopics();
|
|
4097
|
+
let dlqTopics = allTopics.filter((t) => t.endsWith(DLQ_SUFFIX));
|
|
4098
|
+
if (cmd.prefix) {
|
|
4099
|
+
const prefix = cmd.prefix;
|
|
4100
|
+
dlqTopics = dlqTopics.filter(
|
|
4101
|
+
(t) => t.slice(0, -DLQ_SUFFIX.length).startsWith(prefix)
|
|
4102
|
+
);
|
|
4103
|
+
}
|
|
4104
|
+
dlqTopics.sort();
|
|
4105
|
+
const counts = [];
|
|
4106
|
+
for (const dlqTopic of dlqTopics) {
|
|
4107
|
+
const watermarks = await client.fetchTopicOffsets(dlqTopic);
|
|
4108
|
+
counts.push({
|
|
4109
|
+
dlqTopic,
|
|
4110
|
+
baseTopic: dlqTopic.slice(0, -DLQ_SUFFIX.length),
|
|
4111
|
+
count: countFromWatermarks(watermarks)
|
|
4112
|
+
});
|
|
4113
|
+
}
|
|
4114
|
+
if (counts.length === 0) {
|
|
4115
|
+
deps.out(
|
|
4116
|
+
cmd.prefix ? `No DLQ topics found matching prefix "${cmd.prefix}".` : "No DLQ topics found."
|
|
4117
|
+
);
|
|
4118
|
+
} else {
|
|
4119
|
+
const width = Math.max(...counts.map((c) => c.dlqTopic.length));
|
|
4120
|
+
deps.out(`${"TOPIC".padEnd(width)} MESSAGES`);
|
|
4121
|
+
for (const c of counts) {
|
|
4122
|
+
deps.out(`${c.dlqTopic.padEnd(width)} ${c.count}`);
|
|
4123
|
+
}
|
|
4124
|
+
const total = counts.reduce((s, c) => s + c.count, 0);
|
|
4125
|
+
deps.out(`${counts.length} DLQ topic(s), ${total} message(s) total.`);
|
|
4126
|
+
}
|
|
4127
|
+
return { command: "ls", topics: counts };
|
|
4128
|
+
}
|
|
4129
|
+
async function runPeek(cmd, client, deps) {
|
|
4130
|
+
const dlqTopic = `${cmd.topic}${DLQ_SUFFIX}`;
|
|
4131
|
+
const messages = await client.peekMessages(dlqTopic, cmd.limit);
|
|
4132
|
+
if (messages.length === 0) {
|
|
4133
|
+
deps.out(`No messages in ${dlqTopic}.`);
|
|
4134
|
+
return { command: "peek", printed: 0 };
|
|
4135
|
+
}
|
|
4136
|
+
deps.out(`Peeking up to ${cmd.limit} message(s) from ${dlqTopic}:`);
|
|
4137
|
+
let printed = 0;
|
|
4138
|
+
for (const env of messages) {
|
|
4139
|
+
if (printed >= cmd.limit) break;
|
|
4140
|
+
deps.out("");
|
|
4141
|
+
deps.out(
|
|
4142
|
+
`\u2500 offset ${env.offset} \xB7 partition ${env.partition} \xB7 ${env.timestamp}`
|
|
4143
|
+
);
|
|
4144
|
+
const dlqHeaders = Object.entries(env.headers).filter(([k]) => k.startsWith("x-dlq-")).sort(([a], [b]) => a.localeCompare(b));
|
|
4145
|
+
for (const [k, v] of dlqHeaders) {
|
|
4146
|
+
deps.out(` ${k}: ${truncate(String(v), 500)}`);
|
|
4147
|
+
}
|
|
4148
|
+
deps.out(` value: ${truncate(JSON.stringify(env.payload))}`);
|
|
4149
|
+
printed++;
|
|
4150
|
+
}
|
|
4151
|
+
deps.out("");
|
|
4152
|
+
deps.out(`Printed ${printed} message(s).`);
|
|
4153
|
+
return { command: "peek", printed };
|
|
4154
|
+
}
|
|
4155
|
+
async function runReplay(cmd, client, deps) {
|
|
4156
|
+
const options = {
|
|
4157
|
+
dryRun: cmd.dryRun,
|
|
4158
|
+
fromBeginning: cmd.fromBeginning
|
|
4159
|
+
};
|
|
4160
|
+
if (cmd.target !== void 0) options.targetTopic = cmd.target;
|
|
4161
|
+
const mode = cmd.fromBeginning ? "full" : "incremental";
|
|
4162
|
+
const targetDesc = cmd.target ? ` \u2192 ${cmd.target}` : " \u2192 original topic";
|
|
4163
|
+
deps.out(
|
|
4164
|
+
`Replaying ${cmd.topic}${DLQ_SUFFIX}${targetDesc} (${mode}${cmd.dryRun ? ", dry-run" : ""})\u2026`
|
|
4165
|
+
);
|
|
4166
|
+
const { replayed, skipped } = await client.replayDlq(cmd.topic, options);
|
|
4167
|
+
deps.out(
|
|
4168
|
+
cmd.dryRun ? `Dry-run: ${replayed} message(s) would be replayed, ${skipped} skipped.` : `Replayed ${replayed} message(s), ${skipped} skipped.`
|
|
4169
|
+
);
|
|
4170
|
+
return { command: "replay", replayed, skipped, dryRun: cmd.dryRun };
|
|
4171
|
+
}
|
|
4172
|
+
|
|
4173
|
+
// src/cli/index.ts
|
|
4174
|
+
var CLIENT_ID = `dlq-cli-${process.pid}`;
|
|
4175
|
+
var GROUP_ID = `${CLIENT_ID}-peek`;
|
|
4176
|
+
function createRealClient(brokers) {
|
|
4177
|
+
const transport = new ConfluentTransport(CLIENT_ID, brokers);
|
|
4178
|
+
const kafka = new KafkaClient(
|
|
4179
|
+
CLIENT_ID,
|
|
4180
|
+
GROUP_ID,
|
|
4181
|
+
brokers,
|
|
4182
|
+
{ transport, autoCreateTopics: false, strictSchemas: false }
|
|
4183
|
+
);
|
|
4184
|
+
const admin = transport.admin();
|
|
4185
|
+
let adminConnected = false;
|
|
4186
|
+
async function ensureAdmin() {
|
|
4187
|
+
if (!adminConnected) {
|
|
4188
|
+
await admin.connect();
|
|
4189
|
+
adminConnected = true;
|
|
4190
|
+
}
|
|
4191
|
+
}
|
|
3547
4192
|
return {
|
|
3548
|
-
|
|
3549
|
-
|
|
3550
|
-
|
|
3551
|
-
|
|
3552
|
-
|
|
3553
|
-
|
|
3554
|
-
|
|
3555
|
-
|
|
3556
|
-
|
|
3557
|
-
|
|
4193
|
+
async listTopics() {
|
|
4194
|
+
const status = await kafka.checkStatus();
|
|
4195
|
+
if (status.status === "down") {
|
|
4196
|
+
throw new Error(`Broker unreachable: ${status.error}`);
|
|
4197
|
+
}
|
|
4198
|
+
return status.topics;
|
|
4199
|
+
},
|
|
4200
|
+
async fetchTopicOffsets(topic) {
|
|
4201
|
+
await ensureAdmin();
|
|
4202
|
+
return admin.fetchTopicOffsets(topic);
|
|
4203
|
+
},
|
|
4204
|
+
async peekMessages(dlqTopic, limit) {
|
|
4205
|
+
const collected = [];
|
|
4206
|
+
const iterator = kafka.consume(dlqTopic, {
|
|
4207
|
+
groupId: `${dlqTopic}.dlq-peek-${Date.now()}`,
|
|
4208
|
+
fromBeginning: true
|
|
4209
|
+
});
|
|
4210
|
+
await ensureAdmin();
|
|
4211
|
+
const watermarks = await admin.fetchTopicOffsets(dlqTopic);
|
|
4212
|
+
const available = watermarks.reduce(
|
|
4213
|
+
(sum, w) => sum + Math.max(0, Number(w.high) - Number(w.low)),
|
|
4214
|
+
0
|
|
4215
|
+
);
|
|
4216
|
+
const target = Math.min(limit, available);
|
|
4217
|
+
if (target === 0) {
|
|
4218
|
+
await iterator.return?.();
|
|
4219
|
+
return collected;
|
|
4220
|
+
}
|
|
4221
|
+
try {
|
|
4222
|
+
for await (const env of iterator) {
|
|
4223
|
+
collected.push(env);
|
|
4224
|
+
if (collected.length >= target) break;
|
|
4225
|
+
}
|
|
4226
|
+
} finally {
|
|
4227
|
+
await iterator.return?.();
|
|
4228
|
+
}
|
|
4229
|
+
return collected;
|
|
4230
|
+
},
|
|
4231
|
+
replayDlq(topic, options) {
|
|
4232
|
+
return kafka.replayDlq(topic, options);
|
|
4233
|
+
},
|
|
4234
|
+
async close() {
|
|
4235
|
+
if (adminConnected) {
|
|
4236
|
+
await admin.disconnect().catch(() => {
|
|
4237
|
+
});
|
|
4238
|
+
}
|
|
4239
|
+
await kafka.disconnect().catch(() => {
|
|
4240
|
+
});
|
|
4241
|
+
}
|
|
3558
4242
|
};
|
|
3559
4243
|
}
|
|
4244
|
+
async function main() {
|
|
4245
|
+
let cmd;
|
|
4246
|
+
try {
|
|
4247
|
+
cmd = parseArgs(process.argv.slice(2));
|
|
4248
|
+
} catch (err) {
|
|
4249
|
+
if (err instanceof DlqUsageError) {
|
|
4250
|
+
process.stderr.write(`Error: ${err.message}
|
|
3560
4251
|
|
|
3561
|
-
|
|
3562
|
-
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
|
|
3568
|
-
|
|
3569
|
-
|
|
3570
|
-
|
|
3571
|
-
|
|
3572
|
-
|
|
3573
|
-
|
|
3574
|
-
|
|
3575
|
-
|
|
3576
|
-
|
|
3577
|
-
|
|
3578
|
-
|
|
3579
|
-
|
|
4252
|
+
`);
|
|
4253
|
+
process.stderr.write(USAGE);
|
|
4254
|
+
return 2;
|
|
4255
|
+
}
|
|
4256
|
+
throw err;
|
|
4257
|
+
}
|
|
4258
|
+
try {
|
|
4259
|
+
await runDlqCommand(cmd, {
|
|
4260
|
+
createClient: createRealClient,
|
|
4261
|
+
out: (line) => process.stdout.write(`${line}
|
|
4262
|
+
`)
|
|
4263
|
+
});
|
|
4264
|
+
return 0;
|
|
4265
|
+
} catch (err) {
|
|
4266
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
4267
|
+
process.stderr.write(`Error: ${message}
|
|
4268
|
+
`);
|
|
4269
|
+
return 1;
|
|
4270
|
+
}
|
|
4271
|
+
}
|
|
4272
|
+
main().then((code) => {
|
|
4273
|
+
process.exitCode = code;
|
|
4274
|
+
}).catch((err) => {
|
|
4275
|
+
process.stderr.write(`Fatal: ${err?.stack ?? err}
|
|
4276
|
+
`);
|
|
4277
|
+
process.exitCode = 1;
|
|
4278
|
+
});
|
|
4279
|
+
//# sourceMappingURL=index.js.map
|